Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Sync with llama.cpp upstream
Justine Tunneyd228e01d70a7
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex c0c9f17f..fe88e322 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- c780e75305dba1f67691a8dc0e8bc8425838a452- 2024-05-07+ 201cc11afa0a1950e1f632390b2ac6c937a0d8f0+ 2024-05-21LOCAL MODIFICATIONSdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex a1eb0c23..1c801035 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -72,7 +72,6 @@#include <sys/syslimits.h>#endif#define LLAMA_CURL_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083-#define LLAMA_CURL_MAX_HEADER_LENGTH 256#endif // LLAMA_USE_CURLusing json = nlohmann::ordered_json;@@ -950,6 +949,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.interactive = true;return true;}+ if (arg == "--interactive-specials") {+ params.interactive_specials = true;+ return true;+ }if (arg == "--embedding") {params.embedding = true;return true;@@ -962,6 +965,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.instruct = true;return true;}+ if (arg == "-cnv" || arg == "--conversation") {+ params.conversation = true;+ return true;+ }if (arg == "-cml" || arg == "--chatml") {params.chatml = true;return true;@@ -1082,6 +1089,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa}return true;}+ if (arg == "--rpc") {+ if (++i >= argc) {+ invalid_param = true;+ return true;+ }+ params.rpc_servers = argv[i];+ return true;+ }if (arg == "--no-mmap") {params.use_mmap = false;return true;@@ -1368,7 +1383,12 @@ void gpt_params_handle_model_default(gpt_params & params) {}params.hf_file = params.model;} else if (params.model.empty()) {- params.model = "models/" + string_split(params.hf_file, '/').back();+ std::string cache_directory = get_cache_directory();+ const bool success = create_directory_with_parents(cache_directory);+ if (!success) {+ throw std::runtime_error("failed to create cache directory: " + cache_directory);+ }+ params.model = cache_directory + string_split(params.hf_file, '/').back();}} else if (!params.model_url.empty()) {if (params.model.empty()) {@@ -1448,7 +1468,9 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -h, --help show this help message and exit\n");printf(" --version show version and build info\n");printf(" -i, --interactive run in interactive mode\n");+ printf(" --interactive-specials allow special tokens in user text, in interactive mode\n");printf(" --interactive-first run in interactive mode and wait for input right away\n");+ printf(" -cnv, --conversation run in conversation mode (does not print special tokens and suffix/prefix)\n");printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n");printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n");printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n");@@ -1852,6 +1874,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &if (params.n_gpu_layers != -1) {mparams.n_gpu_layers = params.n_gpu_layers;}+ mparams.rpc_servers = params.rpc_servers.c_str();mparams.main_gpu = params.main_gpu;mparams.split_mode = params.split_mode;mparams.tensor_split = params.tensor_split;@@ -1996,18 +2019,18 @@ static bool llama_download_file(const std::string & url, const std::string & pattry {metadata_in >> metadata;fprintf(stderr, "%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(), metadata.dump().c_str());- if (metadata.contains("url") && metadata["url"].is_string()) {- auto previous_url = metadata["url"].get<std::string>();+ if (metadata.contains("url") && metadata.at("url").is_string()) {+ auto previous_url = metadata.at("url").get<std::string>();if (previous_url != url) {fprintf(stderr, "%s: Model URL mismatch: %s != %s\n", __func__, url.c_str(), previous_url.c_str());return false;}}- if (metadata.contains("etag") && metadata["etag"].is_string()) {- etag = metadata["etag"];+ if (metadata.contains("etag") && metadata.at("etag").is_string()) {+ etag = metadata.at("etag");}- if (metadata.contains("lastModified") && metadata["lastModified"].is_string()) {- last_modified = metadata["lastModified"];+ if (metadata.contains("lastModified") && metadata.at("lastModified").is_string()) {+ last_modified = metadata.at("lastModified");}} catch (const nlohmann::json::exception & e) {fprintf(stderr, "%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());@@ -2528,6 +2551,31 @@ bool create_directory_with_parents(const std::string & path) {#endif // _WIN32}+std::string get_cache_directory() {+ std::string cache_directory = "";+ if (getenv("LLAMA_CACHE")) {+ cache_directory = std::getenv("LLAMA_CACHE");+ if (cache_directory.back() != DIRECTORY_SEPARATOR) {+ cache_directory += DIRECTORY_SEPARATOR;+ }+ } else {+ if (IsLinux()) { // [jart]+ if (std::getenv("XDG_CACHE_HOME")) {+ cache_directory = std::getenv("XDG_CACHE_HOME");+ } else {+ cache_directory = std::getenv("HOME") + std::string("/.cache/");+ }+ } else if (IsXnu()) {+ cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");+ } else if (IsWindows()) {+ cache_directory = std::getenv("APPDATA");+ }+ cache_directory += "llama.cpp";+ cache_directory += DIRECTORY_SEPARATOR;+ }+ return cache_directory;+}+void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector<float> & data) {if (data.empty()) {fprintf(stream, "%s:\n", prop_name);@@ -2565,7 +2613,7 @@ void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const chasize_t pos_start = 0;size_t pos_found = 0;- if (!data_str.empty() && (std::isspace(data_str[0]) || std::isspace(data_str.back()))) {+ if (std::isspace(data_str[0]) || std::isspace(data_str.back())) {data_str = std::regex_replace(data_str, std::regex("\n"), "\\n");data_str = std::regex_replace(data_str, std::regex("\""), "\\\"");data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)");@@ -2677,6 +2725,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const ldump_string_yaml_multiline(stream, "in_suffix", params.input_prefix.c_str());fprintf(stream, "instruct: %s # default: false\n", params.instruct ? "true" : "false");fprintf(stream, "interactive: %s # default: false\n", params.interactive ? "true" : "false");+ fprintf(stream, "interactive_specials: %s # default: false\n", params.interactive_specials ? "true" : "false");fprintf(stream, "interactive_first: %s # default: false\n", params.interactive_first ? "true" : "false");fprintf(stream, "keep: %d # default: 0\n", params.n_keep);fprintf(stream, "logdir: %s # default: unset (no logging)\n", params.logdir.c_str());diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 60c2aca4..e58fb1c4 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -87,6 +87,7 @@ struct gpt_params {float yarn_beta_slow = 1.0f; // YaRN high correction dimint32_t yarn_orig_ctx = 0; // YaRN original context lengthfloat defrag_thold = -1.0f; // KV cache defragmentation threshold+ std::string rpc_servers = ""; // comma separated list of RPC serversggml_backend_sched_eval_callback cb_eval = nullptr;void * cb_eval_user_data = nullptr;@@ -145,6 +146,8 @@ struct gpt_params {bool random_prompt = false; // do not randomize prompt if none providedbool use_color = false; // use color to distinguish generations and inputsbool interactive = false; // interactive mode+ bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode+ bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix)bool chatml = false; // chatml mode (used for models trained on chatml syntax)bool prompt_cache_all = false; // save user input and generations to prompt cachebool prompt_cache_ro = false; // open the prompt cache read-only and do not update it@@ -283,6 +286,7 @@ bool llama_should_add_bos_token(const llama_model * model);//bool create_directory_with_parents(const std::string & path);+std::string get_cache_directory();void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector<float> & data);void dump_vector_int_yaml(FILE * stream, const char * prop_name, const std::vector<int> & data);void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const char * data);diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex c32aa53f..fbf7bbb0 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -12,9 +12,6 @@#include <stdlib.h>#include <string.h>-#ifndef NDEBUG-#define NDEBUG // [jart] delete printf debugging-#endif#define MAX(a, b) ((a) > (b) ? (a) : (b))@@ -1188,9 +1185,9 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, ststatic char * fmt_size(size_t size) {static char buffer[128];if (size >= 1024*1024) {- sprintf(buffer, "%zuM", size/1024/1024);+ snprintf(buffer, sizeof(buffer), "%zuM", size/1024/1024);} else {- sprintf(buffer, "%zuK", size/1024);+ snprintf(buffer, sizeof(buffer), "%zuK", size/1024);}return buffer;}@@ -1901,7 +1898,6 @@ void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * ttensor->buffer = buffer;tensor->data = (char *)tensor->view_src->data + tensor->view_offs;- tensor->backend = tensor->view_src->backend;ggml_backend_buffer_init_tensor(buffer, tensor);}diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 599fe31f..2129858a 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -16,14 +16,39 @@#include <map>#include <memory>#include <mutex>+#include <stdarg.h>#include <stdint.h>#include <stdio.h>+#include <stdlib.h>#include <string>#include <vector>++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP acc.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP acc.cuh+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP common.cuh+//+////////////////////////////////////////////////////////////////////////////////++#include "ggml.h"#include "ggml-cuda.h"+#if defined(GGML_USE_HIPBLAS)#define GGML_COMMON_DECL_HIP#define GGML_COMMON_IMPL_HIP@@ -403,6 +428,115 @@ typedef float dfloat; // dequantize floattypedef float2 dfloat2;#endif //GGML_CUDA_F16+#if defined(GGML_USE_HIPBLAS)+#define __CUDA_ARCH__ 1300++#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \+ defined(__gfx1150__) || defined(__gfx1151__)+#define RDNA3+#endif++#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \+ defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)+#define RDNA2+#endif++#ifndef __has_builtin+ #define __has_builtin(x) 0+#endif++typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+#if __has_builtin(__builtin_elementwise_sub_sat)+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);+ return reinterpret_cast<const int &>(c);+#else+ int8x4_t c;+ int16_t tmp;+#pragma unroll+ for (int i = 0; i < 4; i++) {+ tmp = va[i] - vb[i];+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();+ c[i] = tmp;+ }+ return reinterpret_cast<int &>(c);+#endif // __has_builtin(__builtin_elementwise_sub_sat)+}++static __device__ __forceinline__ int __vsub4(const int a, const int b) {+ return __vsubss4(a, b);+}++static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);+ unsigned int c;+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;+ }+ return c;+}++static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {+#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__)+ c = __builtin_amdgcn_sdot4(a, b, c, false);+#elif defined(RDNA3)+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);+#elif defined(__gfx1010__) || defined(__gfx900__)+ int tmp1;+ int tmp2;+ asm("\n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ "+ : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)+ : "v"(a), "v"(b)+ );+#else+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];+#endif+ return c;+}++#if defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000+// __shfl_xor() for half2 was added in ROCm 5.6+static __device__ __forceinline__ half2 __shfl_xor(half2 var, int laneMask, int width) {+ typedef union half2_b32 {+ half2 val;+ int b32;+ } half2_b32_t;+ half2_b32_t tmp;+ tmp.val = var;+ tmp.b32 = __shfl_xor(tmp.b32, laneMask, width);+ return tmp.val;+}+#endif // defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000+#endif // defined(GGML_USE_HIPBLAS)++#define FP16_AVAILABLE (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL++#define FP16_MMA_AVAILABLE !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA++static bool fast_fp16_available(const int cc) {+ return cc >= CC_PASCAL && cc != 610;+}++static bool fp16_mma_available(const int cc) {+ return cc < CC_OFFSET_AMD && cc >= CC_VOLTA;+}+[[noreturn]]static __device__ void no_device_code(const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {@@ -444,16 +578,28 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {}static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+#if FP16_AVAILABLE++#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));- }- return a;+ for (int mask = 16; mask > 0; mask >>= 1) {+ const half2 a_other = __shfl_xor_sync(0xffffffff, a, mask, 32);+ reinterpret_cast<half&>(a.x) += __low2half(a_other);+ reinterpret_cast<half&>(a.y) += __high2half(a_other);+ }+ return a;#else- GGML_UNUSED(a);- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));+ }+ return a;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)++#else+ NO_DEVICE_CODE;+ return a;+#endif // FP16_AVAILABLE}static __device__ __forceinline__ float warp_reduce_max(float x) {@@ -465,20 +611,21 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {}static __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+#if FP16_AVAILABLE-#if CUDART_VERSION >= CUDART_HMAX- return __hmax(a, b);+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX+ return __float2half(fmaxf(__half2float(a), __half2float(b)));#else- return __half2float(a) > __half2float(b) ? a : b;-#endif // CUDART_VERSION >= CUDART_HMAX+ return __hmax(a, b);+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX#else- GGML_UNUSED(a);- GGML_UNUSED(b);- NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX+ NO_DEVICE_CODE;+ GGML_UNUSED(b);+ return a;+#endif // FP16_AVAILABLE}+static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))@@ -486,8 +633,8 @@ static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const halreturn __hmax2(a, b);#elsehalf2 ret;- reinterpret_cast<half&>(ret.x) = __low2float(a) > __low2float(b) ? __low2half(a) : __low2half(b);- reinterpret_cast<half&>(ret.y) = __high2float(a) > __high2float(b) ? __high2half(a) : __high2half(b);+ reinterpret_cast<half&>(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b)));+ reinterpret_cast<half&>(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));return ret;#endif // CUDART_VERSION >= CUDART_HMAX@@ -495,17 +642,9 @@ static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const halGGML_UNUSED(a);GGML_UNUSED(b);NO_DEVICE_CODE;-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION < CUDART_HMAX+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))}-#define FP16_AVAILABLE (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) ? \- defined(RDNA1) || defined(RDNA2) || defined(RDNA3) : __CUDA_ARCH__ >= CC_PASCAL)-#define FP16_MMA_AVAILABLE (!(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA)-#if FP16_MMA_AVAILABLE-#include <mma.h>-#endif--#if defined(GGML_MINIMIZE_CODE_SIZE) && FP16_AVAILABLE // [jart]static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL#pragma unroll@@ -518,7 +657,6 @@ static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {NO_DEVICE_CODE;#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL}-#endif // [jart]#if CUDART_VERSION < CUDART_HMASKstatic __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {@@ -528,94 +666,22 @@ static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half}#endif // CUDART_VERSION < 12000-#if defined(GGML_USE_HIPBLAS)-#define __CUDA_ARCH__ 1300--#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \- defined(__gfx1150__) || defined(__gfx1151__)-#define RDNA3-#endif--#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \- defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)-#define RDNA2-#endif--#ifndef __has_builtin- #define __has_builtin(x) 0-#endif--typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));-typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));-static __device__ __forceinline__ int __vsubss4(const int a, const int b) {- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);-#if __has_builtin(__builtin_elementwise_sub_sat)- const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);- return reinterpret_cast<const int &>(c);-#else- int8x4_t c;- int16_t tmp;-#pragma unroll- for (int i = 0; i < 4; i++) {- tmp = va[i] - vb[i];- if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();- if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();- c[i] = tmp;- }- return reinterpret_cast<int &>(c);-#endif // __has_builtin(__builtin_elementwise_sub_sat)-}--static __device__ __forceinline__ int __vsub4(const int a, const int b) {- return __vsubss4(a, b);-}--static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {- const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);- const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);- unsigned int c;- uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);-#pragma unroll- for (int i = 0; i < 4; ++i) {- vc[i] = va[i] == vb[i] ? 0xff : 0x00;- }- return c;-}--static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {-#if defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__)- c = __builtin_amdgcn_sdot4(a, b, c, false);-#elif defined(RDNA3)- c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);-#elif defined(__gfx1010__) || defined(__gfx900__)- int tmp1;- int tmp2;- asm("\n \- v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \- v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \- v_add3_u32 %0, %1, %2, %0 \n \- v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \- v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \- v_add3_u32 %0, %1, %2, %0 \n \- "- : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)- : "v"(a), "v"(b)- );-#else- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);- c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];-#endif- return c;-}-#endif // defined(GGML_USE_HIPBLAS)-// TODO: move to ggml-common.hstatic const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);+static __device__ __forceinline__ float get_alibi_slope(+ const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1+) {+ if (max_bias <= 0.0f) {+ return 1.0f;+ }+ const float base = h < n_head_log2 ? m0 : m1;+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ return powf(base, exph);+}//////////////////////@@ -700,6 +766,43 @@ struct ggml_tensor_extra_gpu {cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs};++#if (CUDART_VERSION >= 12000) && defined(GGML_CUDA_USE_GRAPHS)+#define USE_CUDA_GRAPH+#endif++struct ggml_graph_node_properties {+ void * node_address;+ ggml_op node_op;+ int64_t ne[GGML_MAX_DIMS];+ size_t nb[GGML_MAX_DIMS];+ void * src_address[GGML_MAX_SRC];+};++struct ggml_cuda_graph {+#ifdef USE_CUDA_GRAPH+ ~ggml_cuda_graph() {+ if (instance != nullptr) {+ CUDA_CHECK(cudaGraphExecDestroy(instance));+ }+ if (graph != nullptr) {+ CUDA_CHECK(cudaGraphDestroy(graph));+ }+ }+ cudaGraph_t graph = nullptr;+ cudaGraphExec_t instance = nullptr;+ size_t num_nodes = 0;+ std::vector<cudaGraphNode_t> nodes;+ std::vector<cudaKernelNodeParams> params;+ bool disable_due_to_gpu_arch = false;+ bool disable_due_to_too_many_updates = false;+ bool disable_due_to_failed_graph_capture = false;+ int number_consecutive_updates = 0;+ std::vector<ggml_graph_node_properties> ggml_graph_properties;+ std::vector<char **> updated_kernel_arg;+#endif+};+struct ggml_backend_cuda_context {int device;std::string name;@@ -708,6 +811,8 @@ struct ggml_backend_cuda_context {cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};+ std::unique_ptr<ggml_cuda_graph> cuda_graph;+explicit ggml_backend_cuda_context(int device) :device(device),name(GGML_CUDA_NAME + std::to_string(device)) {@@ -775,5493 +880,6712 @@ struct ggml_backend_cuda_context {void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-#define CUDA_ALIBI_BLOCK_SIZE 32--void ggml_cuda_op_alibi(ggml_backend_cuda_context & ctx, ggml_tensor * dst);--#define CUDA_ARANGE_BLOCK_SIZE 256--void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,+ const int ne10, const int ne11, const int ne12,+ const int nb1, const int nb2, int offset) {+ const int i = blockDim.x * blockIdx.x + threadIdx.x;+ if (i >= ne) {+ return;+ }+ int src1_idx = i - offset;+ int oz = src1_idx / nb2;+ int oy = (src1_idx - (oz * nb2)) / nb1;+ int ox = src1_idx % nb1;+ if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {+ dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];+ } else {+ dst[i] = x[i];+ }+}-void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,+ const int ne10, const int ne11, const int ne12,+ const int nb1, const int nb2, const int offset, cudaStream_t stream) {+ int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;+ acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);+}-void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ const float * src0_d = (const float *)src0->data;+ const float * src1_d = (const float *)src1->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();-#define CUDA_CLAMP_BLOCK_SIZE 256+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported-void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ int nb1 = dst->op_params[0] / 4; // 4 bytes of float32+ int nb2 = dst->op_params[1] / 4; // 4 bytes of float32+ // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused+ int offset = dst->op_params[3] / 4; // offset in bytes-#define CUDA_CONCAT_BLOCK_SIZE 256+ acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, stream);+}-void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP arange.cu+//+////////////////////////////////////////////////////////////////////////////////-#define CUDA_DEQUANTIZE_BLOCK_SIZE 256-template<typename T>-using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, cudaStream_t stream);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP arange.cuh+//+////////////////////////////////////////////////////////////////////////////////-typedef to_t_cuda_t<float> to_fp32_cuda_t;-typedef to_t_cuda_t<half> to_fp16_cuda_t;-to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);+#define CUDA_ARANGE_BLOCK_SIZE 256-to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);+void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-#define CUDA_CPY_BLOCK_SIZE 32+static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {+ // blockIDx.x: idx of ne0 / BLOCK_SIZE+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }+ dst[nidx] = start + step * nidx;+}-void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);+static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {+ int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;+ arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);+}-void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();-#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32+ GGML_ASSERT(dst->type == GGML_TYPE_F32);-void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ float start;+ float stop;+ float step;+ memcpy(&start, (float *)dst->op_params + 0, sizeof(float));+ memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));+ memcpy(&step, (float *)dst->op_params + 2, sizeof(float));-// dmmv = dequantize_mul_mat_vec+ int64_t steps = (int64_t)ceil((stop - start) / step);+ GGML_ASSERT(ggml_nelements(dst) == steps);-// TODO: remove this?-#ifndef GGML_CUDA_DMMV_X-#define GGML_CUDA_DMMV_X 32-#endif+ arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);+}-#ifndef GGML_CUDA_MMV_Y-#define GGML_CUDA_MMV_Y 1-#endif+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP argsort.cu+//+////////////////////////////////////////////////////////////////////////////////-void ggml_cuda_op_dequantize_mul_mat_vec(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream);-#ifndef GGML_MINIMIZE_CODE_SIZE // [jart]-void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-#endif+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP argsort.cuh+//+////////////////////////////////////////////////////////////////////////////////-#define CUDA_GET_ROWS_BLOCK_SIZE 256-void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-#define CUDA_IM2COL_BLOCK_SIZE 256+template<typename T>+static inline __device__ void ggml_cuda_swap(T & a, T & b) {+ T tmp = a;+ a = b;+ b = tmp;+}-void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template<ggml_sort_order order>+static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {+ // bitonic sort+ int col = threadIdx.x;+ int row = blockIdx.y;-void ggml_cuda_op_mul_mat_q(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream);+ if (col >= ncols_pad) {+ return;+ }-bool ggml_cuda_supports_mmq(enum ggml_type type);+ const float * x_row = x + row * ncols;+ extern __shared__ int dst_row[];-void ggml_cuda_op_mul_mat_vec_q(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream);+ // initialize indices+ dst_row[col] = col;-void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ __syncthreads();-void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ for (int k = 2; k <= ncols_pad; k *= 2) {+ for (int j = k / 2; j > 0; j /= 2) {+ int ixj = col ^ j;+ if (ixj > col) {+ if ((col & k) == 0) {+ if (dst_row[col] >= ncols ||+ (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?+ x_row[dst_row[col]] > x_row[dst_row[ixj]] :+ x_row[dst_row[col]] < x_row[dst_row[ixj]]))+ ) {+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);+ }+ } else {+ if (dst_row[ixj] >= ncols ||+ (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?+ x_row[dst_row[col]] < x_row[dst_row[ixj]] :+ x_row[dst_row[col]] > x_row[dst_row[ixj]]))+ ) {+ ggml_cuda_swap(dst_row[col], dst_row[ixj]);+ }+ }+ }+ __syncthreads();+ }+ }-void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ // copy the result to dst without the padding+ if (col < ncols) {+ dst[row * ncols + col] = dst_row[col];+ }+}-#define CUDA_PAD_BLOCK_SIZE 256+static int next_power_of_2(int x) {+ int n = 1;+ while (n < x) {+ n *= 2;+ }+ return n;+}-void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {+ // bitonic sort requires ncols to be power of 2+ const int ncols_pad = next_power_of_2(ncols);-#define CUDA_POOL2D_BLOCK_SIZE 256+ const dim3 block_dims(ncols_pad, 1, 1);+ const dim3 block_nums(1, nrows, 1);+ const size_t shared_mem = ncols_pad * sizeof(int);-void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);-#define CUDA_QUANTIZE_BLOCK_SIZE 256+ if (order == GGML_SORT_ORDER_ASC) {+ k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);+ } else if (order == GGML_SORT_ORDER_DESC) {+ k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);+ } else {+ GGML_ASSERT(false);+ }+}-void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream);+void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const float * src0_d = (const float *)src0->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();-#define CUDA_ROPE_BLOCK_SIZE 256+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_I32);+ GGML_ASSERT(ggml_is_contiguous(src0));-void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ const int64_t ncols = src0->ne[0];+ const int64_t nrows = ggml_nrows(src0);-#define CUDA_SCALE_BLOCK_SIZE 256+ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];-void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ argsort_f32_i32_cuda(src0_d, (int *)dst_d, ncols, nrows, order, stream);+}-#define CUDA_SOFT_MAX_BLOCK_SIZE 1024+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP binbcast.cu+//+////////////////////////////////////////////////////////////////////////////////-void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP binbcast.cuh+//+////////////////////////////////////////////////////////////////////////////////-#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256-void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-#define CUDA_GELU_BLOCK_SIZE 256-#define CUDA_SILU_BLOCK_SIZE 256-#define CUDA_TANH_BLOCK_SIZE 256-#define CUDA_RELU_BLOCK_SIZE 256-#define CUDA_HARDSIGMOID_BLOCK_SIZE 256-#define CUDA_HARDSWISH_BLOCK_SIZE 256-#define CUDA_SQR_BLOCK_SIZE 256+static __device__ __forceinline__ float op_repeat(const float a, const float b) {+ return b;+ GGML_UNUSED(a);+}-void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __device__ __forceinline__ float op_add(const float a, const float b) {+ return a + b;+}-void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __device__ __forceinline__ float op_mul(const float a, const float b) {+ return a * b;+}-void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __device__ __forceinline__ float op_div(const float a, const float b) {+ return a / b;+}-void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s00,*/ int s01, int s02, int s03,+ /*int s10,*/ int s11, int s12, int s13) {+ const int i0s = blockDim.x*blockIdx.x + threadIdx.x;+ const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);+ const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;+ const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;-void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {+ return;+ }-void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;-void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i3*s3 + i2*s2 + i1*s1;-void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;-void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {+ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);+ }+}-#define CUDA_UPSCALE_BLOCK_SIZE 256+template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>+static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,+ int ne0, int ne1, int ne2, int ne3,+ int ne10, int ne11, int ne12, int ne13,+ /*int s0, */ int s1, int s2, int s3,+ /*int s00,*/ int s01, int s02, int s03,+ /*int s10,*/ int s11, int s12, int s13) {-void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+ const int i = blockDim.x*blockIdx.x + threadIdx.x;-static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const block_q4_0 * x = (const block_q4_0 *) vx;+ const int i3 = i/(ne2*ne1*ne0);+ const int i2 = (i/(ne1*ne0)) % ne2;+ const int i1 = (i/ne0) % ne1;+ const int i0 = i % ne0;- const dfloat d = x[ib].d;+ if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {+ return;+ }- const int vui = x[ib].qs[iqs];+ const int i11 = i1 % ne11;+ const int i12 = i2 % ne12;+ const int i13 = i3 % ne13;- v.x = vui & 0xF;- v.y = vui >> 4;+ const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;+ const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;+ const size_t i_dst = i3*s3 + i2*s2 + i1*s1;-#ifdef GGML_CUDA_F16- v = __hsub2(v, {8.0f, 8.0f});- v = __hmul2(v, {d, d});-#else- v.x = (v.x - 8.0f) * d;- v.y = (v.y - 8.0f) * d;-#endif // GGML_CUDA_F16+ const src0_t * src0_row = src0 + i_src0;+ const src1_t * src1_row = src1 + i_src1;+ dst_t * dst_row = dst + i_dst;++ const int i10 = i0 % ne10;+ dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);}-static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const block_q4_1 * x = (const block_q4_1 *) vx;+template<float (*bin_op)(const float, const float)>+struct bin_bcast_cuda {+ template<typename src0_t, typename src1_t, typename dst_t>+ void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,+ const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,+ cudaStream_t stream) {- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);+ GGML_TENSOR_BINARY_OP_LOCALS- const int vui = x[ib].qs[iqs];+ int nr0 = ne10/ne0;+ int nr1 = ne11/ne1;+ int nr2 = ne12/ne2;+ int nr3 = ne13/ne3;- v.x = vui & 0xF;- v.y = vui >> 4;+ int nr[4] = { nr0, nr1, nr2, nr3 };-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-#else- v.x = (v.x * d) + m;- v.y = (v.y * d) + m;-#endif // GGML_CUDA_F16-}+ // collapse dimensions until first broadcast dimension+ int64_t cne[] = {ne0, ne1, ne2, ne3};+ int64_t cne0[] = {ne00, ne01, ne02, ne03};+ int64_t cne1[] = {ne10, ne11, ne12, ne13};-static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const block_q5_0 * x = (const block_q5_0 *) vx;+ size_t cnb[] = {nb0, nb1, nb2, nb3};+ size_t cnb0[] = {nb00, nb01, nb02, nb03};+ size_t cnb1[] = {nb10, nb11, nb12, nb13};- const dfloat d = x[ib].d;+ auto collapse = [](int64_t cne[]) {+ cne[0] *= cne[1];+ cne[1] = cne[2];+ cne[2] = cne[3];+ cne[3] = 1;+ };- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));+ auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {+ cnb[1] *= cne[1];+ cnb[2] *= cne[2];+ cnb[3] *= cne[3];+ };- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {+ for (int i = 0; i < 4; i++) {+ if (nr[i] != 1) {+ break;+ }+ if (i > 0) {+ collapse_nb(cnb, cne);+ collapse_nb(cnb0, cne0);+ collapse_nb(cnb1, cne1);+ collapse(cne);+ collapse(cne0);+ collapse(cne1);+ }+ }+ }- v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = ((x[ib].qs[iqs] >> 4) | xh_1);+ {+ int64_t ne0 = cne[0];+ int64_t ne1 = cne[1];+ int64_t ne2 = cne[2];+ int64_t ne3 = cne[3];-#ifdef GGML_CUDA_F16- v = __hsub2(v, {16.0f, 16.0f});- v = __hmul2(v, {d, d});-#else- v.x = (v.x - 16.0f) * d;- v.y = (v.y - 16.0f) * d;-#endif // GGML_CUDA_F16-}+ //int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);+ //int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);+ //int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);+ //int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);-static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const block_q5_1 * x = (const block_q5_1 *) vx;+ int64_t ne10 = cne1[0];+ int64_t ne11 = cne1[1];+ int64_t ne12 = cne1[2];+ int64_t ne13 = cne1[3];- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);+ size_t nb0 = cnb[0];+ size_t nb1 = cnb[1];+ size_t nb2 = cnb[2];+ size_t nb3 = cnb[3];- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));+ size_t nb00 = cnb0[0];+ size_t nb01 = cnb0[1];+ size_t nb02 = cnb0[2];+ size_t nb03 = cnb0[3];- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;+ size_t nb10 = cnb1[0];+ size_t nb11 = cnb1[1];+ size_t nb12 = cnb1[2];+ size_t nb13 = cnb1[3];- v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = ((x[ib].qs[iqs] >> 4) | xh_1);+ size_t s0 = nb0 / sizeof(dst_t);+ size_t s1 = nb1 / sizeof(dst_t);+ size_t s2 = nb2 / sizeof(dst_t);+ size_t s3 = nb3 / sizeof(dst_t);-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-#else- v.x = (v.x * d) + m;- v.y = (v.y * d) + m;-#endif // GGML_CUDA_F16-}+ size_t s10 = nb10 / sizeof(src1_t);+ size_t s11 = nb11 / sizeof(src1_t);+ size_t s12 = nb12 / sizeof(src1_t);+ size_t s13 = nb13 / sizeof(src1_t);-static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const block_q8_0 * x = (const block_q8_0 *) vx;+ size_t s00 = nb00 / sizeof(src0_t);+ size_t s01 = nb01 / sizeof(src0_t);+ size_t s02 = nb02 / sizeof(src0_t);+ size_t s03 = nb03 / sizeof(src0_t);- const dfloat d = x[ib].d;+ GGML_ASSERT(nb0 % sizeof(dst_t) == 0);+ GGML_ASSERT(nb1 % sizeof(dst_t) == 0);+ GGML_ASSERT(nb2 % sizeof(dst_t) == 0);+ GGML_ASSERT(nb3 % sizeof(dst_t) == 0);- v.x = x[ib].qs[iqs + 0];- v.y = x[ib].qs[iqs + 1];+ GGML_ASSERT(nb00 % sizeof(src0_t) == 0);+ GGML_ASSERT(nb01 % sizeof(src0_t) == 0);+ GGML_ASSERT(nb02 % sizeof(src0_t) == 0);+ GGML_ASSERT(nb03 % sizeof(src0_t) == 0);-#ifdef GGML_CUDA_F16- v = __hmul2(v, {d, d});-#else- v.x *= d;- v.y *= d;-#endif // GGML_CUDA_F16-}+ GGML_ASSERT(nb10 % sizeof(src1_t) == 0);+ GGML_ASSERT(nb11 % sizeof(src1_t) == 0);+ GGML_ASSERT(nb12 % sizeof(src1_t) == 0);+ GGML_ASSERT(nb13 % sizeof(src1_t) == 0);-static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment+ GGML_ASSERT(s0 == 1);+ GGML_ASSERT(s00 == 1);+ GGML_ASSERT(s10 == 1);- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;+ const int block_size = 128;- return x32;-}+ int64_t hne0 = std::max(ne0/2LL, 1LL);-static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {- const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment+ dim3 block_dims;+ block_dims.x = std::min<unsigned int>(hne0, block_size);+ block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);+ block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;+ dim3 block_nums(+ (hne0 + block_dims.x - 1) / block_dims.x,+ (ne1 + block_dims.y - 1) / block_dims.y,+ (ne2*ne3 + block_dims.z - 1) / block_dims.z+ );- return x32;+ if (block_nums.z > 65535) {+ // this is the maximum number of blocks in z dimension, fallback to 1D grid kernel+ int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;+ k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s00, */ s01, s02, s03,+ /* s10, */ s11, s12, s13);+ } else {+ k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne0, ne1, ne2, ne3,+ ne10, ne11, ne12, ne13,+ /* s0, */ s1, s2, s3,+ /* s00, */ s01, s02, s03,+ /* s10, */ s11, s12, s13);+ }+ }+ }+};++template<class op>+static void ggml_cuda_op_bin_bcast(+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {++ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {+ op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {+ op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);+ } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {+ op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);+ } else {+ fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,+ ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));+ GGML_ASSERT(false);+ }}-static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());}-static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {- return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());}+void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());+}-// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called-// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q+void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());+}-#define VDR_Q4_0_Q8_1_MMVQ 2-#define VDR_Q4_0_Q8_1_MMQ 4+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP clamp.cu+//+////////////////////////////////////////////////////////////////////////////////-template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(- const int * v, const int * u, const float & d4, const half2 & ds8) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP clamp.cuh+//+////////////////////////////////////////////////////////////////////////////////-#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2*i+0], sumi);- sumi = __dp4a(vi1, u[2*i+1], sumi);+#define CUDA_CLAMP_BLOCK_SIZE 256++void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++static __global__ void clamp_f32(const float * x, float * dst, const float min, const float max, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;}- const float2 ds8f = __half22float2(ds8);+ dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);+}- // second part effectively subtracts 8 from each quant value- return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+static void clamp_f32_cuda(const float * x, float * dst, const float min, const float max, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;+ clamp_f32<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);}-#define VDR_Q4_1_Q8_1_MMVQ 2-#define VDR_Q4_1_Q8_1_MMQ 4-template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(- const int * v, const int * u, const half2 & dm4, const half2 & ds8) {+void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const float * src0_d = (const float *)src0->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);-#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;+ float min;+ float max;+ memcpy(&min, dst->op_params, sizeof(float));+ memcpy(&max, (float *) dst->op_params + 1, sizeof(float));- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2*i+0], sumi);- sumi = __dp4a(vi1, u[2*i+1], sumi);- }+ clamp_f32_cuda(src0_d, dst_d, min, max, ggml_nelements(src0), stream);+}-#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm4, ds8));- const float d4d8 = tmp.x;- const float m4s8 = tmp.y;-#else- const float2 dm4f = __half22float2(dm4);- const float2 ds8f = __half22float2(ds8);- const float d4d8 = dm4f.x * ds8f.x;- const float m4s8 = dm4f.y * ds8f.y;-#endif // GGML_CUDA_F16+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP concat.cu+//+////////////////////////////////////////////////////////////////////////////////- // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it- return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}-#define VDR_Q5_0_Q8_1_MMVQ 2-#define VDR_Q5_0_Q8_1_MMQ 4+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP concat.cuh+//+////////////////////////////////////////////////////////////////////////////////-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(- const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+#define CUDA_CONCAT_BLOCK_SIZE 256-#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values+void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {+ int nidx = threadIdx.x + blockIdx.x * blockDim.x;+ if (nidx >= ne0) {+ return;+ }+ // operation+ int offset_dst =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ if (blockIdx.z < ne02) { // src0+ int offset_src =+ nidx ++ blockIdx.y * ne0 ++ blockIdx.z * ne0 * gridDim.y;+ dst[offset_dst] = x[offset_src];+ } else {+ int offset_src =+ nidx ++ blockIdx.y * ne0 ++ (blockIdx.z - ne02) * ne0 * gridDim.y;+ dst[offset_dst] = y[offset_src];}+}- const float2 ds8f = __half22float2(ds8);+static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {+ int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;+ dim3 gridDim(num_blocks, ne1, ne2);+ concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);+}- // second part effectively subtracts 16 from each quant value- return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ const float * src0_d = (const float *)src0->data;+ const float * src1_d = (const float *)src1->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);++ for (int i3 = 0; i3 < dst->ne[3]; i3++) {+ concat_f32_cuda(src0_d + i3 * (src0->nb[3] / 4), src1_d + i3 * (src1->nb[3] / 4), dst_d + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], stream);+ }}-#define VDR_Q5_1_Q8_1_MMVQ 2-#define VDR_Q5_1_Q8_1_MMQ 4+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP convert.cu+//+////////////////////////////////////////////////////////////////////////////////-template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(- const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP convert.cuh+//+////////////////////////////////////////////////////////////////////////////////-#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values- }+#define CUDA_DEQUANTIZE_BLOCK_SIZE 256++template<typename T>+using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, cudaStream_t stream);++typedef to_t_cuda_t<float> to_fp32_cuda_t;+typedef to_t_cuda_t<half> to_fp16_cuda_t;++to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);++to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP dequantize.cuh+//+////////////////////////////////////////////////////////////////////////////////+++static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const block_q4_0 * x = (const block_q4_0 *) vx;++ const dfloat d = x[ib].d;++ const int vui = x[ib].qs[iqs];++ v.x = vui & 0xF;+ v.y = vui >> 4;#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm5, ds8));- const float d5d8 = tmp.x;- const float m5s8 = tmp.y;+ v = __hsub2(v, {8.0f, 8.0f});+ v = __hmul2(v, {d, d});#else- const float2 dm5f = __half22float2(dm5);- const float2 ds8f = __half22float2(ds8);- const float d5d8 = dm5f.x * ds8f.x;- const float m5s8 = dm5f.y * ds8f.y;+ v.x = (v.x - 8.0f) * d;+ v.y = (v.y - 8.0f) * d;#endif // GGML_CUDA_F16+}- // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it- return sumi*d5d8 + m5s8 / (QI5_1 / vdr);+static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const block_q4_1 * x = (const block_q4_1 *) vx;++ const dfloat d = __low2half(x[ib].dm);+ const dfloat m = __high2half(x[ib].dm);++ const int vui = x[ib].qs[iqs];++ v.x = vui & 0xF;+ v.y = vui >> 4;+#ifdef GGML_CUDA_F16+ v = __hmul2(v, {d, d});+ v = __hadd2(v, {m, m});#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ v.x = (v.x * d) + m;+ v.y = (v.y * d) + m;+#endif // GGML_CUDA_F16}-#define VDR_Q8_0_Q8_1_MMVQ 2-#define VDR_Q8_0_Q8_1_MMQ 8+static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const block_q5_0 * x = (const block_q5_0 *) vx;-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(- const int * v, const int * u, const float & d8_0, const float & d8_1) {+ const dfloat d = x[ib].d;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));-#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }+ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;- return d8_0*d8_1 * sumi;+ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);++#ifdef GGML_CUDA_F16+ v = __hsub2(v, {16.0f, 16.0f});+ v = __hmul2(v, {d, d});#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ v.x = (v.x - 16.0f) * d;+ v.y = (v.y - 16.0f) * d;+#endif // GGML_CUDA_F16}-template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(- const int * v, const int * u, const half2 & dm8, const half2 & ds8) {+static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const block_q5_1 * x = (const block_q5_1 *) vx;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;+ const dfloat d = __low2half(x[ib].dm);+ const dfloat m = __high2half(x[ib].dm);-#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }+ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));++ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;++ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);#ifdef GGML_CUDA_F16- const float2 tmp = __half22float2(__hmul2(dm8, ds8));- const float d8d8 = tmp.x;- const float m8s8 = tmp.y;+ v = __hmul2(v, {d, d});+ v = __hadd2(v, {m, m});#else- const float2 dm8f = __half22float2(dm8);- const float2 ds8f = __half22float2(ds8);- const float d8d8 = dm8f.x * ds8f.x;- const float m8s8 = dm8f.y * ds8f.y;+ v.x = (v.x * d) + m;+ v.y = (v.y * d) + m;#endif // GGML_CUDA_F16-- // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it- return sumi*d8d8 + m8s8 / (QI8_1 / vdr);-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define VDR_Q2_K_Q8_1_MMVQ 1-#define VDR_Q2_K_Q8_1_MMQ 2+static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const block_q8_0 * x = (const block_q8_0 *) vx;-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(- const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const half2 & dm2, const float * __restrict__ d8) {+ const dfloat d = x[ib].d;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ v.x = x[ib].qs[iqs + 0];+ v.y = x[ib].qs[iqs + 1];-#pragma unroll- for (int i = 0; i < QR2_K; ++i) {- const int sc = scales[2*i];+#ifdef GGML_CUDA_F16+ v = __hmul2(v, {d, d});+#else+ v.x *= d;+ v.y *= d;+#endif // GGML_CUDA_F16+}- const int vi = (v >> (2*i)) & 0x03030303;+#define CUDA_Q8_0_NE_ALIGN 2048- sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {+ const int64_t i = (int64_t)2*(blockDim.x*blockIdx.x + threadIdx.x);- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;- sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values+ if (i >= k) {+ return;}- const float2 dm2f = __half22float2(dm2);+ const int64_t ib = i/qk; // block index+ const int64_t iqs = (i%qk)/qr; // quant index+ const int64_t iybs = i - i%qk; // y block start index+ const int64_t y_offset = qr == 1 ? 1 : qk/2;- return dm2f.x*sumf_d - dm2f.y*sumf_m;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ // dequantize+ dfloat2 v;+ dequantize_kernel(vx, ib, iqs, v);++ y[iybs + iqs + 0] = v.x;+ y[iybs + iqs + y_offset] = v.y;}-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const half2 & dm2, const float & d8) {+template <bool need_check>+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {+#if __CUDA_ARCH__ >= CC_PASCAL+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi_d = 0;- int sumi_m = 0;+ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;+ const int * x0 = ((int *) vx) + blockIdx.x * nint;+ half2 * y2 = (half2 *) (y + i0);++ __shared__ int vals[nint];#pragma unroll- for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {- int sumi_d_sc = 0;+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {+ break;+ }- const int sc = scales[i0 / (QI8_1/2)];+ const int ix = ix0 + threadIdx.x;+ vals[ix] = x0[ix];+ }- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;+ __syncthreads();#pragma unroll- for (int i = i0; i < i0 + QI8_1/2; ++i) {- sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product- sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {+ return;}- sumi_d += sumi_d_sc * (sc & 0xF);- }-- const float2 dm2f = __half22float2(dm2);+ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);+ const half d = *b0;+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];- return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);+ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));+ }#else+ GGML_UNUSED(vx);+ GGML_UNUSED(y);+ GGML_UNUSED(k);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+#endif // __CUDA_ARCH__ >= CC_PASCAL}-#define VDR_Q3_K_Q8_1_MMVQ 1-#define VDR_Q3_K_Q8_1_MMQ 2--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(- const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,- const int & scale_offset, const float & d3, const float * __restrict__ d8) {+template<typename dst_t>+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf = 0.0f;+ const int64_t i = blockIdx.x;-#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- const int isc = scale_offset + 2*i;-- const int isc_low = isc % (QK_K/32);- const int sc_shift_low = 4 * (isc / (QK_K/32));- const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;-- const int isc_high = isc % (QK_K/64);- const int sc_shift_high = 2 * (isc / (QK_K/64));- const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;-- const int sc = (sc_low | sc_high) - 32;+ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }- const int vil = (vl >> (2*i)) & 0x03030303;+ dst_t * y = yy + 256*i + 32*ir + 4*il;- const int vih = ((vh >> i) << 2) & 0x04040404;+ const block_q4_0 * x = (const block_q4_0 *)vx + ib;+ const float d = __half2float(x->d);+ const float dm = -8*d;- const int vi = __vsubss4(vil, vih);+ const uint8_t * q = x->qs + 4*il;- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product+ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d * (q[l] & 0xF) + dm;+ y[l+16] = d * (q[l] >> 4) + dm;}-- return d3 * sumf;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,- const float & d3, const float & d8) {--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- int sumi = 0;--#pragma unroll- for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {- int sumi_sc = 0;+template<typename dst_t>+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {- for (int i = i0; i < i0 + QI8_1/2; ++i) {- sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product- }+ const int64_t i = blockIdx.x;- sumi += sumi_sc * scales[i0 / (QI8_1/2)];+ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t ib = 8*i + ir;+ if (ib >= nb32) {+ return;}- return d3*d8 * sumi;-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ dst_t * y = yy + 256*i + 32*ir + 4*il;-#define VDR_Q4_K_Q8_1_MMVQ 2-#define VDR_Q4_K_Q8_1_MMQ 8+ const block_q4_1 * x = (const block_q4_1 *)vx + ib;+ const float2 d = __half22float2(x->dm);-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {+ const uint8_t * q = x->qs + 4*il;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d.x * (q[l] & 0xF) + d.y;+ y[l+16] = d.x * (q[l] >> 4) + d.y;+ }+}-#pragma unroll- for (int i = 0; i < QR4_K; ++i) {- const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;- const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;+//================================== k-quants- const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u+template<typename dst_t>+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values- }+ const int64_t i = blockIdx.x;+ const block_q2_K * x = (const block_q2_K *) vx;- const float2 dm4f = __half22float2(dm4);+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t n = tid/32;+ const int64_t l = tid - 32*n;+ const int64_t is = 8*n + l/16;- return dm4f.x*sumf_d - dm4f.y*sumf_m;+ const uint8_t q = x[i].qs[32*n + l];+ dst_t * y = yy + i*QK_K + 128*n;+ float dall = __low2half(x[i].dm);+ float dmin = __high2half(x[i].dm);+ y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);+ y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);+ y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);+ y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}--// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {+ const int64_t is = tid/16; // 0 or 1+ const int64_t il = tid%16; // 0...15+ const uint8_t q = x[i].qs[il] >> (2*is);+ dst_t * y = yy + i*QK_K + 16*is + il;+ float dall = __low2half(x[i].dm);+ float dmin = __high2half(x[i].dm);+ y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);+ y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4);+#endif-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+}-#pragma unroll- for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {- int sumi_d = 0;+template<typename dst_t>+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {-#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product- }+ const int64_t i = blockIdx.x;+ const block_q3_K * x = (const block_q3_K *) vx;- const float2 ds8f = __half22float2(ds8[i]);+#if QK_K == 256+ const int64_t r = threadIdx.x/4;+ const int64_t tid = r/2;+ const int64_t is0 = r%2;+ const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);+ const int64_t n = tid / 4;+ const int64_t j = tid - 4*n;- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val- }+ uint8_t m = 1 << (4*n + j);+ int64_t is = 8*n + 2*j + is0;+ int shift = 2*j;- const float2 dm4f = __half22float2(dm4);+ int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :+ is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :+ is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :+ (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);+ float d_all = x[i].d;+ float dl = d_all * (us - 32);- return dm4f.x*sumf_d - dm4f.y*sumf_m;+ dst_t * y = yy + i*QK_K + 128*n + 32*j;+ const uint8_t * q = x[i].qs + 32*n;+ const uint8_t * hm = x[i].hmask;+ for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+ const int64_t tid = threadIdx.x;+ const int64_t is = tid/16; // 0 or 1+ const int64_t il = tid%16; // 0...15+ const int64_t im = il/8; // 0...1+ const int64_t in = il%8; // 0...7-#define VDR_Q5_K_Q8_1_MMVQ 2-#define VDR_Q5_K_Q8_1_MMQ 8+ dst_t * y = yy + i*QK_K + 16*is + il;-// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(- const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {+ const uint8_t q = x[i].qs[il] >> (2*is);+ const uint8_t h = x[i].hmask[in] >> (2*is + im);+ const float d = (float)x[i].d;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+ if (is == 0) {+ y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));+ y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));+ } else {+ y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));+ y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));+ }+#endif-#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;- const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;+}- const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;- const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;+#if QK_K == 256+static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {+ if (j < 4) {+ d = q[j] & 63; m = q[j + 4] & 63;+ } else {+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);+ }+}+#endif- const int v0i = vl0i | vh0i;- const int v1i = vl1i | vh1i;+template<typename dst_t>+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q4_K * x = (const block_q4_K *) vx;- const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u+ const int64_t i = blockIdx.x;- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]);+#if QK_K == 256+ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t is = 2*il;+ const int64_t n = 4;- }+ dst_t * y = yy + i*QK_K + 64*il + n*ir;- const float2 dm5f = __half22float2(dm5);+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- return dm5f.x*sumf_d - dm5f.y*sumf_m;+ const uint8_t * q = x[i].qs + 32*il + n*ir;+ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[i].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[i].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;+ for (int l = 0; l < n; ++l) {+ y[l + 0] = d1 * (q[l] & 0xF) - m1;+ y[l +32] = d2 * (q[l] >> 4) - m2;+ }#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}--// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,- const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {+ const int64_t tid = threadIdx.x;+ const uint8_t * q = x[i].qs;+ dst_t * y = yy + i*QK_K;+ const float d = (float)x[i].dm[0];+ const float m = (float)x[i].dm[1];+ y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);+ y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4);+#endif+}-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;- float sumf_m = 0.0f;+template<typename dst_t>+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q5_K * x = (const block_q5_K *) vx;-#pragma unroll- for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {- int sumi_d = 0;+ const int64_t i = blockIdx.x;-#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product- }+#if QK_K == 256+ // assume 64 threads - this is very slightly better than the one below+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/16; // il is in 0...3+ const int64_t ir = tid%16; // ir is in 0...15+ const int64_t is = 2*il; // is is in 0...6- const float2 ds8f = __half22float2(ds8[i]);+ dst_t * y = yy + i*QK_K + 64*il + 2*ir;- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val- }+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- const float2 dm4f = __half22float2(dm4);+ const uint8_t * ql = x[i].qs + 32*il + 2*ir;+ const uint8_t * qh = x[i].qh + 2*ir;- return dm4f.x*sumf_d - dm4f.y*sumf_m;+ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[i].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[i].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;+ uint8_t hm = 1 << (2*il);+ y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;+ y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;+ hm <<= 1;+ y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;+ y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+ const int64_t tid = threadIdx.x;+ const uint8_t q = x[i].qs[tid];+ const int64_t im = tid/8; // 0...3+ const int64_t in = tid%8; // 0...7+ const int64_t is = tid/16; // 0 or 1+ const uint8_t h = x[i].qh[in] >> im;+ const float d = x[i].d;+ dst_t * y = yy + i*QK_K + tid;+ y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));+ y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16));+#endif}-#define VDR_Q6_K_Q8_1_MMVQ 1-#define VDR_Q6_K_Q8_1_MMQ 8--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(- const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,- const float & d, const float * __restrict__ d8) {--#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf = 0.0f;+template<typename dst_t>+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const block_q6_K * x = (const block_q6_K *) vx;-#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- const int sc = scales[4*i];+ const int64_t i = blockIdx.x;+#if QK_K == 256- const int vil = (vl >> (4*i)) & 0x0F0F0F0F;+ // assume 64 threads - this is very slightly better than the one below+ const int64_t tid = threadIdx.x;+ const int64_t ip = tid/32; // ip is 0 or 1+ const int64_t il = tid - 32*ip; // 0...32+ const int64_t is = 8*ip + il/16;- const int vih = ((vh >> (4*i)) << 4) & 0x30303030;+ dst_t * y = yy + i*QK_K + 128*ip + il;- const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32+ const float d = x[i].d;- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product- }+ const uint8_t * ql = x[i].ql + 64*ip + il;+ const uint8_t qh = x[i].qh[32*ip + il];+ const int8_t * sc = x[i].scales + is;- return d*sumf;+ y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);+ y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);+ y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);+ y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}-// contiguous u/y values-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(- const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,- const float & d6, const float * __restrict__ d8) {+ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t ip = tid/16; // 0 or 1+ const int64_t il = tid - 16*ip; // 0...15-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- float sumf_d = 0.0f;+ dst_t * y = yy + i*QK_K + 16*ip + il;-#pragma unroll- for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {- int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale+ const float d = x[i].d;-#pragma unroll- for (int i = i0; i < i0 + 2; ++i) {- sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product- sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product+ const uint8_t ql = x[i].ql[16*ip + il];+ const uint8_t qh = x[i].qh[il] >> (2*ip);+ const int8_t * sc = x[i].scales;- sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product- sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product- }+ y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32);+ y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32);+#endif+}- sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);- }+template<typename dst_t>+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- return d6 * sumf_d;+ const int64_t i = blockIdx.x;+ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * q2 = x[i].qs + 4*ib;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);+ const uint32_t aux32 = q2[2] | (q2[3] << 16);+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);#elseNO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A-}+#endif-static __device__ __forceinline__ float vec_dot_q4_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+}- const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;+template<typename dst_t>+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- int v[VDR_Q4_0_Q8_1_MMVQ];- int u[2*VDR_Q4_0_Q8_1_MMVQ];+ const int64_t i = blockIdx.x;+ const block_iq2_xs * x = (const block_iq2_xs *) vx;-#pragma unroll- for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);- }+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * q2 = x[i].qs + 4*ib;+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);}+template<typename dst_t>+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {-static __device__ __forceinline__ float vec_dot_q4_1_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;-- int v[VDR_Q4_1_Q8_1_MMVQ];- int u[2*VDR_Q4_1_Q8_1_MMVQ];+ const int64_t i = blockIdx.x;+ const block_iq2_s * x = (const block_iq2_s *) vx;-#pragma unroll- for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);- }+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);}-static __device__ __forceinline__ float vec_dot_q5_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;+template<typename dst_t>+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- int vl[VDR_Q5_0_Q8_1_MMVQ];- int vh[VDR_Q5_0_Q8_1_MMVQ];- int u[2*VDR_Q5_0_Q8_1_MMVQ];+ const int64_t i = blockIdx.x;+ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;-#pragma unroll- for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);- vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * q3 = x[i].qs + 8*ib;+ const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);}+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);}-static __device__ __forceinline__ float vec_dot_q5_1_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;+template<typename dst_t>+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- int vl[VDR_Q5_1_Q8_1_MMVQ];- int vh[VDR_Q5_1_Q8_1_MMVQ];- int u[2*VDR_Q5_1_Q8_1_MMVQ];+ const int64_t i = blockIdx.x;+ const block_iq3_s * x = (const block_iq3_s *) vx;-#pragma unroll- for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);- vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));- u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint8_t * qs = x[i].qs + 8*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));+ const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));+ const uint8_t signs = x[i].signs[4*ib + il];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);+ y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);}+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);}-static __device__ __forceinline__ float vec_dot_q8_0_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;+template<typename dst_t>+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {- int v[VDR_Q8_0_Q8_1_MMVQ];- int u[VDR_Q8_0_Q8_1_MMVQ];+ const int64_t i = blockIdx.x;+ const block_iq1_s * x = (const block_iq1_s *) vx;-#pragma unroll- for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_int8(bq8_0->qs, iqs + i);- u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;+ const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;+ grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;+ grid32[0] &= 0x0f0f0f0f;+ for (int j = 0; j < 8; ++j) {+ y[j] = d * (q[j] + delta);}+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));}-static __device__ __forceinline__ float vec_dot_q2_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q2_K * bq2_K = (const block_q2_K *) vbq;-- const int bq8_offset = QR2_K * (iqs / QI8_1);- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);-- const uint8_t * scales = bq2_K->scales + scale_offset;+template<typename dst_t>+static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);- int u[QR2_K];- float d8[QR2_K];+ const int64_t i = blockIdx.x;+ const block_iq1_m * x = (const block_iq1_m *) vx;-#pragma unroll- for (int i = 0; i < QR2_K; ++ i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);+ const int64_t tid = threadIdx.x;+#if QK_K == 256+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * sc = (const uint16_t *)x[i].scales;+ iq1m_scale_t scale;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);+ const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);+ const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;+ grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;+ grid32[0] &= 0x0f0f0f0f;+ for (int j = 0; j < 8; ++j) {+ y[j] = d * (q[j] + delta);}+#else+ NO_DEVICE_CODE;+#endif- return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);}-static __device__ __forceinline__ float vec_dot_q3_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q3_K * bq3_K = (const block_q3_K *) vbq;-- const int bq8_offset = QR3_K * (iqs / (QI3_K/2));- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);-- const float d = bq3_K->d;-- const int vl = get_int_from_uint8(bq3_K->qs, iqs);- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted- const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;+template<typename dst_t>+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {- int u[QR3_K];- float d8[QR3_K];+ const int64_t i = blockIdx.x;+ const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);-#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;+ const uint8_t * q4 = x[ib].qs + 4*il;+ const float d = (float)x[ib].d;+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];}- return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);}-static __device__ __forceinline__ float vec_dot_q4_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {--#ifndef GGML_QKK_64- const block_q4_K * bq4_K = (const block_q4_K *) vbq;-- int v[2];- int u[2*QR4_K];- float d8[QR4_K];-- // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6- const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));-- // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12- // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44- // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76- // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108-- const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));- v[0] = q4[0];- v[1] = q4[4];+#if QK_K != 64+template<typename dst_t>+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;+ const block_iq4_xs * x = (const block_iq4_xs *)vx;- const uint16_t * scales = (const uint16_t *)bq4_K->scales;- uint16_t aux[2];- const int j = bq8_offset/2;- if (j < 2) {- aux[0] = scales[j+0] & 0x3f3f;- aux[1] = scales[j+2] & 0x3f3f;- } else {- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 4*il;+ const uint8_t * q4 = x[i].qs + 16*ib + 4*il;+ const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];+ y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];}- const uint8_t * sc = (const uint8_t *)aux;- const uint8_t * m = sc + 2;+}+#endif- for (int i = 0; i < QR4_K; ++i) {- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);+template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {+ const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);+}- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);- u[2*i+0] = q8[0];- u[2*i+1] = q8[4];+static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {+ const bool need_check = false;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ } else {+ const bool need_check = true;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);}+}- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);-+template<typename dst_t>+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);#else+ dequantize_block_q2_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const block_q4_K * bq4_K = (const block_q4_K *) vbq;-- float sumf_d = 0.0f;- float sumf_m = 0.0f;+template<typename dst_t>+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q3_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- uint16_t aux16[2];- const uint8_t * s = (const uint8_t *)aux16;+template<typename dst_t>+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);+}- const uint16_t * a = (const uint16_t *)bq4_K->scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;+template<typename dst_t>+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);+}- const float dall = bq4_K->dm[0];- const float dmin = bq4_K->dm[1];+template<typename dst_t>+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);+}- const float d8_1 = __low2float(bq8_1[0].ds);- const float d8_2 = __low2float(bq8_1[1].ds);+template<typename dst_t>+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q5_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));- const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);- const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));- const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);+template<typename dst_t>+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+#if QK_K == 256+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);+#else+ dequantize_block_q6_K<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- const int * q4 = (const int *)bq4_K->qs + (iqs/2);- const int v1 = q4[0];- const int v2 = q4[4];+template<typename dst_t>+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);+}- const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0));- const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0));- const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0));- const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0));+template<typename dst_t>+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);+}- sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]);- sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]);+template<typename dst_t>+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);+}- return dall * sumf_d - dmin * sumf_m;+template<typename dst_t>+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);+}-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+template<typename dst_t>+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);+}-#endif+template<typename dst_t>+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);}-static __device__ __forceinline__ float vec_dot_q5_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+template<typename dst_t>+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);+}-#ifndef GGML_QKK_64- const block_q5_K * bq5_K = (const block_q5_K *) vbq;+template<typename dst_t>+static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);+}- int vl[2];- int vh[2];- int u[2*QR5_K];- float d8[QR5_K];+template<typename dst_t>+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+#if QK_K == 64+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);+#else+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);+#endif+}- const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));- const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));- const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));+template <typename src_t, typename dst_t>+static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;- vl[0] = ql[0];- vl[1] = ql[4];+ if (i >= k) {+ return;+ }- vh[0] = qh[0] >> bq8_offset;- vh[1] = qh[4] >> bq8_offset;+ const src_t * x = (src_t *) vx;- const uint16_t * scales = (const uint16_t *)bq5_K->scales;- uint16_t aux[2];- const int j = bq8_offset/2;- if (j < 2) {- aux[0] = scales[j+0] & 0x3f3f;- aux[1] = scales[j+2] & 0x3f3f;- } else {- aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);- aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);- }- const uint8_t * sc = (const uint8_t *)aux;- const uint8_t * m = sc + 2;+ y[i] = x[i];+}-#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);+template <typename src_t, typename dst_t>+static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;+ convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);+}- const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);- u[2*i+0] = q8[0];- u[2*i+1] = q8[4];+to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;+ case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ if (ggml_cuda_info().devices[ggml_cuda_get_device()].cc >= CC_PASCAL) {+ return dequantize_block_q8_0_f16_cuda;+ }+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ1_M:+ return dequantize_row_iq1_m_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_F32:+ return convert_unary_cuda<float>;+ default:+ return nullptr;}+}- return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);+to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;+ case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ1_M:+ return dequantize_row_iq1_m_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_F16:+ return convert_unary_cuda<half>;+ default:+ return nullptr;+ }+}-#else+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP cpy.cu+//+////////////////////////////////////////////////////////////////////////////////-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const block_q5_K * bq5_K = (const block_q5_K *) vbq;- const int8_t * s = bq5_K->scales;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP cpy.cuh+//+////////////////////////////////////////////////////////////////////////////////- const float d = bq5_K->d;- const float d8_1 = __low2half(bq8_1[0].ds);- const float d8_2 = __low2half(bq8_1[1].ds);+#define CUDA_CPY_BLOCK_SIZE 32- const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));- const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);- const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));- const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);+void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);- const int * ql = (const int *)bq5_K->qs + (iqs/2);- const int vl1 = ql[0];- const int vl2 = ql[4];+void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- const int step = 4 * (iqs/2); // 0, 4, 8, 12- const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6- const int in = step%8; // 0, 4, 0, 4- const int vh = (*((const int *)(bq5_K->qh + in))) >> im;+void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1);- const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f);- const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f);- const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f);- const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f);+typedef void (*cpy_kernel_t)(const char * cx, char * cdst);- const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1])- + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]);+static __device__ void cpy_1_f32_f32(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ float * dsti = (float *) cdsti;- return d * sumf_d;+ *dsti = *xi;+}-#else- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ half * dsti = (half *) cdsti;-#endif+ *dsti = __float2half(*xi);}-static __device__ __forceinline__ float vec_dot_q6_K_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-- const block_q6_K * bq6_K = (const block_q6_K *) vbq;+static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ half * dsti = (half *) cdsti;- const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);- const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);- const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));+ *dsti = *xi;+}- const int vl = get_int_from_uint8(bq6_K->ql, iqs);- const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;+static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {+ const half * xi = (const half *) cxi;+ float * dsti = (float *) cdsti;- const int8_t * scales = bq6_K->scales + scale_offset;+ *dsti = *xi;+}- int u[QR6_K];- float d8[QR6_K];+template <cpy_kernel_t cpy_1>+static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {+ const int64_t i = blockDim.x*blockIdx.x + threadIdx.x;-#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);+ if (i >= ne) {+ return;}- return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);-}+ // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor+ // then combine those indices with the corresponding byte offsets to get the total offsets+ const int64_t i03 = i/(ne00 * ne01 * ne02);+ const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;-static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if QK_K == 256- const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;+ const int64_t i13 = i/(ne10 * ne11 * ne12);+ const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;-#if QR2_XXS == 8- const int ib32 = iqs;- const uint16_t * q2 = bq2->qs + 4*ib32;- const uint8_t * aux8 = (const uint8_t *)q2;- const int8_t * q8 = bq8_1[ib32].qs;- uint32_t aux32 = q2[2] | (q2[3] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);- const uint8_t signs = ksigns_iq2xs[aux32 & 127];- for (int j = 0; j < 8; ++j) {- sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- aux32 >>= 7;- }- const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * sumi;-#else- // iqs is 0...15- const int ib32 = iqs/2;- const int il = iqs%2;- const uint16_t * q2 = bq2->qs + 4*ib32;- const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]);- const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]);- const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * __low2float(bq8_1[ib32].ds) * 0.25f;- const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127];- const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127];- const int8_t * q8 = bq8_1[ib32].qs + 16*il;- int sumi1 = 0, sumi2 = 0;- for (int j = 0; j < 8; ++j) {- sumi1 += q8[j+0] * grid1[j] * (signs1 & kmask_iq2xs[j] ? -1 : 1);- sumi2 += q8[j+8] * grid2[j] * (signs2 & kmask_iq2xs[j] ? -1 : 1);- }- return d * (sumi1 + sumi2);-#endif-#else- NO_DEVICE_CODE;-#endif+ cpy_1(cx + x_offset, cdst + dst_offset);}-static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;+static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q8_0 * dsti = (block_q8_0 *) cdsti;- const int ib32 = iqs;- const uint16_t * q2 = bq2->qs + 4*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));- const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));- const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);- sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);- sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);- q8 += 8;+ float amax = 0.0f; // absolute max++ for (int j = 0; j < QK8_0; j++) {+ const float v = xi[j];+ amax = fmaxf(amax, fabsf(v));}- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));- const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));- const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);- sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);- sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);- q8 += 8;++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;++ dsti->d = d;++ for (int j = 0; j < QK8_0; ++j) {+ const float x0 = xi[j]*id;++ dsti->qs[j] = roundf(x0);}- const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-#else- GGML_UNUSED(ksigns64);- NO_DEVICE_CODE;-#endif-#else- GGML_UNUSED(ksigns64);- NO_DEVICE_CODE;-#endif}-// TODO-static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq2_s * bq2 = (const block_iq2_s *) vbq;+static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_0 * dsti = (block_q4_0 *) cdsti;- const int ib32 = iqs;- const int8_t * q8 = bq8_1[ib32].qs;- const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);- sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);- q8 += 8;+ float amax = 0.0f;+ float vmax = 0.0f;++ for (int j = 0; j < QK4_0; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }}- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);- sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);- q8 += 8;++ const float d = vmax / -8;+ const float id = d ? 1.0f/d : 0.0f;++ dsti->d = d;++ for (int j = 0; j < QK4_0/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK4_0/2 + j]*id;++ const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));++ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;}- const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-#else- GGML_UNUSED(ksigns64);- NO_DEVICE_CODE;-#endif-#else- GGML_UNUSED(ksigns64);- NO_DEVICE_CODE;-#endif}-static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;+static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q4_1 * dsti = (block_q4_1 *) cdsti;- const int ib32 = iqs;- const uint8_t * q3 = bq2->qs + 8*ib32;- const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- uint32_t aux32 = gas[0] | (gas[1] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];- const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];- const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));- const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);- q8 += 8;- aux32 >>= 7;+ float vmin = FLT_MAX;+ float vmax = -FLT_MAX;++ for (int j = 0; j < QK4_1; ++j) {+ const float v = xi[j];++ if (v < vmin) vmin = v;+ if (v > vmax) vmax = v;}- const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;- return d * sumi;-#else- NO_DEVICE_CODE;-#endif-#else- NO_DEVICE_CODE;-#endif-}-// TODO: don't use lookup table for signs-static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-#if QK_K == 256- const block_iq3_s * bq2 = (const block_iq3_s *) vbq;+ const float d = (vmax - vmin) / ((1 << 4) - 1);+ const float id = d ? 1.0f/d : 0.0f;- const int ib32 = iqs;- const uint8_t * qs = bq2->qs + 8*ib32;- const int8_t * q8 = bq8_1[ib32].qs;- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));- const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));- uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);- sumi = __dp4a(grid_l, *((int *)q8+0), sumi);- sumi = __dp4a(grid_h, *((int *)q8+1), sumi);- q8 += 8;+ dsti->dm.x = d;+ dsti->dm.y = vmin;++ for (int j = 0; j < QK4_1/2; ++j) {+ const float x0 = (xi[0 + j] - vmin)*id;+ const float x1 = (xi[QK4_1/2 + j] - vmin)*id;++ const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));+ const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));++ dsti->qs[j] = xi0;+ dsti->qs[j] |= xi1 << 4;}- const float d = (float)bq2->d * (1 + 2*((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds);- return d * sumi;-#else- NO_DEVICE_CODE;-#endif-#else- NO_DEVICE_CODE;-#endif}-static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if QK_K == 256- const block_iq1_s * bq1 = (const block_iq1_s *) vbq;+static __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q5_0 * dsti = (block_q5_0 *) cdsti;- const int ib32 = iqs;- int sumi = 0;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const int * q8 = (const int *)bq8_1[ib32].qs;- for (int l = 0; l < 4; ++l) {- const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));- int grid0 = grid[0] & 0x0f0f0f0f;- int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;- sumi = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi));- }-#else- const int8_t * q8 = bq8_1[ib32].qs;- for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));- for (int j = 0; j < 4; ++j) {- sumi += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4);+ float amax = 0.0f;+ float vmax = 0.0f;++ for (int j = 0; j < QK5_0; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;}- q8 += 8;}-#endif- const float delta = bq1->qh[ib32] & 0x8000 ? -1-IQ1S_DELTA : -1+IQ1S_DELTA;- const float d1q = (float)bq1->d * (2*((bq1->qh[ib32] >> 12) & 7) + 1);- const float d = d1q * __low2float (bq8_1[ib32].ds);- const float m = d1q * __high2float(bq8_1[ib32].ds);- return d * sumi + m * delta;-#else- NO_DEVICE_CODE;-#endif-}-static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#if QK_K == 256- const block_iq1_m * bq1 = (const block_iq1_m *) vbq;+ const float d = vmax / -16;+ const float id = d ? 1.0f/d : 0.0f;- const int ib32 = iqs;- int sumi[2] = {0, 0};- float sumf[2] = {0.f, 0.f};-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const int * q8 = (const int *)bq8_1[ib32].qs;- for (int l = 0; l < 4; ++l) {- const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 7) << 8)));- int grid0 = grid[0] & 0x0f0f0f0f;- int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;- sumi[l/2] = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi[l/2]));- const float delta = (bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 0x08 ? -1-IQ1M_DELTA : -1+IQ1M_DELTA;- const int sumy = __dp4a(q8[2*l+1], 0x01010101, __dp4a(q8[2*l+0], 0x01010101, 0));- sumf[l/2] += delta*sumy;- }-#else- const int8_t * q8 = bq8_1[ib32].qs;- for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));- int sumy = 0;- for (int j = 0; j < 4; ++j) {- sumi[l/2] += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4);- sumy += q8[j] + q8[j+4];- }- const float delta = (bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 0x08 ? -1-IQ1M_DELTA : -1+IQ1M_DELTA;- sumf[l/2] += delta*sumy;- q8 += 8;+ dsti->d = d;++ uint32_t qh = 0;+ for (int j = 0; j < QK5_0/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK5_0/2 + j]*id;++ const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));+ const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));++ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);}-#endif- iq1m_scale_t scale;- const uint16_t * sc = (const uint16_t *)bq1->scales;- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);- const float d = (float)scale.f16 * __low2float (bq8_1[ib32].ds);- return d * ((sumi[0] + sumf[0]) * (2*((sc[ib32/2] >> 6*(ib32%2)) & 0x7) + 1) + (sumi[1] + sumf[1]) * (2*((sc[ib32/2] >> (6*(ib32%2)+3)) & 0x7) + 1));-#else- NO_DEVICE_CODE;-#endif+ memcpy(dsti->qh, &qh, sizeof(qh));}-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics-static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,- int & val1, int & val2) {+static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_q5_1 * dsti = (block_q5_1 *) cdsti;- uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;- aux32 = q4 & 0x0f0f0f0f;- uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);- uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);- val1 = v1 | (v2 << 16);- aux32 = (q4 >> 4) & 0x0f0f0f0f;- v1 = values[q8[0]] | (values[q8[1]] << 8);- v2 = values[q8[2]] | (values[q8[3]] << 8);- val2 = v1 | (v2 << 16);-}-#endif+ float min = xi[0];+ float max = xi[0];-static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ for (int j = 1; j < QK5_1; ++j) {+ const float v = xi[j];+ min = v < min ? v : min;+ max = v > max ? v : max;+ }- const block_iq4_nl * bq = (const block_iq4_nl *) vbq;+ const float d = (max - min) / 31;+ const float id = d ? 1.0f/d : 0.0f;-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;- const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;+ dsti->dm.x = d;+ dsti->dm.y = min;- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;+ uint32_t qh = 0;+ for (int j = 0; j < QK5_1/2; ++j) {+ const float x0 = (xi[0 + j] - min)*id;+ const float x1 = (xi[QK5_1/2 + j] - min)*id;- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {- const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);- get_int_from_table_16(aux, values, v1, v2);- sumi1 = __dp4a(v1, q8[l+0], sumi1);- sumi2 = __dp4a(v2, q8[l+4], sumi2);+ const uint8_t xi0 = (uint8_t)(x0 + 0.5f);+ const uint8_t xi1 = (uint8_t)(x1 + 0.5f);++ dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);}+ memcpy(dsti->qh, &qh, sizeof(qh));+}-#else- const uint8_t * q4 = bq->qs + 4*iqs;- const int8_t * q8 = bq8_1->qs + 4*iqs;- int sumi1 = 0, sumi2 = 0;- for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) {- sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf];- sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4];+static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {+ if (x <= val[0]) return 0;+ if (x >= val[n-1]) return n-1;+ int ml = 0, mu = n-1;+ while (mu-ml > 1) {+ int mav = (ml+mu)/2;+ if (x < val[mav]) mu = mav; else ml = mav;}-#endif- const float d = (float)bq->d * __low2float(bq8_1->ds);- return d * (sumi1 + sumi2);+ return x - val[mu-1] < val[mu] - x ? mu-1 : mu;}-static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(- const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+static __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {+ const float * xi = (const float *) cxi;+ block_iq4_nl * dsti = (block_iq4_nl *) cdsti;-#if QK_K == 256-#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float amax = 0.0f;+ float vmax = 0.0f;- const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;- const uint8_t * values = (const uint8_t *)kvalues_iq4nl;+ for (int j = 0; j < QK4_NL; ++j) {+ const float v = xi[j];+ if (amax < fabsf(v)) {+ amax = fabsf(v);+ vmax = v;+ }+ }- // iqs is 0...7- const int ib32 = iqs;- const int32_t * q8 = (const int *)bq8_1[ib32].qs;- const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;- const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);- const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int j = 0; j < 4; ++j) {- get_int_from_table_16(q4[j], values, v1, v2);- sumi1 = __dp4a(v1, q8[j+0], sumi1);- sumi2 = __dp4a(v2, q8[j+4], sumi2);+ float d = vmax / kvalues_iq4nl[0];+ const float id = d ? 1.0f/d : 0.0f;++ float sumqx = 0, sumq2 = 0;+ for (int j = 0; j < QK4_NL/2; ++j) {+ const float x0 = xi[0 + j]*id;+ const float x1 = xi[QK4_NL/2 + j]*id;+ const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);+ const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);+ dsti->qs[j] = xi0 | (xi1 << 4);+ const float v0 = kvalues_iq4nl[xi0];+ const float v1 = kvalues_iq4nl[xi1];+ const float w0 = xi[0 + j]*xi[0 + j];+ const float w1 = xi[QK4_NL/2 + j]*xi[QK4_NL/2 + j];+ sumqx += w0*v0*xi[j] + w1*v1*xi[QK4_NL/2 + j];+ sumq2 += w0*v0*v0 + w1*v1*v1;}- return d * (sumi1 + sumi2);-#else- NO_DEVICE_CODE;-#endif-#else- return vec_dot_iq4_xs_q8_1(vbq, bq8_1, iqs);-#endif+ dsti->d = sumq2 > 0 ? sumqx/sumq2 : d;}-static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,- const int ne10, const int ne11, const int ne12,- const int nb1, const int nb2, int offset) {- const int i = blockDim.x * blockIdx.x + threadIdx.x;+template <cpy_kernel_t cpy_blck, int qk>+static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,+ const int nb12, const int nb13) {+ const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;+if (i >= ne) {return;}- int src1_idx = i - offset;- int oz = src1_idx / nb2;- int oy = (src1_idx - (oz * nb2)) / nb1;- int ox = src1_idx % nb1;- if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {- dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];- } else {- dst[i] = x[i];- }-}-static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,- const int ne10, const int ne11, const int ne12,- const int nb1, const int nb2, const int offset, cudaStream_t stream) {- int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;- acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);-}+ const int i03 = i/(ne00 * ne01 * ne02);+ const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);+ const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;+ const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;+ const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;-void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const ggml_tensor * src1 = dst->src[1];- const float * src0_d = (const float *)src0->data;- const float * src1_d = (const float *)src1->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+ const int i13 = i/(ne10 * ne11 * ne12);+ const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);+ const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;+ const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;+ const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);- GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported+ cpy_blck(cx + x_offset, cdst + dst_offset);+}- int nb1 = dst->op_params[0] / 4; // 4 bytes of float32- int nb2 = dst->op_params[1] / 4; // 4 bytes of float32- // int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused- int offset = dst->op_params[3] / 4; // offset in bytes+static void ggml_cpy_f16_f32_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, stream);+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-static __global__ void alibi_f32(const float * x, float * dst, const int ncols, const int k_rows,- const int n_heads_log2_floor, const float m0, const float m1) {- const int col = blockDim.x*blockIdx.x + threadIdx.x;+static void ggml_cpy_f32_f32_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- if (col >= ncols) {- return;- }+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- const int row = blockDim.y*blockIdx.y + threadIdx.y;- const int i = row*ncols + col;+static void ggml_cpy_f32_f16_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- const int k = row/k_rows;+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- float m_k;- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);- }+static void ggml_cpy_f32_q8_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- dst[i] = col * m_k + x[i];+ GGML_ASSERT(ne % QK8_0 == 0);+ const int num_blocks = ne / QK8_0;+ cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-static void alibi_f32_cuda(const float * x, float * dst, const int ncols, const int nrows,- const int k_rows, const int n_heads_log2_floor, const float m0,- const float m1, cudaStream_t stream) {- const dim3 block_dims(CUDA_ALIBI_BLOCK_SIZE, 1, 1);- const int num_blocks_x = (ncols + CUDA_ALIBI_BLOCK_SIZE - 1) / (CUDA_ALIBI_BLOCK_SIZE);- const dim3 block_nums(num_blocks_x, nrows, 1);- alibi_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, k_rows, n_heads_log2_floor, m0, m1);+static void ggml_cpy_f32_q4_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {++ GGML_ASSERT(ne % QK4_0 == 0);+ const int num_blocks = ne / QK4_0;+ cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-void ggml_cuda_op_alibi(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const float * src0_d = (const float *)src0->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+static void ggml_cpy_f32_q4_1_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_ASSERT(ne % QK4_1 == 0);+ const int num_blocks = ne / QK4_1;+ cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t nrows = ggml_nrows(src0);+static void ggml_cpy_f32_q5_0_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));+ GGML_ASSERT(ne % QK5_0 == 0);+ const int num_blocks = ne / QK5_0;+ cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- //GGML_ASSERT(ne01 + n_past == ne00);- GGML_ASSERT(n_head == ne02);+static void ggml_cpy_f32_q5_1_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));+ GGML_ASSERT(ne % QK5_1 == 0);+ const int num_blocks = ne / QK5_1;+ cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+}- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);+static void ggml_cpy_f32_iq4_nl_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {- alibi_f32_cuda(src0_d, dst_d, ne00, nrows, ne01, n_heads_log2_floor, m0, m1, stream);+ GGML_ASSERT(ne % QK4_NL == 0);+ const int num_blocks = ne / QK4_NL;+ cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {- // blockIDx.x: idx of ne0 / BLOCK_SIZE- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {- return;- }- dst[nidx] = start + step * nidx;-}+static void ggml_cpy_f16_f16_cuda(+ const char * cx, char * cdst, const int ne,+ const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,+ const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {-static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {- int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;- arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);+ const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;+ cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>+ (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);}-void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {+ const int64_t ne = ggml_nelements(src0);+ GGML_ASSERT(ne == ggml_nelements(src1));- GGML_ASSERT(dst->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);+ GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);- float start;- float stop;- float step;- memcpy(&start, (float *)dst->op_params + 0, sizeof(float));- memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));- memcpy(&step, (float *)dst->op_params + 2, sizeof(float));+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int64_t ne02 = src0->ne[2];- int64_t steps = (int64_t)ceil((stop - start) / step);- GGML_ASSERT(ggml_nelements(dst) == steps);+ //GGML_ASSERT(src0->ne[3] == 1);- arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);-}+ const int64_t nb00 = src0->nb[0];+ const int64_t nb01 = src0->nb[1];+ const int64_t nb02 = src0->nb[2];+ const int64_t nb03 = src0->nb[3];-template<typename T>-static inline __device__ void ggml_cuda_swap(T & a, T & b) {- T tmp = a;- a = b;- b = tmp;-}+ const int64_t ne10 = src1->ne[0];+ const int64_t ne11 = src1->ne[1];+ const int64_t ne12 = src1->ne[2];-template<ggml_sort_order order>-static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {- // bitonic sort- int col = threadIdx.x;- int row = blockIdx.y;+ //GGML_ASSERT(src1->ne[3] == 1);- if (col >= ncols_pad) {- return;- }+ const int64_t nb10 = src1->nb[0];+ const int64_t nb11 = src1->nb[1];+ const int64_t nb12 = src1->nb[2];+ const int64_t nb13 = src1->nb[3];- const float * x_row = x + row * ncols;- extern __shared__ int dst_row[];+ cudaStream_t main_stream = ctx.stream();- // initialize indices- dst_row[col] = col;+ char * src0_ddc = (char *) src0->data;+ char * src1_ddc = (char *) src1->data;- __syncthreads();-- for (int k = 2; k <= ncols_pad; k *= 2) {- for (int j = k / 2; j > 0; j /= 2) {- int ixj = col ^ j;- if (ixj > col) {- if ((col & k) == 0) {- if (dst_row[col] >= ncols ||- (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?- x_row[dst_row[col]] > x_row[dst_row[ixj]] :- x_row[dst_row[col]] < x_row[dst_row[ixj]]))- ) {- ggml_cuda_swap(dst_row[col], dst_row[ixj]);- }- } else {- if (dst_row[ixj] >= ncols ||- (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?- x_row[dst_row[col]] < x_row[dst_row[ixj]] :- x_row[dst_row[col]] > x_row[dst_row[ixj]]))- ) {- ggml_cuda_swap(dst_row[col], dst_row[ixj]);- }- }- }- __syncthreads();- }+ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {+ ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {+ ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {+ ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {+ ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {+ ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {+ ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {+ ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {+ ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {+ ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else {+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,+ ggml_type_name(src0->type), ggml_type_name(src1->type));+ GGML_ASSERT(false);}+}- // copy the result to dst without the padding- if (col < ncols) {- dst[row * ncols + col] = dst_row[col];- }+void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ ggml_cuda_cpy(ctx, src0, dst);}-static int next_power_of_2(int x) {- int n = 1;- while (n < x) {- n *= 2;+void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1) {+ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {+ return (void*) cpy_f32_f16<cpy_1_f32_f32>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {+ return (void*) cpy_f32_f16<cpy_1_f32_f16>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {+ return (void*) cpy_f32_q<cpy_blck_f32_q8_0, QK8_0>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {+ return (void*) cpy_f32_q<cpy_blck_f32_q4_0, QK4_0>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {+ return (void*) cpy_f32_q<cpy_blck_f32_q4_1, QK4_1>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {+ return (void*) cpy_f32_q<cpy_blck_f32_q5_0, QK5_0>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {+ return (void*) cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL>;+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {+ return (void*) cpy_f32_q<cpy_blck_f32_q5_1, QK5_1>;+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {+ return (void*) cpy_f32_f16<cpy_1_f32_f16>;+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {+ return (void*) cpy_f32_f16<cpy_1_f16_f32>;+ } else {+ fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,+ ggml_type_name(src0->type), ggml_type_name(src1->type));+ GGML_ASSERT(false);}- return n;}-static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {- // bitonic sort requires ncols to be power of 2- const int ncols_pad = next_power_of_2(ncols);- const dim3 block_dims(ncols_pad, 1, 1);- const dim3 block_nums(1, nrows, 1);- const size_t shared_mem = ncols_pad * sizeof(int);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP diagmask.cu+//+////////////////////////////////////////////////////////////////////////////////- GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);- if (order == GGML_SORT_ORDER_ASC) {- k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);- } else if (order == GGML_SORT_ORDER_DESC) {- k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);- } else {- GGML_ASSERT(false);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP diagmask.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32++void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {+ const int col = blockDim.y*blockIdx.y + threadIdx.y;+ const int row = blockDim.x*blockIdx.x + threadIdx.x;++ if (col >= ncols) {+ return;}++ const int i = row*ncols + col;+ //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];+ //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU+ dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;}-void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {+ const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);+ const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;+ const dim3 block_nums(nrows_x, block_num_x, 1);+ diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);+}++void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const float * src0_d = (const float *)src0->data;float * dst_d = (float *)dst->data;cudaStream_t stream = ctx.stream();GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_I32);- GGML_ASSERT(ggml_is_contiguous(src0));-- const int64_t ncols = src0->ne[0];- const int64_t nrows = ggml_nrows(src0);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];+ const int64_t ne00 = src0->ne[0];+ const int64_t ne01 = src0->ne[1];+ const int nrows0 = ggml_nrows(src0);- argsort_f32_i32_cuda(src0_d, (int *)dst_d, ncols, nrows, order, stream);-}+ const int n_past = ((int32_t *) dst->op_params)[0];-static __device__ __forceinline__ float op_repeat(const float a, const float b) {- return b;- GGML_UNUSED(a);+ diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);}-static __device__ __forceinline__ float op_add(const float a, const float b) {- return a + b;-}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP dmmv.cu+//+////////////////////////////////////////////////////////////////////////////////-static __device__ __forceinline__ float op_mul(const float a, const float b) {- return a * b;-}-static __device__ __forceinline__ float op_div(const float a, const float b) {- return a / b;-}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP dmmv.cuh+//+////////////////////////////////////////////////////////////////////////////////-template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>-static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,- int ne0, int ne1, int ne2, int ne3,- int ne10, int ne11, int ne12, int ne13,- /*int s0, */ int s1, int s2, int s3,- /*int s00,*/ int s01, int s02, int s03,- /*int s10,*/ int s11, int s12, int s13) {- const int i0s = blockDim.x*blockIdx.x + threadIdx.x;- const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);- const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;- const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;- if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {- return;- }+// dmmv = dequantize_mul_mat_vec- const int i11 = i1 % ne11;- const int i12 = i2 % ne12;- const int i13 = i3 % ne13;+// TODO: remove this?+#ifndef GGML_CUDA_DMMV_X+#define GGML_CUDA_DMMV_X 32+#endif- const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;- const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;- const size_t i_dst = i3*s3 + i2*s2 + i1*s1;+#ifndef GGML_CUDA_MMV_Y+#define GGML_CUDA_MMV_Y 1+#endif- const src0_t * src0_row = src0 + i_src0;- const src1_t * src1_row = src1 + i_src1;- dst_t * dst_row = dst + i_dst;+void ggml_cuda_op_dequantize_mul_mat_vec(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);- for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {- const int i10 = i0 % ne10;- dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);- }-}+#ifndef K_QUANTS_PER_ITERATION+#define K_QUANTS_PER_ITERATION 2+#else+static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");+#endif-template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>-static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,- int ne0, int ne1, int ne2, int ne3,- int ne10, int ne11, int ne12, int ne13,- /*int s0, */ int s1, int s2, int s3,- /*int s00,*/ int s01, int s02, int s03,- /*int s10,*/ int s11, int s12, int s13) {+static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");- const int i3 = i/(ne2*ne1*ne0);- const int i2 = (i/(ne1*ne0)) % ne2;- const int i1 = (i/ne0) % ne1;- const int i0 = i % ne0;+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;- if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {- return;- }+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- const int i11 = i1 % ne11;- const int i12 = i2 % ne12;- const int i13 = i3 % ne13;+ const block_q2_K * x = (const block_q2_K *)vx + ib0;- const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;- const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;- const size_t i_dst = i3*s3 + i2*s2 + i1*s1;+ float tmp = 0; // partial sum for thread in warp- const src0_t * src0_row = src0 + i_src0;- const src1_t * src1_row = src1 + i_src1;- dst_t * dst_row = dst + i_dst;+#if QK_K == 256+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- const int i10 = i0 % ne10;- dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);-}+ const int step = 16/K_QUANTS_PER_ITERATION;-template<float (*bin_op)(const float, const float)>-struct bin_bcast_cuda {- template<typename src0_t, typename src1_t, typename dst_t>- void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,- const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,- cudaStream_t stream) {+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0...15 or 0...7- GGML_TENSOR_BINARY_OP_LOCALS+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2+ const int q_offset = 32*im + l0;+ const int s_offset = 8*im;+ const int y_offset = 128*im + l0;- int nr0 = ne10/ne0;- int nr1 = ne11/ne1;- int nr2 = ne12/ne2;- int nr3 = ne13/ne3;+ uint32_t aux[4];+ const uint8_t * d = (const uint8_t *)aux;+ const uint8_t * m = (const uint8_t *)(aux + 2);- int nr[4] = { nr0, nr1, nr2, nr3 };+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {- // collapse dimensions until first broadcast dimension- int64_t cne[] = {ne0, ne1, ne2, ne3};- int64_t cne0[] = {ne00, ne01, ne02, ne03};- int64_t cne1[] = {ne10, ne11, ne12, ne13};+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * q = x[i].qs + q_offset;- size_t cnb[] = {nb0, nb1, nb2, nb3};- size_t cnb0[] = {nb00, nb01, nb02, nb03};- size_t cnb1[] = {nb10, nb11, nb12, nb13};+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- auto collapse = [](int64_t cne[]) {- cne[0] *= cne[1];- cne[1] = cne[2];- cne[2] = cne[3];- cne[3] = 1;- };+ const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);+ aux[0] = a[0] & 0x0f0f0f0f;+ aux[1] = a[1] & 0x0f0f0f0f;+ aux[2] = (a[0] >> 4) & 0x0f0f0f0f;+ aux[3] = (a[1] >> 4) & 0x0f0f0f0f;- auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {- cnb[1] *= cne[1];- cnb[2] *= cne[2];- cnb[3] *= cne[3];- };+ float sum1 = 0, sum2 = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)+ + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)+ + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)+ + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)+ + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)+ + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)+ + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)+ +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);+ sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]+ + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];- if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {- for (int i = 0; i < 4; i++) {- if (nr[i] != 1) {- break;- }- if (i > 0) {- collapse_nb(cnb, cne);- collapse_nb(cnb0, cne0);- collapse_nb(cnb1, cne1);- collapse(cne);- collapse(cne0);- collapse(cne1);- }- }}+ tmp += dall * sum1 - dmin * sum2;- {- int64_t ne0 = cne[0];- int64_t ne1 = cne[1];- int64_t ne2 = cne[2];- int64_t ne3 = cne[3];-- //int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);- //int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);- //int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);- //int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);-- int64_t ne10 = cne1[0];- int64_t ne11 = cne1[1];- int64_t ne12 = cne1[2];- int64_t ne13 = cne1[3];-- size_t nb0 = cnb[0];- size_t nb1 = cnb[1];- size_t nb2 = cnb[2];- size_t nb3 = cnb[3];+ }+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3+ const int offset = tid * K_QUANTS_PER_ITERATION;- size_t nb00 = cnb0[0];- size_t nb01 = cnb0[1];- size_t nb02 = cnb0[2];- size_t nb03 = cnb0[3];+ uint32_t uaux[2];+ const uint8_t * d = (const uint8_t *)uaux;- size_t nb10 = cnb1[0];- size_t nb11 = cnb1[1];- size_t nb12 = cnb1[2];- size_t nb13 = cnb1[3];+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- size_t s0 = nb0 / sizeof(dst_t);- size_t s1 = nb1 / sizeof(dst_t);- size_t s2 = nb2 / sizeof(dst_t);- size_t s3 = nb3 / sizeof(dst_t);+ const float * y = yy + i * QK_K + offset;+ const uint8_t * q = x[i].qs + offset;+ const uint32_t * s = (const uint32_t *)x[i].scales;- size_t s10 = nb10 / sizeof(src1_t);- size_t s11 = nb11 / sizeof(src1_t);- size_t s12 = nb12 / sizeof(src1_t);- size_t s13 = nb13 / sizeof(src1_t);+ uaux[0] = s[0] & 0x0f0f0f0f;+ uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;- size_t s00 = nb00 / sizeof(src0_t);- size_t s01 = nb01 / sizeof(src0_t);- size_t s02 = nb02 / sizeof(src0_t);- size_t s03 = nb03 / sizeof(src0_t);+ const float2 dall = __half22float2(x[i].dm);- GGML_ASSERT(nb0 % sizeof(dst_t) == 0);- GGML_ASSERT(nb1 % sizeof(dst_t) == 0);- GGML_ASSERT(nb2 % sizeof(dst_t) == 0);- GGML_ASSERT(nb3 % sizeof(dst_t) == 0);+ float sum1 = 0, sum2 = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ const uint8_t ql = q[l];+ sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)+ + y[l+16] * d[1] * ((ql >> 2) & 3)+ + y[l+32] * d[2] * ((ql >> 4) & 3)+ + y[l+48] * d[3] * ((ql >> 6) & 3);+ sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];+ }+ tmp += dall.x * sum1 - dall.y * sum2;+ }+#endif- GGML_ASSERT(nb00 % sizeof(src0_t) == 0);- GGML_ASSERT(nb01 % sizeof(src0_t) == 0);- GGML_ASSERT(nb02 % sizeof(src0_t) == 0);- GGML_ASSERT(nb03 % sizeof(src0_t) == 0);+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);- GGML_ASSERT(nb10 % sizeof(src1_t) == 0);- GGML_ASSERT(nb11 % sizeof(src1_t) == 0);- GGML_ASSERT(nb12 % sizeof(src1_t) == 0);- GGML_ASSERT(nb13 % sizeof(src1_t) == 0);+ if (threadIdx.x == 0) {+ dst[row] = tmp;+ }+}- GGML_ASSERT(s0 == 1);- GGML_ASSERT(s00 == 1);- GGML_ASSERT(s10 == 1);+static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- const int block_size = 128;+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;- int64_t hne0 = std::max(ne0/2LL, 1LL);+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- dim3 block_dims;- block_dims.x = std::min<unsigned int>(hne0, block_size);- block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);- block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);+ const block_q3_K * x = (const block_q3_K *)vx + ib0;- dim3 block_nums(- (hne0 + block_dims.x - 1) / block_dims.x,- (ne1 + block_dims.y - 1) / block_dims.y,- (ne2*ne3 + block_dims.z - 1) / block_dims.z- );+ float tmp = 0; // partial sum for thread in warp- if (block_nums.z > 65535) {- // this is the maximum number of blocks in z dimension, fallback to 1D grid kernel- int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;- k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne0, ne1, ne2, ne3,- ne10, ne11, ne12, ne13,- /* s0, */ s1, s2, s3,- /* s00, */ s01, s02, s03,- /* s10, */ s11, s12, s13);- } else {- k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne0, ne1, ne2, ne3,- ne10, ne11, ne12, ne13,- /* s0, */ s1, s2, s3,- /* s00, */ s01, s02, s03,- /* s10, */ s11, s12, s13);- }- }- }-};+#if QK_K == 256-template<class op>-static void ggml_cuda_op_bin_bcast(- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {+ const uint16_t kmask1 = 0x0303;+ const uint16_t kmask2 = 0x0f0f;- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {- op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);- } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {- op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);- } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {- op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);- } else {- fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,- ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));- GGML_ASSERT(false);- }-}+ const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop+ const int step = 16/K_QUANTS_PER_ITERATION;+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0....15 or 0...7-void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());-}+ const uint8_t m = 1 << (4*im);-void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());-}+ const int l0 = n*in; // 0...15 or 0...14 in steps of 2+ const int q_offset = 32*im + l0;+ const int y_offset = 128*im + l0;-void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());-}+ uint16_t utmp[4];+ const int8_t * s = (const int8_t *)utmp;-void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());-}+ const uint16_t s_shift = 4*im;-static __global__ void clamp_f32(const float * x, float * dst, const float min, const float max, const int k) {- const int i = blockDim.x*blockIdx.x + threadIdx.x;+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {- if (i >= k) {- return;- }+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * q = x[i].qs + q_offset;+ const uint8_t * h = x[i].hmask + l0;- dst[i] = x[i] < min ? min : (x[i] > max ? max : x[i]);-}+ const uint16_t * a = (const uint16_t *)x[i].scales;+ utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);+ utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);+ utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);+ utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);-static void clamp_f32_cuda(const float * x, float * dst, const float min, const float max, const int k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;- clamp_f32<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);-}+ const float d = x[i].d;+ float sum = 0;+ for (int l = 0; l < n; ++l) {+ sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))+ + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))+ + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))+ + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));+ sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))+ + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))+ + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))+ + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));+ }+ tmp += d * sum;-void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const float * src0_d = (const float *)src0->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();-- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ }+#else- float min;- float max;- memcpy(&min, dst->op_params, sizeof(float));- memcpy(&max, (float *) dst->op_params + 1, sizeof(float));+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3+ const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14+ const int in = offset/8; // 0 or 1+ const int im = offset%8; // 0...7- clamp_f32_cuda(src0_d, dst_d, min, max, ggml_nelements(src0), stream);- CUDA_CHECK(cudaGetLastError());-}+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {-static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {- return;- }- // operation- int offset_dst =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- if (blockIdx.z < ne02) { // src0- int offset_src =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- dst[offset_dst] = x[offset_src];- } else {- int offset_src =- nidx +- blockIdx.y * ne0 +- (blockIdx.z - ne02) * ne0 * gridDim.y;- dst[offset_dst] = y[offset_src];- }-}+ const float * y = yy + i * QK_K + offset;+ const uint8_t * q = x[i].qs + offset;+ const uint8_t * s = x[i].scales;-static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {- int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;- dim3 gridDim(num_blocks, ne1, ne2);- concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);-}+ const float dall = (float)x[i].d;-void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const ggml_tensor * src1 = dst->src[1];- const float * src0_d = (const float *)src0->data;- const float * src1_d = (const float *)src1->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+ float sum = 0;+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {+ const uint8_t hl = x[i].hmask[im+l] >> in;+ const uint8_t ql = q[l];+ sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))+ + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))+ + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))+ + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));+ }+ tmp += sum;+ }+#endif- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);- for (int i3 = 0; i3 < dst->ne[3]; i3++) {- concat_f32_cuda(src0_d + i3 * (src0->nb[3] / 4), src1_d + i3 * (src1->nb[3] / 4), dst_d + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], stream);+ if (threadIdx.x == 0) {+ dst[row] = tmp;}}-#define CUDA_Q8_0_NE_ALIGN 2048+static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {- const int64_t i = (int64_t)2*(blockDim.x*blockIdx.x + threadIdx.x);+ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- if (i >= k) {- return;- }+ const block_q4_K * x = (const block_q4_K *)vx + ib0;- const int64_t ib = i/qk; // block index- const int64_t iqs = (i%qk)/qr; // quant index- const int64_t iybs = i - i%qk; // y block start index- const int64_t y_offset = qr == 1 ? 1 : qk/2;+#if QK_K == 256+ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;- // dequantize- dfloat2 v;- dequantize_kernel(vx, ib, iqs, v);+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1- y[iybs + iqs + 0] = v.x;- y[iybs + iqs + y_offset] = v.y;-}+ const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4-template <bool need_check>-static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {-#if __CUDA_ARCH__ >= CC_PASCAL- constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;+ const int il = tid/step; // 0...3+ const int ir = tid - step*il; // 0...7 or 0...3+ const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4- const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;- const int * x0 = ((int *) vx) + blockIdx.x * nint;- half2 * y2 = (half2 *) (y + i0);+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224+ const int in = il%2;- __shared__ int vals[nint];+ const int l0 = n*(2*ir + in);+ const int q_offset = 32*im + l0;+ const int y_offset = 64*im + l0;-#pragma unroll- for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {- if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {- break;- }+ uint16_t aux[4];+ const uint8_t * sc = (const uint8_t *)aux;- const int ix = ix0 + threadIdx.x;- vals[ix] = x0[ix];- }+#if K_QUANTS_PER_ITERATION == 2+ uint32_t q32[4];+ const uint8_t * q4 = (const uint8_t *)q32;+#else+ uint16_t q16[4];+ const uint8_t * q4 = (const uint8_t *)q16;+#endif- __syncthreads();+ float tmp = 0; // partial sum for thread in warp-#pragma unroll- for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {- if (need_check && i0 + iy + 2*threadIdx.x >= k) {- return;- }+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {- const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);- const half d = *b0;- const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];+ const float * y1 = yy + i*QK_K + y_offset;+ const float * y2 = y1 + 128;- y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));- }-#else- GGML_UNUSED(vx);- GGML_UNUSED(y);- GGML_UNUSED(k);- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_PASCAL-}+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);-template<typename dst_t>-static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux[0] = a[im+0] & kmask1;+ aux[1] = a[im+2] & kmask1;+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);- const int64_t i = blockIdx.x;+#if K_QUANTS_PER_ITERATION == 2+ const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);+ const uint32_t * q2 = q1 + 16;- // assume 32 threads- const int64_t tid = threadIdx.x;- const int64_t il = tid/8;- const int64_t ir = tid%8;- const int64_t ib = 8*i + ir;- if (ib >= nb32) {- return;- }+ q32[0] = q1[0] & 0x0f0f0f0f;+ q32[1] = q1[0] & 0xf0f0f0f0;+ q32[2] = q2[0] & 0x0f0f0f0f;+ q32[3] = q2[0] & 0xf0f0f0f0;- dst_t * y = yy + 256*i + 32*ir + 4*il;+ float4 s = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ for (int l = 0; l < 4; ++l) {+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+ 4];+ s.z += y2[l] * q4[l+8]; s.w += y2[l+32] * q4[l+12];+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];+ }+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;+#else+ const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);+ const uint16_t * q2 = q1 + 32;- const block_q4_0 * x = (const block_q4_0 *)vx + ib;- const float d = __half2float(x->d);- const float dm = -8*d;+ q16[0] = q1[0] & 0x0f0f;+ q16[1] = q1[0] & 0xf0f0;+ q16[2] = q2[0] & 0x0f0f;+ q16[3] = q2[0] & 0xf0f0;- const uint8_t * q = x->qs + 4*il;+ float4 s = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ for (int l = 0; l < 2; ++l) {+ s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];+ s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];+ }+ tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;+#endif- for (int l = 0; l < 4; ++l) {- y[l+ 0] = d * (q[l] & 0xF) + dm;- y[l+16] = d * (q[l] >> 4) + dm;}-}+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);-template<typename dst_t>-static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ const int step = tid * K_QUANTS_PER_ITERATION;- const int64_t i = blockIdx.x;+ uint16_t aux16[2];+ const uint8_t * s = (const uint8_t *)aux16;- // assume 32 threads- const int64_t tid = threadIdx.x;- const int64_t il = tid/8;- const int64_t ir = tid%8;- const int64_t ib = 8*i + ir;- if (ib >= nb32) {- return;- }+ float tmp = 0;- dst_t * y = yy + 256*i + 32*ir + 4*il;+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ const uint8_t * q = x[i].qs + step;+ const float * y = yy + i*QK_K + step;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;+ const float d = (float)x[i].dm[0];+ const float m = (float)x[i].dm[1];+ float sum = 0.f;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])+ + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])+ + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])+ + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);+ }+ tmp += sum;+ }- const block_q4_1 * x = (const block_q4_1 *)vx + ib;- const float2 d = __half22float2(x->dm);+#endif- const uint8_t * q = x->qs + 4*il;+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);- for (int l = 0; l < 4; ++l) {- y[l+ 0] = d.x * (q[l] & 0xF) + d.y;- y[l+16] = d.x * (q[l] >> 4) + d.y;+ if (tid == 0) {+ dst[row] = tmp;}}-//================================== k-quants+static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols) {-template<typename dst_t>-static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int row = blockIdx.x;+ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;- const int64_t i = blockIdx.x;- const block_q2_K * x = (const block_q2_K *) vx;+ const block_q5_K * x = (const block_q5_K *)vx + ib0;++ float tmp = 0; // partial sum for thread in warp- const int64_t tid = threadIdx.x;#if QK_K == 256- const int64_t n = tid/32;- const int64_t l = tid - 32*n;- const int64_t is = 8*n + l/16;+ const uint16_t kmask1 = 0x3f3f;+ const uint16_t kmask2 = 0x0f0f;+ const uint16_t kmask3 = 0xc0c0;- const uint8_t q = x[i].qs[32*n + l];- dst_t * y = yy + i*QK_K + 128*n;+ const int tid = threadIdx.x/2; // 0...15+ const int ix = threadIdx.x%2;- float dall = __low2half(x[i].dm);- float dmin = __high2half(x[i].dm);- y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);- y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);- y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);- y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);-#else- const int64_t is = tid/16; // 0 or 1- const int64_t il = tid%16; // 0...15- const uint8_t q = x[i].qs[il] >> (2*is);- dst_t * y = yy + i*QK_K + 16*is + il;- float dall = __low2half(x[i].dm);- float dmin = __high2half(x[i].dm);- y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);- y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4);-#endif+ const int il = tid/4; // 0...3+ const int ir = tid - 4*il;// 0...3+ const int n = 2;-}+ const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224+ const int in = il%2;-template<typename dst_t>-static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int l0 = n*(2*ir + in);+ const int q_offset = 32*im + l0;+ const int y_offset = 64*im + l0;- const int64_t i = blockIdx.x;- const block_q3_K * x = (const block_q3_K *) vx;+ const uint8_t hm1 = 1 << (2*im);+ const uint8_t hm2 = hm1 << 4;-#if QK_K == 256- const int64_t r = threadIdx.x/4;- const int64_t tid = r/2;- const int64_t is0 = r%2;- const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);- const int64_t n = tid / 4;- const int64_t j = tid - 4*n;+ uint16_t aux[4];+ const uint8_t * sc = (const uint8_t *)aux;- uint8_t m = 1 << (4*n + j);- int64_t is = 8*n + 2*j + is0;- int shift = 2*j;+ uint16_t q16[8];+ const uint8_t * q4 = (const uint8_t *)q16;- int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :- is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :- is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :- (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);- float d_all = x[i].d;- float dl = d_all * (us - 32);+ for (int i = ix; i < num_blocks_per_row; i += 2) {- dst_t * y = yy + i*QK_K + 128*n + 32*j;- const uint8_t * q = x[i].qs + 32*n;- const uint8_t * hm = x[i].hmask;+ const uint8_t * ql1 = x[i].qs + q_offset;+ const uint8_t * qh = x[i].qh + l0;+ const float * y1 = yy + i*QK_K + y_offset;+ const float * y2 = y1 + 128;- for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));-#else- const int64_t tid = threadIdx.x;- const int64_t is = tid/16; // 0 or 1- const int64_t il = tid%16; // 0...15- const int64_t im = il/8; // 0...1- const int64_t in = il%8; // 0...7+ const float dall = __low2half(x[i].dm);+ const float dmin = __high2half(x[i].dm);- dst_t * y = yy + i*QK_K + 16*is + il;+ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux[0] = a[im+0] & kmask1;+ aux[1] = a[im+2] & kmask1;+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);- const uint8_t q = x[i].qs[il] >> (2*is);- const uint8_t h = x[i].hmask[in] >> (2*is + im);- const float d = (float)x[i].d;+ float4 sum = {0.f, 0.f, 0.f, 0.f};+ float smin = 0;+ const uint16_t * q1 = (const uint16_t *)ql1;+ const uint16_t * q2 = q1 + 32;+ q16[0] = q1[0] & 0x0f0f;+ q16[1] = q1[8] & 0x0f0f;+ q16[2] = (q1[0] >> 4) & 0x0f0f;+ q16[3] = (q1[8] >> 4) & 0x0f0f;+ q16[4] = q2[0] & 0x0f0f;+ q16[5] = q2[8] & 0x0f0f;+ q16[6] = (q2[0] >> 4) & 0x0f0f;+ q16[7] = (q2[8] >> 4) & 0x0f0f;+ for (int l = 0; l < n; ++l) {+ sum.x += y1[l+ 0] * (q4[l +0] + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))+ + y1[l+16] * (q4[l +2] + (qh[l+16] & (hm1 << 0) ? 16 : 0));+ sum.y += y1[l+32] * (q4[l +4] + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))+ + y1[l+48] * (q4[l +6] + (qh[l+16] & (hm1 << 1) ? 16 : 0));+ sum.z += y2[l+ 0] * (q4[l +8] + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))+ + y2[l+16] * (q4[l+10] + (qh[l+16] & (hm2 << 0) ? 16 : 0));+ sum.w += y2[l+32] * (q4[l+12] + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))+ + y2[l+48] * (q4[l+14] + (qh[l+16] & (hm2 << 1) ? 16 : 0));+ smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]+ + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];+ }+ tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;+ }- if (is == 0) {- y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));- y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));- } else {- y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4));- y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4));+#else+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);+ const int step = tid * K_QUANTS_PER_ITERATION;+ const int im = step/8;+ const int in = step%8;++ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ const uint8_t * q = x[i].qs + step;+ const int8_t * s = x[i].scales;+ const float * y = yy + i*QK_K + step;+ const float d = x[i].d;+ float sum = 0.f;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ const uint8_t h = x[i].qh[in+j] >> im;+ sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))+ + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))+ + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))+ + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));+ }+ tmp += sum;}#endif-}+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);-#if QK_K == 256-static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {- if (j < 4) {- d = q[j] & 63; m = q[j + 4] & 63;- } else {- d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);- m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);+ if (threadIdx.x == 0) {+ dst[row] = tmp;}}-#endif-template<typename dst_t>-static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q4_K * x = (const block_q4_K *) vx;+static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {- const int64_t i = blockIdx.x;+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");++ const int row = blockIdx.x*blockDim.y + threadIdx.y;+ if (row > nrows) return;++ const int num_blocks_per_row = ncols / QK_K;+ const int ib0 = row*num_blocks_per_row;++ const block_q6_K * x = (const block_q6_K *)vx + ib0;#if QK_K == 256- // assume 32 threads- const int64_t tid = threadIdx.x;- const int64_t il = tid/8;- const int64_t ir = tid%8;- const int64_t is = 2*il;- const int64_t n = 4;- dst_t * y = yy + i*QK_K + 64*il + n*ir;+ const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16+ const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8- const uint8_t * q = x[i].qs + 32*il + n*ir;+ const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...+ const int in = tid - step*im; // 0...15 or 0...7- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const float d1 = dall * sc; const float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const float d2 = dall * sc; const float m2 = dmin * m;- for (int l = 0; l < n; ++l) {- y[l + 0] = d1 * (q[l] & 0xF) - m1;- y[l +32] = d2 * (q[l] >> 4) - m2;- }+#if K_QUANTS_PER_ITERATION == 1+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15+ const int is = 0;#else- const int64_t tid = threadIdx.x;- const uint8_t * q = x[i].qs;- dst_t * y = yy + i*QK_K;- const float d = (float)x[i].dm[0];- const float m = (float)x[i].dm[1];- y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);- y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4);+ const int l0 = 4 * in; // 0, 4, 8, ..., 28+ const int is = in / 4;#endif-}+ const int ql_offset = 64*im + l0;+ const int qh_offset = 32*im + l0;+ const int s_offset = 8*im + is;+ const int y_offset = 128*im + l0;-template<typename dst_t>-static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q5_K * x = (const block_q5_K *) vx;+ float tmp = 0; // partial sum for thread in warp- const int64_t i = blockIdx.x;--#if QK_K == 256- // assume 64 threads - this is very slightly better than the one below- const int64_t tid = threadIdx.x;- const int64_t il = tid/16; // il is in 0...3- const int64_t ir = tid%16; // ir is in 0...15- const int64_t is = 2*il; // is is in 0...6-- dst_t * y = yy + i*QK_K + 64*il + 2*ir;-- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {- const uint8_t * ql = x[i].qs + 32*il + 2*ir;- const uint8_t * qh = x[i].qh + 2*ir;+ const float * y = yy + i * QK_K + y_offset;+ const uint8_t * ql = x[i].ql + ql_offset;+ const uint8_t * qh = x[i].qh + qh_offset;+ const int8_t * s = x[i].scales + s_offset;- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const float d1 = dall * sc; const float m1 = dmin * m;- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const float d2 = dall * sc; const float m2 = dmin * m;+ const float d = x[i].d;- uint8_t hm = 1 << (2*il);- y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;- y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;- hm <<= 1;- y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;- y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;+#if K_QUANTS_PER_ITERATION == 1+ float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)+ + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)+ + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)+ + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)+ + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)+ + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)+ + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)+ +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);+ tmp += sum;#else- const int64_t tid = threadIdx.x;- const uint8_t q = x[i].qs[tid];- const int64_t im = tid/8; // 0...3- const int64_t in = tid%8; // 0...7- const int64_t is = tid/16; // 0 or 1- const uint8_t h = x[i].qh[in] >> im;- const float d = x[i].d;- dst_t * y = yy + i*QK_K + tid;- y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));- y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16));+ float sum = 0;+ for (int l = 0; l < 4; ++l) {+ sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)+ + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)+ + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)+ + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);+ }+ tmp += sum;#endif-}--template<typename dst_t>-static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const block_q6_K * x = (const block_q6_K *) vx;- const int64_t i = blockIdx.x;-#if QK_K == 256+ }- // assume 64 threads - this is very slightly better than the one below- const int64_t tid = threadIdx.x;- const int64_t ip = tid/32; // ip is 0 or 1- const int64_t il = tid - 32*ip; // 0...32- const int64_t is = 8*ip + il/16;+#else- dst_t * y = yy + i*QK_K + 128*ip + il;+ const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7+ const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3- const float d = x[i].d;+ const int step = tid * K_QUANTS_PER_ITERATION;- const uint8_t * ql = x[i].ql + 64*ip + il;- const uint8_t qh = x[i].qh[32*ip + il];- const int8_t * sc = x[i].scales + is;+ float tmp = 0; // partial sum for thread in warp- y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);- y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);- y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);- y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);-#else+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- // assume 32 threads- const int64_t tid = threadIdx.x;- const int64_t ip = tid/16; // 0 or 1- const int64_t il = tid - 16*ip; // 0...15+ const float * y = yy + i * QK_K + step;+ const uint8_t * ql = x[i].ql + step;+ const uint8_t * qh = x[i].qh + step;+ const int8_t * s = x[i].scales;- dst_t * y = yy + i*QK_K + 16*ip + il;+ const float d = x[i+0].d;- const float d = x[i].d;+ float sum = 0;+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {+ sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)+ + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)+ + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)+ + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);+ }+ tmp += sum;- const uint8_t ql = x[i].ql[16*ip + il];- const uint8_t qh = x[i].qh[il] >> (2*ip);- const int8_t * sc = x[i].scales;+ }- y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32);- y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32);#endif-}--template<typename dst_t>-static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int64_t i = blockIdx.x;- const block_iq2_xxs * x = (const block_iq2_xxs *) vx;- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint16_t * q2 = x[i].qs + 4*ib;- const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);- const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-#else- NO_DEVICE_CODE;-#endif+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);+ if (tid == 0) {+ dst[row] = tmp;+ }}-template<typename dst_t>-static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+static __device__ void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){+ const half * x = (const half *) vx;- const int64_t i = blockIdx.x;- const block_iq2_xs * x = (const block_iq2_xs *) vx;+ // automatic half -> float type cast if dfloat == float+ v.x = x[ib + iqs + 0];+ v.y = x[ib + iqs + 1];+}- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint16_t * q2 = x[i].qs + 4*ib;- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));- const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-#else- NO_DEVICE_CODE;-#endif+template <int qk, int qr, dequantize_kernel_t dequantize_kernel>+static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows) {+ // qk = quantized weights per x block+ // qr = number of quantized weights per data value in x block+ const int64_t row = (int64_t)blockIdx.x*blockDim.y + threadIdx.y;-}+ if (row >= nrows) {+ return;+ }-template<typename dst_t>-static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int tid = threadIdx.x;- const int64_t i = blockIdx.x;- const block_iq2_s * x = (const block_iq2_s *) vx;+ const int iter_stride = 2*GGML_CUDA_DMMV_X;+ const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter+ const int y_offset = qr == 1 ? 1 : qk/2;- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));- const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;- const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];- for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+// partial sum for each thread+#ifdef GGML_CUDA_F16+ half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics#else- NO_DEVICE_CODE;-#endif+ float tmp = 0.0f;+#endif // GGML_CUDA_F16-}+ for (int i = 0; i < ncols; i += iter_stride) {+ const int col = i + vals_per_iter*tid;+ const int64_t ib = ((int64_t)row*ncols + col)/qk; // x block index+ const int iqs = (col%qk)/qr; // x quant index+ const int iybs = col - col%qk; // y block start index-template<typename dst_t>-static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+// processing >2 values per i iter is faster for fast GPUs+#pragma unroll+ for (int j = 0; j < vals_per_iter; j += 2) {+ // process 2 vals per j iter- const int64_t i = blockIdx.x;- const block_iq3_xxs * x = (const block_iq3_xxs *) vx;+ // dequantize+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val+ dfloat2 v;+ dequantize_kernel(vx, ib, iqs + j/qr, v);- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * q3 = x[i].qs + 8*ib;- const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;- const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);- const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);- const uint32_t aux32 = gas[0] | (gas[1] << 16);- const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];- for (int j = 0; j < 4; ++j) {- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);- }+ // matrix multiplication+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2+#ifdef GGML_CUDA_F16+ tmp += __hmul2(v, {+ y[iybs + iqs + j/qr + 0],+ y[iybs + iqs + j/qr + y_offset]+ });#else- NO_DEVICE_CODE;-#endif+ tmp += v.x * y[iybs + iqs + j/qr + 0];+ tmp += v.y * y[iybs + iqs + j/qr + y_offset];+#endif // GGML_CUDA_F16+ }+ }+ // sum up partial sums and write back result+ tmp = warp_reduce_sum(tmp);++ if (tid == 0) {+#ifdef GGML_CUDA_F16+ dst[row] = tmp.x + tmp.y;+#else+ dst[row] = tmp;+#endif // GGML_CUDA_F16+ }}-template<typename dst_t>-static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int64_t i = blockIdx.x;- const block_iq3_s * x = (const block_iq3_s *) vx;-- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint8_t * qs = x[i].qs + 8*ib;- const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));- const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));- const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));- const uint8_t signs = x[i].signs[4*ib + il];- for (int j = 0; j < 4; ++j) {- y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);- y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);- }-#else- NO_DEVICE_CODE;-#endif--}--template<typename dst_t>-static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int64_t i = blockIdx.x;- const block_iq1_s * x = (const block_iq1_s *) vx;-- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;- const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);- uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;- grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;- grid32[0] &= 0x0f0f0f0f;- for (int j = 0; j < 8; ++j) {- y[j] = d * (q[j] + delta);- }-#else- NO_DEVICE_CODE;-#endif-+static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int64_t i = blockIdx.x;- const block_iq1_m * x = (const block_iq1_m *) vx;-- const int64_t tid = threadIdx.x;-#if QK_K == 256- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 8*il;- const uint16_t * sc = (const uint16_t *)x[i].scales;- iq1m_scale_t scale;- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);- const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);- const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);- const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;- uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;- grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;- grid32[0] &= 0x0f0f0f0f;- for (int j = 0; j < 8; ++j) {- y[j] = d * (q[j] + delta);- }-#else- NO_DEVICE_CODE;-#endif-+static void dequantize_mul_mat_vec_q4_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}--template<typename dst_t>-static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {-- const int64_t i = blockIdx.x;- const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);-- const int64_t tid = threadIdx.x;- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 4*il;- const uint8_t * q4 = x[ib].qs + 4*il;- const float d = (float)x[ib].d;- for (int j = 0; j < 4; ++j) {- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];- }-+static void dequantize_mul_mat_vec_q5_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-#if QK_K != 64-template<typename dst_t>-static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {- const int64_t i = blockIdx.x;- const block_iq4_xs * x = (const block_iq4_xs *)vx;-- const int64_t tid = threadIdx.x;- const int64_t il = tid/8; // 0...3- const int64_t ib = tid%8; // 0...7- dst_t * y = yy + i*QK_K + 32*ib + 4*il;- const uint8_t * q4 = x[i].qs + 16*ib + 4*il;- const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);- for (int j = 0; j < 4; ++j) {- y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];- }+static void dequantize_mul_mat_vec_q5_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-#endif-template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {- const int num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);- dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);+static void dequantize_mul_mat_vec_q8_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;- if (k % CUDA_Q8_0_NE_ALIGN == 0) {- const bool need_check = false;- dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);- } else {- const bool need_check = true;- dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);- }+static void dequantize_mul_mat_vec_q2_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q2_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q2_K<<<nb, 32, 0, stream>>>(vx, y);-#endif+static void dequantize_mul_mat_vec_q3_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q3_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q3_K<<<nb, 32, 0, stream>>>(vx, y);-#endif+static void dequantize_mul_mat_vec_q4_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q4_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb32 = k / 32;- const int nb = (k + 255) / 256;- dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);+static void dequantize_mul_mat_vec_q5_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const dim3 block_dims(32, 1, 1);+ dequantize_mul_mat_vec_q5_k<<<nrows, block_dims, 0, stream>>>(vx, y, dst, ncols);}-template<typename dst_t>-static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb32 = k / 32;- const int nb = (k + 255) / 256;- dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);+static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int ny = 2 / K_QUANTS_PER_ITERATION;+ const int block_num_y = (nrows + ny - 1) / ny;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(32, ny, 1);+ dequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);+static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ dequantize_mul_mat_vec<1, 1, convert_f16>+ <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);}-template<typename dst_t>-static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q5_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+void ggml_cuda_op_dequantize_mul_mat_vec(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {+ GGML_UNUSED(ctx);+ const int64_t ne00 = src0->ne[0];+ const int64_t row_diff = row_high - row_low;-template<typename dst_t>-static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;-#if QK_K == 256- dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);-#else- dequantize_block_q6_K<<<nb, 32, 0, stream>>>(vx, y);-#endif-}+ GGML_ASSERT(src1->type == GGML_TYPE_F32);-template<typename dst_t>-static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);-}--template<typename dst_t>-static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;-#if QK_K == 64- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);-#else- dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);-#endif-}--template <typename src_t, typename dst_t>-static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {- const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;-- if (i >= k) {- return;- }-- const src_t * x = (src_t *) vx;-- y[i] = x[i];-}+ // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics+#ifdef GGML_CUDA_F16+ ggml_cuda_pool_alloc<half> src1_dfloat_a(ctx.pool());+ half * src1_dfloat = nullptr; // dfloat == half-template <typename src_t, typename dst_t>-static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {- const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;- convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}+ bool src1_convert_f16 =+ src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||+ src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||+ src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;-to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {- int id;- switch (type) {- case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;- case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;- case GGML_TYPE_Q5_0:- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;- case GGML_TYPE_Q5_1:- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;- case GGML_TYPE_Q8_0:- CUDA_CHECK(cudaGetDevice(&id));- if (ggml_cuda_info().devices[id].cc >= CC_PASCAL) {- return dequantize_block_q8_0_f16_cuda;- }- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;- case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;- case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;- case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;- case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;- case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_IQ2_XXS:- return dequantize_row_iq2_xxs_cuda;- case GGML_TYPE_IQ2_XS:- return dequantize_row_iq2_xs_cuda;- case GGML_TYPE_IQ2_S:- return dequantize_row_iq2_s_cuda;- case GGML_TYPE_IQ3_XXS:- return dequantize_row_iq3_xxs_cuda;- case GGML_TYPE_IQ1_S:- return dequantize_row_iq1_s_cuda;- case GGML_TYPE_IQ1_M:- return dequantize_row_iq1_m_cuda;- case GGML_TYPE_IQ4_NL:- return dequantize_row_iq4_nl_cuda;- case GGML_TYPE_IQ4_XS:- return dequantize_row_iq4_xs_cuda;- case GGML_TYPE_IQ3_S:- return dequantize_row_iq3_s_cuda;- case GGML_TYPE_F32:- return convert_unary_cuda<float>;- default:- return nullptr;+ if (src1_convert_f16) {+ src1_dfloat = src1_dfloat_a.alloc(ne00);+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);}-}+#else+ const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion+#endif // GGML_CUDA_F16-to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {- switch (type) {+ switch (src0->type) {case GGML_TYPE_Q4_0:- return dequantize_row_q4_0_cuda;+ dequantize_mul_mat_vec_q4_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q4_1:- return dequantize_row_q4_1_cuda;+ dequantize_mul_mat_vec_q4_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q5_0:- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ dequantize_mul_mat_vec_q5_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q5_1:- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ dequantize_mul_mat_vec_q5_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q8_0:- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ dequantize_mul_mat_vec_q8_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q2_K:- return dequantize_row_q2_K_cuda;+ dequantize_mul_mat_vec_q2_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q3_K:- return dequantize_row_q3_K_cuda;+ dequantize_mul_mat_vec_q3_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q4_K:- return dequantize_row_q4_K_cuda;+ dequantize_mul_mat_vec_q4_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q5_K:- return dequantize_row_q5_K_cuda;+ dequantize_mul_mat_vec_q5_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_Q6_K:- return dequantize_row_q6_K_cuda;- case GGML_TYPE_IQ2_XXS:- return dequantize_row_iq2_xxs_cuda;- case GGML_TYPE_IQ2_XS:- return dequantize_row_iq2_xs_cuda;- case GGML_TYPE_IQ2_S:- return dequantize_row_iq2_s_cuda;- case GGML_TYPE_IQ3_XXS:- return dequantize_row_iq3_xxs_cuda;- case GGML_TYPE_IQ1_S:- return dequantize_row_iq1_s_cuda;- case GGML_TYPE_IQ1_M:- return dequantize_row_iq1_m_cuda;- case GGML_TYPE_IQ4_NL:- return dequantize_row_iq4_nl_cuda;- case GGML_TYPE_IQ4_XS:- return dequantize_row_iq4_xs_cuda;- case GGML_TYPE_IQ3_S:- return dequantize_row_iq3_s_cuda;+ dequantize_mul_mat_vec_q6_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);+ break;case GGML_TYPE_F16:- return convert_unary_cuda<half>;+ convert_mul_mat_vec_f16_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);+ break;default:- return nullptr;+ GGML_ASSERT(false);+ break;}-}--typedef void (*cpy_kernel_t)(const char * cx, char * cdst);--static __device__ void cpy_1_f32_f32(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- float * dsti = (float *) cdsti;-- *dsti = *xi;-}--static __device__ void cpy_1_f32_f16(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- half * dsti = (half *) cdsti;- *dsti = __float2half(*xi);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+ GGML_UNUSED(src1_ddq_i);+ GGML_UNUSED(src1_ncols);+ GGML_UNUSED(src1_padded_row_size);}-static __device__ void cpy_1_f16_f16(const char * cxi, char * cdsti) {- const half * xi = (const half *) cxi;- half * dsti = (half *) cdsti;-- *dsti = *xi;-}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn.cu+//+////////////////////////////////////////////////////////////////////////////////-static __device__ void cpy_1_f16_f32(const char * cxi, char * cdsti) {- const half * xi = (const half *) cxi;- float * dsti = (float *) cdsti;- *dsti = *xi;-}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-common.cuh+//+////////////////////////////////////////////////////////////////////////////////-template <cpy_kernel_t cpy_1>-static __global__ void cpy_f32_f16(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,- const int nb12, const int nb13) {- const int64_t i = blockDim.x*blockIdx.x + threadIdx.x;- if (i >= ne) {- return;- }- // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor- // then combine those indices with the corresponding byte offsets to get the total offsets- const int64_t i03 = i/(ne00 * ne01 * ne02);- const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);- const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;- const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;- const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;+#define FATTN_KQ_STRIDE 256+#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.+#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.- const int64_t i13 = i/(ne10 * ne11 * ne12);- const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);- const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;- const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;- const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;-- cpy_1(cx + x_offset, cdst + dst_offset);-}+typedef void (* fattn_kernel_t)(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3);-static __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q8_0 * dsti = (block_q8_0 *) cdsti;+template<int D, int parallel_blocks> // D == head size+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(D, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_combine_results(+ const float * __restrict__ VKQ_parts,+ const float2 * __restrict__ VKQ_meta,+ float * __restrict__ dst) {+ VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x;+ VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x;+ dst += D * gridDim.y*blockIdx.x;- float amax = 0.0f; // absolute max+ const int tid = threadIdx.x;+ __builtin_assume(tid < D);- for (int j = 0; j < QK8_0; j++) {- const float v = xi[j];- amax = fmaxf(amax, fabsf(v));+ __shared__ float2 meta[parallel_blocks];+ if (tid < 2*parallel_blocks) {+ ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid];}- const float d = amax / ((1 << 7) - 1);- const float id = d ? 1.0f/d : 0.0f;+ __syncthreads();- dsti->d = d;+ float kqmax = meta[0].x;+#pragma unroll+ for (int l = 1; l < parallel_blocks; ++l) {+ kqmax = max(kqmax, meta[l].x);+ }- for (int j = 0; j < QK8_0; ++j) {- const float x0 = xi[j]*id;+ float VKQ_numerator = 0.0f;+ float VKQ_denominator = 0.0f;+#pragma unroll+ for (int l = 0; l < parallel_blocks; ++l) {+ const float diff = meta[l].x - kqmax;+ const float KQ_max_scale = expf(diff);+ const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD);+ *((uint32_t *) &KQ_max_scale) &= ftz_mask;- dsti->qs[j] = roundf(x0);+ VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid];+ VKQ_denominator += KQ_max_scale * meta[l].y;}-}-static __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q4_0 * dsti = (block_q4_0 *) cdsti;+ dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator;+}- float amax = 0.0f;- float vmax = 0.0f;+template <int D, int parallel_blocks>+void launch_fattn(ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, int nwarps, int cols_per_block) {+ const ggml_tensor * Q = dst->src[0];+ const ggml_tensor * K = dst->src[1];+ const ggml_tensor * V = dst->src[2];- for (int j = 0; j < QK4_0; ++j) {- const float v = xi[j];- if (amax < fabsf(v)) {- amax = fabsf(v);- vmax = v;- }- }+ const ggml_tensor * mask = dst->src[3];- const float d = vmax / -8;- const float id = d ? 1.0f/d : 0.0f;+ ggml_tensor * KQV = dst;- dsti->d = d;+ GGML_ASSERT(Q->type == GGML_TYPE_F32);+ GGML_ASSERT(K->type == GGML_TYPE_F16);+ GGML_ASSERT(V->type == GGML_TYPE_F16);+ GGML_ASSERT(KQV->type == GGML_TYPE_F32);- for (int j = 0; j < QK4_0/2; ++j) {- const float x0 = xi[0 + j]*id;- const float x1 = xi[QK4_0/2 + j]*id;+ GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);+ GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) &&+ "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big");- const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));- const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));+ GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding.");- dsti->qs[j] = xi0;- dsti->qs[j] |= xi1 << 4;- }-}+ ggml_cuda_pool & pool = ctx.pool();+ cudaStream_t main_stream = ctx.stream();-static __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q4_1 * dsti = (block_q4_1 *) cdsti;+ ggml_cuda_pool_alloc<float> dst_tmp(pool);+ ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);- float vmin = FLT_MAX;- float vmax = -FLT_MAX;+ if (parallel_blocks > 1) {+ dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));+ dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));+ }- for (int j = 0; j < QK4_1; ++j) {- const float v = xi[j];+ const dim3 block_dim(WARP_SIZE, nwarps, 1);+ const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]);+ const int shmem = 0;- if (v < vmin) vmin = v;- if (v > vmax) vmax = v;- }+ float scale = 1.0f;+ float max_bias = 0.0f;- const float d = (vmax - vmin) / ((1 << 4) - 1);- const float id = d ? 1.0f/d : 0.0f;+ memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float));+ memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float));- dsti->dm.x = d;- dsti->dm.y = vmin;+ const uint32_t n_head = Q->ne[2];+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));- for (int j = 0; j < QK4_1/2; ++j) {- const float x0 = (xi[0 + j] - vmin)*id;- const float x1 = (xi[QK4_1/2 + j] - vmin)*id;+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);- const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));- const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));+ fattn_kernel<<<blocks_num, block_dim, shmem, main_stream>>>(+ (const char *) Q->data,+ (const char *) K->data,+ (const char *) V->data,+ mask ? ((const char *) mask->data) : nullptr,+ (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,+ scale, max_bias, m0, m1, n_head_log2,+ Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],+ K->ne[0], K->ne[1], K->ne[2], K->ne[3],+ mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,+ Q->nb[1], Q->nb[2], Q->nb[3],+ K->nb[1], K->nb[2], K->nb[3],+ KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]+ );+ CUDA_CHECK(cudaGetLastError());- dsti->qs[j] = xi0;- dsti->qs[j] |= xi1 << 4;+ if ((parallel_blocks) == 1) {+ return;}-}-static __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q5_0 * dsti = (block_q5_0 *) cdsti;+ const dim3 block_dim_combine(D, 1, 1);+ const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);+ const int shmem_combine = 0;- float amax = 0.0f;- float vmax = 0.0f;+ flash_attn_combine_results<D, parallel_blocks>+ <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>+ (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);+ CUDA_CHECK(cudaGetLastError());+}- for (int j = 0; j < QK5_0; ++j) {- const float v = xi[j];- if (amax < fabsf(v)) {- amax = fabsf(v);- vmax = v;- }- }+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-tile-f16.cuh+//+////////////////////////////////////////////////////////////////////////////////- const float d = vmax / -16;- const float id = d ? 1.0f/d : 0.0f;- dsti->d = d;+void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- uint32_t qh = 0;- for (int j = 0; j < QK5_0/2; ++j) {- const float x0 = xi[0 + j]*id;- const float x1 = xi[QK5_0/2 + j]*id;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-tile-f32.cuh+//+////////////////////////////////////////////////////////////////////////////////- const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));- const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));- dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);- qh |= ((xi0 & 0x10u) >> 4) << (j + 0);- qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);- }- memcpy(dsti->qh, &qh, sizeof(qh));-}+void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-static __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_q5_1 * dsti = (block_q5_1 *) cdsti;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-vec-f16.cuh+//+////////////////////////////////////////////////////////////////////////////////- float min = xi[0];- float max = xi[0];- for (int j = 1; j < QK5_1; ++j) {- const float v = xi[j];- min = v < min ? v : min;- max = v > max ? v : max;- }+void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- const float d = (max - min) / 31;- const float id = d ? 1.0f/d : 0.0f;+void ggml_cuda_flash_attn_ext_vec_f16_no_mma(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- dsti->dm.x = d;- dsti->dm.y = min;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-vec-f32.cuh+//+////////////////////////////////////////////////////////////////////////////////- uint32_t qh = 0;- for (int j = 0; j < QK5_1/2; ++j) {- const float x0 = (xi[0 + j] - min)*id;- const float x1 = (xi[QK5_1/2 + j] - min)*id;- const uint8_t xi0 = (uint8_t)(x0 + 0.5f);- const uint8_t xi1 = (uint8_t)(x1 + 0.5f);+void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);- qh |= ((xi0 & 0x10u) >> 4) << (j + 0);- qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);- }- memcpy(dsti->qh, &qh, sizeof(qh));-}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn.cuh+//+////////////////////////////////////////////////////////////////////////////////-static __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {- if (x <= val[0]) return 0;- if (x >= val[n-1]) return n-1;- int ml = 0, mu = n-1;- while (mu-ml > 1) {- int mav = (ml+mu)/2;- if (x < val[mav]) mu = mav; else ml = mav;- }- return x - val[mu-1] < val[mu] - x ? mu-1 : mu;-}+void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst);-static __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {- const float * xi = (const float *) cxi;- block_iq4_nl * dsti = (block_iq4_nl *) cdsti;- float amax = 0.0f;- float vmax = 0.0f;+#if FP16_MMA_AVAILABLE+#include <mma.h>+#endif- for (int j = 0; j < QK4_NL; ++j) {- const float v = xi[j];- if (amax < fabsf(v)) {- amax = fabsf(v);- vmax = v;+// D == head size, VKQ_stride == num VKQ rows calculated in parallel:+template<int D, int ncols, int nwarps, int VKQ_stride, int parallel_blocks, typename KQ_acc_t>+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(nwarps*WARP_SIZE, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_ext_f16(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+#if FP16_MMA_AVAILABLE+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");+ static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");+ constexpr int frag_m = ncols == 8 ? 32 : 16;+ constexpr int frag_n = ncols == 8 ? 8 : 16;+ static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::row_major> frag_a_K;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_a_V;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_b;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;+ typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;++ constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.+ constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.+ static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");++ // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:+ constexpr int D_padded = D + 8;+ constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;+ constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0;+ const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2);++ const int stride_Q = nb01 / sizeof(float);+ const int stride_KV = nb11 / sizeof(half);++ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);+ const half slopeh = __float2half(slopef);+ const half2 slope2 = make_half2(slopef, slopef);++ frag_b Q_b[D/16][ncols/frag_n];++ // A single buffer for temporarily holding tiles of KQ and VKQ parts:+ constexpr int mem_KQ = ncols*kqs_padded*kqar;+ constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;+ __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];+ float * KQ_f = (float *) KQ;+ half2 * KQ2 = (half2 *) KQ;++ float KQ_rowsum_f[ncols/nwarps] = {0.0f};+ float KQ_max_f[ncols/nwarps];+ float KQ_max_scale_f[ncols/nwarps] = {0.0f};++#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ KQ_max_f[j] = -FLT_MAX/2.0f;+ }++ half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};+ half2 KQ_max_h2[ncols/nwarps];+ half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};++#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);+ }++ __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.+ half2 * VKQ2 = (half2 *) VKQ;+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;+#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {+ break;+ }+ VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);+ }+ }++ // Convert Q to half and apply scale, temporarily store in KQ:+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;+#pragma unroll+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D && i >= D) {+ break;+ }+ KQ[j*D_padded + i] = ic0 + j < ne01 ? Q_f[j*stride_Q + i] * scale : 0.0f;+ }+ }++ __syncthreads();++ // Load Q into tensor core fragments/registers since it will be used frequently:+#pragma unroll+ for (int i0 = 0; i0 < D; i0 += 16) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ + j0*D_padded + i0, D_padded);+ }+ }++ __syncthreads();++ // Iterate over ne11 == previous tokens:+ for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) {+ // Calculate tile of KQ:+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {+ frag_c_KQ KQ_c[ncols/frag_n];+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::fill_fragment(KQ_c[j], 0.0f);+ }+#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {+ frag_a_K K_a;+ nvcuda::wmma::load_matrix_sync(K_a, K_h + (k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);+ }+ }+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, nvcuda::wmma::mem_col_major);+ }+ }++ __syncthreads();++ // Calculate softmax for each KQ column using the current max. value.+ // The divisor is stored in KQ_rowsum and will be applied at the end.+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ if (std::is_same<KQ_acc_t, float>::value) {+ float KQ_f_tmp[FATTN_KQ_STRIDE / WARP_SIZE];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ_f_tmp[k0/WARP_SIZE] = KQ_f[j*kqs_padded + k];+ }++ float KQ_max_new = KQ_max_f[j0/nwarps];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ_f_tmp[k0/WARP_SIZE] += mask ? __half2float(slopeh*maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;+ KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/WARP_SIZE]);+ }+ KQ_max_new = warp_reduce_max(KQ_max_new);++ const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;+ KQ_max_scale_f[j0/nwarps] = expf(diff);+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {+ KQ_max_scale_f[j0/nwarps] = 0.0f;+ }+ KQ_max_f[j0/nwarps] = KQ_max_new;++ float KQ_rowsum_add = 0.0f;+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ const float diff = KQ_f_tmp[k0/WARP_SIZE] - KQ_max_f[j0/nwarps];+ KQ_f_tmp[k0/WARP_SIZE] = expf(diff);+ if (diff <= SOFTMAX_FTZ_THRESHOLD) {+ KQ_f_tmp[k0/WARP_SIZE] = 0.0f;+ }+ KQ_rowsum_add += KQ_f_tmp[k0/WARP_SIZE];+ KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/WARP_SIZE];+ }+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);++ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:+ KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;+ } else {+ half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*WARP_SIZE)];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ2_tmp[k0/WARP_SIZE] = KQ2[j*(kqs_padded/2) + k];+ }++ half2 KQ_max_new = KQ_max_h2[j0/nwarps];+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ KQ2_tmp[k0/WARP_SIZE] += mask ? slope2*mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);+ KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/WARP_SIZE]);+ }+ KQ_max_new = __half2half2(warp_reduce_max(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));+ const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;+ KQ_max_scale_h2[j0/nwarps] = h2exp(diff);+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));+ *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;+ KQ_max_h2[j0/nwarps] = KQ_max_new;++ half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {+ const int k = k0 + threadIdx.x;++ const half2 diff = KQ2_tmp[k0/WARP_SIZE] - KQ_max_h2[j0/nwarps];+ KQ2_tmp[k0/WARP_SIZE] = h2exp(diff);+ const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));+ *((uint32_t *) &KQ2_tmp[k0/WARP_SIZE]) &= ftz_mask;+ KQ_rowsum_add += KQ2_tmp[k0/WARP_SIZE];+ KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/WARP_SIZE];+ }+ KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);++ // Scale previous KQ_rowsum to account for a potential increase in KQ_max:+ KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;+ }+ }++ __syncthreads();++ frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;+ nvcuda::wmma::load_matrix_sync(+ KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],+ KQ + j0*(kqar*kqs_padded) + k,+ kqar*kqs_padded);+ }+ }++ frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];+#pragma unroll+ for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], 0.0f);+ }++#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {+ const int k = k0 + (threadIdx.y % VKQ_ratio)*16;++ frag_a_V v_a;+ nvcuda::wmma::load_matrix_sync(v_a, V_h + (k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);+#pragma unroll+ for (int j = 0; j < ncols/frag_n; ++j) {+ nvcuda::wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);+ }+ }+ }++ __syncthreads();++ const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += frag_n) {+ nvcuda::wmma::store_matrix_sync(+ KQ + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),+ VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],+ D_padded, nvcuda::wmma::mem_col_major);+ }+ }++ __syncthreads();++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ half2 VKQ_scale;+ if (std::is_same<KQ_acc_t, float>::value) {+ VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);+ } else {+ VKQ_scale = KQ_max_scale_h2[j0/nwarps];+ }++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D/2 && i >= D/2) {+ break;+ }++ half2 VKQ_add = make_half2(0.0f, 0.0f);+#pragma unroll+ for (int l = 0; l < VKQ_ratio; ++l) {+ VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];+ }+ VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;+ }+ }++ __syncthreads();+ }++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j_VKQ = j0 + threadIdx.y;+ if (ic0 + j_VKQ >= ne01) {+ return;+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;++ float KQ_rowsum_j;+ if (std::is_same<KQ_acc_t, float>::value) {+ KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];+ } else {+ KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);+ }++#pragma unroll+ for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;+ if (i0 + WARP_SIZE > D && i >= D) {+ break;+ }+ float dst_val = VKQ[j_VKQ*D_padded + i];+ if (parallel_blocks == 1) {+ dst_val /= KQ_rowsum_j;+ }+ dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val;+ }++ if (parallel_blocks == 1 || threadIdx.x != 0) {+ continue;+ }++ float2 dst_meta_val;+ if (std::is_same<KQ_acc_t, float>::value) {+ dst_meta_val.x = KQ_max_f[j0/nwarps];+ } else {+ dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);+ }+ dst_meta_val.y = KQ_rowsum_j;+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val;+ }+#else+ NO_DEVICE_CODE;+#endif // FP16_MMA_AVAILABLE+}++constexpr int get_max_power_of_2(int x) {+ return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;+}++static_assert(get_max_power_of_2(1) == 1, "Test failed.");+static_assert(get_max_power_of_2(2) == 2, "Test failed.");+static_assert(get_max_power_of_2(4) == 4, "Test failed.");+static_assert(get_max_power_of_2(6) == 2, "Test failed.");++// Number of VKQ rows calculated in parallel:+constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {+ return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;+}++static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");+static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");+static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");+static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");+static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");+static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");+static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");++template <int D, int cols_per_block, int nwarps, typename KQ_acc_t>+void launch_fattn_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];++ constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16;+ const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3];+ const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;++ if (4*blocks_num_pb1 < 2*nsm) {+ constexpr int parallel_blocks = 4;+ fattn_kernel_t fattn_kernel = flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ return;+ }+ if (2*blocks_num_pb1 < 2*nsm) {+ constexpr int parallel_blocks = 2;+ fattn_kernel_t fattn_kernel = flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ return;+ }+ constexpr int parallel_blocks = 1;+ fattn_kernel_t fattn_kernel = flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+}++void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * KQV = dst;+ const ggml_tensor * Q = dst->src[0];++ ggml_cuda_set_device(ctx.device);+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ const int32_t precision = KQV->op_params[2];++ // On AMD the tile kernels perform poorly, use the vec kernel instead:+ if (cc >= CC_OFFSET_AMD) {+ if (precision == GGML_PREC_DEFAULT) {+ ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst);+ } else {+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);+ }+ return;+ }++ if (!fast_fp16_available(cc)) {+ if (Q->ne[1] <= 8) {+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);+ } else {+ ggml_cuda_flash_attn_ext_tile_f32(ctx, dst);+ }+ return;+ }++ if (!fp16_mma_available(cc)) {+ if (Q->ne[1] <= 8) {+ ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst);+ } else {+ ggml_cuda_flash_attn_ext_tile_f16(ctx, dst);+ }+ return;+ }++ if (precision != GGML_PREC_DEFAULT) {+ if (Q->ne[1] == 1 && (Q->ne[0] == 64 || Q->ne[0] == 128)) {+ ggml_cuda_flash_attn_ext_vec_f32(ctx, dst);+ return;+ }++ if (Q->ne[1] <= 32 || Q->ne[0] > 128) {+ constexpr int cols_per_block = 16;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, float>(ctx, dst);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ } else {+ constexpr int cols_per_block = 32;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, float>(ctx, dst);+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, float>(ctx, dst);+ break;+ // case 256:+ // launch_fattn_f16<256, cols_per_block, nwarps, float>(ctx, dst);+ // break;+ default:+ GGML_ASSERT(false);+ break;+ }+ }+ return;+ }++ if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) {+ ggml_cuda_flash_attn_ext_vec_f16(ctx, dst);+ return;+ }++ if (Q->ne[1] <= 8 && Q->ne[0] % WARP_SIZE == 0) {+ constexpr int cols_per_block = 8;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+ }++ if (Q->ne[1] <= 32) {+ constexpr int cols_per_block = 16;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+ }++ constexpr int cols_per_block = 32;+ constexpr int nwarps = 4;+ switch (Q->ne[0]) {+ case 64:+ launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 80:+ launch_fattn_f16< 80, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 96:+ launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 112:+ launch_fattn_f16<112, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 128:+ launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst);+ break;+ case 256:+ launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst);+ break;+ default:+ GGML_ASSERT(false);+ break;+ }+ return;+}++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-tile-f16.cu+//+////////////////////////////////////////////////////////////////////////////////+++#define FATTN_KQ_STRIDE_TILE_F16 64++template<int D, int ncols, int nwarps, int parallel_blocks> // D == head size+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(nwarps*WARP_SIZE, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_tile_ext_f16(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+#if FP16_AVAILABLE+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + ne11*ic0;++ const int stride_KV2 = nb11 / sizeof(half2);++ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);+ const half slopeh = __float2half(slopef);++ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");++ __shared__ half KQ[ncols*FATTN_KQ_STRIDE_TILE_F16];+ half2 * KQ2 = (half2 *) KQ;++ __shared__ half2 KV_tmp[FATTN_KQ_STRIDE_TILE_F16][D/2 + 1]; // Pad D to avoid memory bank conflicts.++ half kqmax[ncols/nwarps];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ kqmax[j0/nwarps] = -HALF_MAX_HALF;+ }+ half2 kqsum[ncols/nwarps] = {{0.0f, 0.0f}};++ half2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}};++ // Convert Q to half2 and store in registers:+ __shared__ half2 Q_h2[ncols][D/2];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i];+ Q_h2[j][i] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y);+ }+ }++ __syncthreads();++ const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F16;+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F16) {+ // Calculate KQ tile and keep track of new maximum KQ values:++ half kqmax_new[ncols/nwarps];+#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ kqmax_new[j] = kqmax[j];+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += nwarps) {+ const int i_KQ = i_KQ_0 + threadIdx.y;++#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {+ const int k_KQ = k_KQ_0 + threadIdx.x;++ KV_tmp[i_KQ][k_KQ] = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];+ }+ }++ __syncthreads();++ half2 sum2[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE][ncols/nwarps] = {{{0.0f, 0.0f}}};++#pragma unroll+ for (int k_KQ = 0; k_KQ < D/2; ++k_KQ) {+ half2 K_k[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE];+ half2 Q_k[ncols/nwarps];++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {+ const int i_KQ = i_KQ_0 + threadIdx.x;++ K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ];+ }+#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ const int j_KQ = j_KQ_0 + threadIdx.y;++ Q_k[j_KQ_0/nwarps] = Q_h2[j_KQ][k_KQ];+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {+#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE]*Q_k[j_KQ_0/nwarps];+ }+ }+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) {+ const int i_KQ = i_KQ_0 + threadIdx.x;++#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ const int j_KQ = j_KQ_0 + threadIdx.y;++ half sum = __low2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]) + __high2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);+ sum += mask ? slopeh*maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f);++ kqmax_new[j_KQ_0/nwarps] = ggml_cuda_hmax(kqmax_new[j_KQ_0/nwarps], sum);++ KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F16 + i_KQ] = sum;+ }+ }++ __syncthreads();++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]);+ const half2 KQ_max_scale = __half2half2(hexp(kqmax[j0/nwarps] - kqmax_new[j0/nwarps]));+ kqmax[j0/nwarps] = kqmax_new[j0/nwarps];++#pragma unroll+ for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F16/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ const half2 diff = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] - __half2half2(kqmax[j0/nwarps]);+ const half2 val = h2exp(diff);+ kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + val;+ KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] = val;+ }++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ VKQ[j0/nwarps][i0/WARP_SIZE] *= KQ_max_scale;+ }+ }++ __syncthreads();++#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += nwarps) {+ const int k = k0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ KV_tmp[k][i] = V_h2[(k_VKQ_0 + k)*stride_KV2 + i];+ }+ }++ __syncthreads();++#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += 2) {+ half2 V_k[(D/2)/WARP_SIZE][2];+ half2 KQ_k[ncols/nwarps];++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ V_k[i0/WARP_SIZE][0] = KV_tmp[k0 + 0][i];+ V_k[i0/WARP_SIZE][1] = KV_tmp[k0 + 1][i];+ }+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ KQ_k[j0/nwarps] = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + k0/2];+ }++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][0]* __low2half2(KQ_k[j0/nwarps]);+ VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][1]*__high2half2(KQ_k[j0/nwarps]);+ }+ }+ }++ __syncthreads();+ }++#pragma unroll+ for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) {+ const int j_VKQ = j_VKQ_0 + threadIdx.y;++ half kqsum_j = __low2half(kqsum[j_VKQ_0/nwarps]) + __high2half(kqsum[j_VKQ_0/nwarps]);+ kqsum_j = warp_reduce_sum(kqsum_j);++#pragma unroll+ for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) {+ const int i0 = i00 + 2*threadIdx.x;++ half2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)];+ if (parallel_blocks == 1) {+ dst_val /= __half2half2(kqsum_j);+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = __low2float(dst_val);+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = __high2float(dst_val);+ }++ if (parallel_blocks != 1 && threadIdx.x == 0) {+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j);+ }+ }+#else+ NO_DEVICE_CODE;+#endif // FP16_AVAILABLE+}++template <int cols_per_block, int parallel_blocks>+void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];+ switch (Q->ne[0]) {+ case 64: {+ constexpr int D = 64;+ constexpr int nwarps = 8;+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16<D, cols_per_block, nwarps, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 128: {+ constexpr int D = 128;+ constexpr int nwarps = 8;+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16<D, cols_per_block, nwarps, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ default: {+ GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128.");+ } break;+ }+}++void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * KQV = dst;+ const ggml_tensor * Q = dst->src[0];++ const int32_t precision = KQV->op_params[2];+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);++ if (Q->ne[1] <= 16) {+ constexpr int cols_per_block = 16;+ constexpr int parallel_blocks = 4;+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }++ if (Q->ne[1] <= 32) {+ constexpr int cols_per_block = 32;+ constexpr int parallel_blocks = 4;+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }++ constexpr int cols_per_block = 32;+ constexpr int parallel_blocks = 1;+ launch_fattn_tile_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+}++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-tile-f32.cu+//+////////////////////////////////////////////////////////////////////////////////+++#define FATTN_KQ_STRIDE_TILE_F32 32++template<int D, int ncols, int nwarps, int parallel_blocks> // D == head size+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(nwarps*WARP_SIZE, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_tile_ext_f32(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + ne11*ic0;++ const int stride_KV2 = nb11 / sizeof(half2);++ const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);++ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");++ __shared__ float KQ[ncols*FATTN_KQ_STRIDE_TILE_F32];++ __shared__ float KV_tmp[FATTN_KQ_STRIDE_TILE_F32][D + 1]; // Pad D to avoid memory bank conflicts.+ float2 * KV_tmp2 = (float2 *) KV_tmp;++ float kqmax[ncols/nwarps];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ kqmax[j0/nwarps] = -FLT_MAX/2.0f;+ }+ float kqsum[ncols/nwarps] = {0.0f};++ float2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}};++ // Convert Q to half2 and store in registers:+ __shared__ float Q_f[ncols][D];+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < D; i0 += 2*WARP_SIZE) {+ float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i0/2 + threadIdx.x];+ Q_f[j][i0 + 0*WARP_SIZE + threadIdx.x] = tmp.x * scale;+ Q_f[j][i0 + 1*WARP_SIZE + threadIdx.x] = tmp.y * scale;}}- float d = vmax / kvalues_iq4nl[0];- const float id = d ? 1.0f/d : 0.0f;+ __syncthreads();- float sumqx = 0, sumq2 = 0;- for (int j = 0; j < QK4_NL/2; ++j) {- const float x0 = xi[0 + j]*id;- const float x1 = xi[QK4_NL/2 + j]*id;- const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);- const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);- dsti->qs[j] = xi0 | (xi1 << 4);- const float v0 = kvalues_iq4nl[xi0];- const float v1 = kvalues_iq4nl[xi1];- const float w0 = xi[0 + j]*xi[0 + j];- const float w1 = xi[QK4_NL/2 + j]*xi[QK4_NL/2 + j];- sumqx += w0*v0*xi[j] + w1*v1*xi[QK4_NL/2 + j];- sumq2 += w0*v0*v0 + w1*v1*v1;+ const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F32;+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F32) {+ // Calculate KQ tile and keep track of new maximum KQ values:++ float kqmax_new[ncols/nwarps];+#pragma unroll+ for (int j = 0; j < ncols/nwarps; ++j) {+ kqmax_new[j] = kqmax[j];+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += nwarps) {+ const int i_KQ = i_KQ_0 + threadIdx.y;++#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 2*WARP_SIZE) {+ const half2 tmp = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ_0/2 + threadIdx.x];+ KV_tmp[i_KQ][k_KQ_0 + 0*WARP_SIZE + threadIdx.x] = __low2float(tmp);+ KV_tmp[i_KQ][k_KQ_0 + 1*WARP_SIZE + threadIdx.x] = __high2float(tmp);+ }+ }++ __syncthreads();++ float sum[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE][ncols/nwarps] = {{0.0f}};++#pragma unroll+ for (int k_KQ = 0; k_KQ < D; ++k_KQ) {+ float K_k[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE];+ float Q_k[ncols/nwarps];++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {+ const int i_KQ = i_KQ_0 + threadIdx.x;++ K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ];+ }+#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ const int j_KQ = j_KQ_0 + threadIdx.y;++ Q_k[j_KQ_0/nwarps] = Q_f[j_KQ][k_KQ];+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {+#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE] * Q_k[j_KQ_0/nwarps];+ }+ }+ }++#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) {+ const int i_KQ = i_KQ_0 + threadIdx.x;++#pragma unroll+ for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) {+ const int j_KQ = j_KQ_0 + threadIdx.y;++ sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += mask ? slope*__half2float(maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ]) : 0.0f;++ kqmax_new[j_KQ_0/nwarps] = fmaxf(kqmax_new[j_KQ_0/nwarps], sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]);++ KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F32 + i_KQ] = sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps];+ }+ }++ __syncthreads();++#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]);+ const float KQ_max_scale = expf(kqmax[j0/nwarps] - kqmax_new[j0/nwarps]);+ kqmax[j0/nwarps] = kqmax_new[j0/nwarps];++ float kqsum_add = 0.0f;+#pragma unroll+ for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F32; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ const float diff = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] - kqmax[j0/nwarps];+ const float val = expf(diff);+ kqsum_add += val;+ KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] = val;+ }+ kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + kqsum_add;++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ VKQ[j0/nwarps][i0/WARP_SIZE].x *= KQ_max_scale;+ VKQ[j0/nwarps][i0/WARP_SIZE].y *= KQ_max_scale;+ }+ }++ __syncthreads();++#pragma unroll+ for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F32; k0 += nwarps) {+ const int k = k0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ KV_tmp2[k*(D/2) + i].x = __low2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]);+ KV_tmp2[k*(D/2) + i].y = __high2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]);+ }+ }++ __syncthreads();++#pragma unroll+ for (int k = 0; k < FATTN_KQ_STRIDE_TILE_F32; ++k) {+ float2 V_k[(D/2)/WARP_SIZE];+ float KQ_k[ncols/nwarps];++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ V_k[i0/WARP_SIZE] = KV_tmp2[k*(D/2) + i];+ }+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ KQ_k[j0/nwarps] = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + k];+ }++#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+#pragma unroll+ for (int j0 = 0; j0 < ncols; j0 += nwarps) {+ VKQ[j0/nwarps][i0/WARP_SIZE].x += V_k[i0/WARP_SIZE].x*KQ_k[j0/nwarps];+ VKQ[j0/nwarps][i0/WARP_SIZE].y += V_k[i0/WARP_SIZE].y*KQ_k[j0/nwarps];+ }+ }+ }++ __syncthreads();}- dsti->d = sumq2 > 0 ? sumqx/sumq2 : d;+#pragma unroll+ for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) {+ const int j_VKQ = j_VKQ_0 + threadIdx.y;++ float kqsum_j = kqsum[j_VKQ_0/nwarps];+ kqsum_j = warp_reduce_sum(kqsum_j);++#pragma unroll+ for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) {+ const int i0 = i00 + 2*threadIdx.x;++ float2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)];+ if (parallel_blocks == 1) {+ dst_val.x /= kqsum_j;+ dst_val.y /= kqsum_j;+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = dst_val.x;+ dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = dst_val.y;+ }++ if (parallel_blocks != 1 && threadIdx.x == 0) {+ dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j);+ }+ }}-template <cpy_kernel_t cpy_blck, int qk>-static __global__ void cpy_f32_q(const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11,- const int nb12, const int nb13) {- const int i = (blockDim.x*blockIdx.x + threadIdx.x)*qk;+template <int cols_per_block, int parallel_blocks>+void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];+ switch (Q->ne[0]) {+ case 64: {+ constexpr int D = 64;+ constexpr int nwarps = 8;+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32<D, cols_per_block, nwarps, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 128: {+ constexpr int D = 128;+ constexpr int nwarps = 8;+ fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32<D, cols_per_block, nwarps, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ default: {+ GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128.");+ } break;+ }+}++void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * KQV = dst;+ const ggml_tensor * Q = dst->src[0];++ const int32_t precision = KQV->op_params[2];+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);++ if (Q->ne[1] <= 16) {+ constexpr int cols_per_block = 16;+ constexpr int parallel_blocks = 4;+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }++ if (Q->ne[1] <= 32) {+ constexpr int cols_per_block = 32;+ constexpr int parallel_blocks = 4;+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }++ constexpr int cols_per_block = 32;+ constexpr int parallel_blocks = 1;+ launch_fattn_tile_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+}++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-vec-f16.cu+//+////////////////////////////////////////////////////////////////////////////////+++template<int D, int ncols, int parallel_blocks> // D == head size+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(D, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_vec_ext_f16(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+#if FP16_AVAILABLE+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.++ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.++ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + ne11*ic0;++ const int stride_KV = nb11 / sizeof(half);+ const int stride_KV2 = nb11 / sizeof(half2);++ const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);+ const half slopeh = __float2half(slopef);++ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");+ constexpr int nwarps = D / WARP_SIZE;+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ __builtin_assume(tid < D);++ __shared__ half KQ[ncols*D];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ KQ[j*D + tid] = -HALF_MAX_HALF;+ }+ half2 * KQ2 = (half2 *) KQ;++ half kqmax[ncols];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqmax[j] = -HALF_MAX_HALF;+ }+ half kqsum[ncols] = {0.0f};++ __shared__ half kqmax_shared[ncols][WARP_SIZE];+ __shared__ half kqsum_shared[ncols][WARP_SIZE];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ if (threadIdx.y == 0) {+ kqmax_shared[j][threadIdx.x] = -HALF_MAX_HALF;+ kqsum_shared[j][threadIdx.x] = 0.0f;+ }+ }+ __syncthreads();++ // Convert Q to half2 and store in registers:+ half2 Q_h2[ncols][D/(2*WARP_SIZE)];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;++ const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i];+ Q_h2[j][i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y);+ }+ }++ half2 VKQ[ncols] = {{0.0f, 0.0f}};++ const int k_start = parallel_blocks == 1 ? 0 : ip*D;+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {+ // Calculate KQ tile and keep track of new maximum KQ values:++ // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression,+ // see https://github.com/ggerganov/llama.cpp/pull/7061 .+ // Therefore this variable is defined twice but only used once (so that the compiler can optimize out the unused variable).+ half kqmax_new = kqmax[0];+ half kqmax_new_arr[ncols];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqmax_new_arr[j] = kqmax[j];+ }- if (i >= ne) {- return;- }+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {+ const int i_KQ = i_KQ_0 + threadIdx.y;- const int i03 = i/(ne00 * ne01 * ne02);- const int i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);- const int i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;- const int i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;- const int x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;+ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {+ break;+ }- const int i13 = i/(ne10 * ne11 * ne12);- const int i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);- const int i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;- const int i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;- const int dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;+ half2 sum2[ncols] = {{0.0f, 0.0f}};+#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {+ const int k_KQ = k_KQ_0 + threadIdx.x;- cpy_blck(cx + x_offset, cdst + dst_offset);-}+ const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ sum2[j] += K_ik * Q_h2[j][k_KQ_0/WARP_SIZE];+ }+ }-static void ggml_cpy_f16_f32_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ sum2[j] = warp_reduce_sum(sum2[j]);+ half sum = __low2half(sum2[j]) + __high2half(sum2[j]);+ sum += mask ? slopeh*maskh[j*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f);- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f16_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ if (ncols == 1) {+ kqmax_new = ggml_cuda_hmax(kqmax_new, sum);+ } else {+ kqmax_new_arr[j] = ggml_cuda_hmax(kqmax_new_arr[j], sum);+ }-static void ggml_cpy_f32_f32_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ if (threadIdx.x == 0) {+ KQ[j*D + i_KQ] = sum;+ }+ }+ }- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f32_f32><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ half kqmax_new_j = ncols == 1 ? kqmax_new : kqmax_new_arr[j];-static void ggml_cpy_f32_f16_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ kqmax_new_j = warp_reduce_max(kqmax_new_j);+ if (threadIdx.x == 0) {+ kqmax_shared[j][threadIdx.y] = kqmax_new_j;+ }+ }- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f32_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ __syncthreads();-static void ggml_cpy_f32_q8_0_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ half kqmax_new_j = kqmax_shared[j][threadIdx.x];+ kqmax_new_j = warp_reduce_max(kqmax_new_j);- GGML_ASSERT(ne % QK8_0 == 0);- const int num_blocks = ne / QK8_0;- cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ const half KQ_max_scale = hexp(kqmax[j] - kqmax_new_j);+ kqmax[j] = kqmax_new_j;-static void ggml_cpy_f32_q4_0_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ const half val = hexp(KQ[j*D + tid] - kqmax[j]);+ kqsum[j] = kqsum[j]*KQ_max_scale + val;+ KQ[j*D + tid] = val;- GGML_ASSERT(ne % QK4_0 == 0);- const int num_blocks = ne / QK4_0;- cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ VKQ[j] *= __half2half2(KQ_max_scale);+ }-static void ggml_cpy_f32_q4_1_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ __syncthreads();- GGML_ASSERT(ne % QK4_1 == 0);- const int num_blocks = ne / QK4_1;- cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+#pragma unroll+ for (int k0 = 0; k0 < D; k0 += 2) {+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) {+ break;+ }-static void ggml_cpy_f32_q5_0_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+ half2 V_k;+ reinterpret_cast<half&>(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid];+ reinterpret_cast<half&>(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ VKQ[j] += V_k*KQ2[j*(D/2) + k0/2];+ }+ }- GGML_ASSERT(ne % QK5_0 == 0);- const int num_blocks = ne / QK5_0;- cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ __syncthreads();+ }-static void ggml_cpy_f32_q5_1_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqsum[j] = warp_reduce_sum(kqsum[j]);+ if (threadIdx.x == 0) {+ kqsum_shared[j][threadIdx.y] = kqsum[j];+ }+ }- GGML_ASSERT(ne % QK5_1 == 0);- const int num_blocks = ne / QK5_1;- cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ __syncthreads();-static void ggml_cpy_f32_iq4_nl_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+#pragma unroll+ for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {+ kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x];+ kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]);- GGML_ASSERT(ne % QK4_NL == 0);- const int num_blocks = ne / QK4_NL;- cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);+ half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ]));+ if (parallel_blocks == 1) {+ dst_val /= kqsum[j_VKQ];+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;+ dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val;+ }++ if (parallel_blocks != 1 && tid < ncols) {+ dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]);+ }+#else+ NO_DEVICE_CODE;+#endif // FP16_AVAILABLE}-static void ggml_cpy_f16_f16_cuda(- const char * cx, char * cdst, const int ne,- const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02,- const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, cudaStream_t stream) {+void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ ggml_tensor * KQV = dst;+ ggml_tensor * Q = dst->src[0];- const int num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;- cpy_f32_f16<cpy_1_f16_f16><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>- (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);-}+ const int32_t precision = KQV->op_params[2];+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);-void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {- const int64_t ne = ggml_nelements(src0);- GGML_ASSERT(ne == ggml_nelements(src1));+ constexpr int cols_per_block = 1;+ constexpr int parallel_blocks = 4;+ switch (Q->ne[0]) {+ case 64: {+ constexpr int D = 64;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 128: {+ constexpr int D = 128;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 256: {+ constexpr int D = 256;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ default:+ GGML_ASSERT(false);+ break;+ }+}- GGML_ASSERT(ggml_nbytes(src0) <= INT_MAX);- GGML_ASSERT(ggml_nbytes(src1) <= INT_MAX);+template <int cols_per_block, int parallel_blocks>+void launch_fattn_vec_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];+ switch (Q->ne[0]) {+ case 64: {+ constexpr int D = 64;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 128: {+ constexpr int D = 128;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ default: {+ GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128.");+ } break;+ }+}++void ggml_cuda_flash_attn_ext_vec_f16_no_mma(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * KQV = dst;+ const ggml_tensor * Q = dst->src[0];++ const int32_t precision = KQV->op_params[2];+ GGML_ASSERT(precision == GGML_PREC_DEFAULT);++ if (Q->ne[1] == 1) {+ ggml_cuda_flash_attn_ext_vec_f16(ctx, dst);+ return;+ }- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];+ if (Q->ne[1] == 2) {+ constexpr int cols_per_block = 2;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }- //GGML_ASSERT(src0->ne[3] == 1);+ if (Q->ne[1] <= 4) {+ constexpr int cols_per_block = 4;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }- const int64_t nb00 = src0->nb[0];- const int64_t nb01 = src0->nb[1];- const int64_t nb02 = src0->nb[2];- const int64_t nb03 = src0->nb[3];+ if (Q->ne[1] <= 8) {+ constexpr int cols_per_block = 8;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];+ constexpr int cols_per_block = 8;+ constexpr int parallel_blocks = 1;+ launch_fattn_vec_f16_64_128<cols_per_block, parallel_blocks>(ctx, dst);+}- //GGML_ASSERT(src1->ne[3] == 1);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP fattn-vec-f32.cu+//+////////////////////////////////////////////////////////////////////////////////- const int64_t nb10 = src1->nb[0];- const int64_t nb11 = src1->nb[1];- const int64_t nb12 = src1->nb[2];- const int64_t nb13 = src1->nb[3];- cudaStream_t main_stream = ctx.stream();+template<int D, int ncols, int parallel_blocks> // D == head size+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+__launch_bounds__(D, 1)+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+static __global__ void flash_attn_vec_ext_f32(+ const char * __restrict__ Q,+ const char * __restrict__ K,+ const char * __restrict__ V,+ const char * __restrict__ mask,+ float * __restrict__ dst,+ float2 * __restrict__ dst_meta,+ const float scale,+ const float max_bias,+ const float m0,+ const float m1,+ const uint32_t n_head_log2,+ const int ne00,+ const int ne01,+ const int ne02,+ const int ne03,+ const int ne10,+ const int ne11,+ const int ne12,+ const int ne13,+ const int ne31,+ const int nb31,+ const int nb01,+ const int nb02,+ const int nb03,+ const int nb11,+ const int nb12,+ const int nb13,+ const int ne0,+ const int ne1,+ const int ne2,+ const int ne3) {+ //In this kernel Q, K, V are matrices while i, j, k are matrix indices.- char * src0_ddc = (char *) src0->data;- char * src1_ddc = (char *) src1->data;+ const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on.+ const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.- if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f32_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f32_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {- ggml_cpy_f32_q8_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {- ggml_cpy_f32_q4_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {- ggml_cpy_f32_q4_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {- ggml_cpy_f32_q5_0_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {- ggml_cpy_f32_iq4_nl_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {- ggml_cpy_f32_q5_1_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {- ggml_cpy_f16_f16_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {- ggml_cpy_f16_f32_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);- } else {- fprintf(stderr, "%s: unsupported type combination (%s to %s)\n", __func__,- ggml_type_name(src0->type), ggml_type_name(src1->type));- GGML_ASSERT(false);- }-}+ const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.+ const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0);+ const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));+ const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape+ const half * maskh = (const half *) mask + ne11*ic0;-void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- ggml_cuda_cpy(ctx, src0, dst);-}+ const int stride_KV = nb11 / sizeof(half);+ const int stride_KV2 = nb11 / sizeof(half2);-static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {- const int col = blockDim.y*blockIdx.y + threadIdx.y;- const int row = blockDim.x*blockIdx.x + threadIdx.x;+ const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1);- if (col >= ncols) {- return;- }+ static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64.");+ constexpr int nwarps = D / WARP_SIZE;+ const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;+ __builtin_assume(tid < D);- const int i = row*ncols + col;- //dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];- //dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU- dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;-}+ __shared__ float KQ[ncols*D];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ KQ[j*D + tid] = -FLT_MAX/2.0f;+ }-static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {- const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);- const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;- const dim3 block_nums(nrows_x, block_num_x, 1);- diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);-}+ float kqmax[ncols];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqmax[j] = -FLT_MAX/2.0f;+ }+ float kqsum[ncols] = {0.0f};-void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const float * src0_d = (const float *)src0->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+ __shared__ float kqmax_shared[ncols][WARP_SIZE];+ __shared__ float kqsum_shared[ncols][WARP_SIZE];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ if (threadIdx.y == 0) {+ kqmax_shared[j][threadIdx.x] = -FLT_MAX/2.0f;+ kqsum_shared[j][threadIdx.x] = 0.0f;+ }+ }+ __syncthreads();- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ // Convert Q to half2 and store in registers:+ float2 Q_h2[ncols][D/(2*WARP_SIZE)];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+#pragma unroll+ for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {+ const int i = i0 + threadIdx.x;- const int64_t ne00 = src0->ne[0];- const int64_t ne01 = src0->ne[1];- const int nrows0 = ggml_nrows(src0);+ Q_h2[j][i0/WARP_SIZE] = Q_f2[j*(nb01/sizeof(float2)) + i];+ Q_h2[j][i0/WARP_SIZE].x *= scale;+ Q_h2[j][i0/WARP_SIZE].y *= scale;+ }+ }- const int n_past = ((int32_t *) dst->op_params)[0];+ float VKQ[ncols] = {0.0f};- diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);-}+ const int k_start = parallel_blocks == 1 ? 0 : ip*D;+ for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {+ // Calculate KQ tile and keep track of new maximum KQ values:-#ifndef K_QUANTS_PER_ITERATION-#define K_QUANTS_PER_ITERATION 2-#else-static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2");-#endif+ float kqmax_new_arr[ncols];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqmax_new_arr[j] = kqmax[j];+ }-static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {+#pragma unroll+ for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {+ const int i_KQ = i_KQ_0 + threadIdx.y;- static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");+ if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {+ break;+ }- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;+ float sum[ncols] = {0.0f};+#pragma unroll+ for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {+ const int k_KQ = k_KQ_0 + threadIdx.x;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+ const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ sum[j] += __low2float(K_ik) * Q_h2[j][k_KQ_0/WARP_SIZE].x;+ sum[j] += __high2float(K_ik) * Q_h2[j][k_KQ_0/WARP_SIZE].y;+ }+ }- const block_q2_K * x = (const block_q2_K *)vx + ib0;+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ sum[j] = warp_reduce_sum(sum[j]);+ sum[j] += mask ? slope*__half2float(maskh[j*ne11 + k_VKQ_0 + i_KQ]) : 0.0f;- float tmp = 0; // partial sum for thread in warp+ kqmax_new_arr[j] = fmaxf(kqmax_new_arr[j], sum[j]);-#if QK_K == 256- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+ if (threadIdx.x == 0) {+ KQ[j*D + i_KQ] = sum[j];+ }+ }+ }- const int step = 16/K_QUANTS_PER_ITERATION;+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ float kqmax_new_j = kqmax_new_arr[j];- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7+ kqmax_new_j = warp_reduce_max(kqmax_new_j);+ if (threadIdx.x == 0) {+ kqmax_shared[j][threadIdx.y] = kqmax_new_j;+ }+ }- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int s_offset = 8*im;- const int y_offset = 128*im + l0;+ __syncthreads();- uint32_t aux[4];- const uint8_t * d = (const uint8_t *)aux;- const uint8_t * m = (const uint8_t *)(aux + 2);+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ float kqmax_new_j = kqmax_shared[j][threadIdx.x];+ kqmax_new_j = warp_reduce_max(kqmax_new_j);- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+ const float KQ_max_scale = expf(kqmax[j] - kqmax_new_j);+ kqmax[j] = kqmax_new_j;- const float * y = yy + i * QK_K + y_offset;- const uint8_t * q = x[i].qs + q_offset;+ const float val = expf(KQ[j*D + tid] - kqmax[j]);+ kqsum[j] = kqsum[j]*KQ_max_scale + val;+ KQ[j*D + tid] = val;- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ VKQ[j] *= KQ_max_scale;+ }- const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);- aux[0] = a[0] & 0x0f0f0f0f;- aux[1] = a[1] & 0x0f0f0f0f;- aux[2] = (a[0] >> 4) & 0x0f0f0f0f;- aux[3] = (a[1] >> 4) & 0x0f0f0f0f;+ __syncthreads();- float sum1 = 0, sum2 = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)- + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)- + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)- + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)- + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)- + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)- + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)- +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);- sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]- + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];+#pragma unroll+ for (int k = 0; k < D; ++k) {+ if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k >= ne11) {+ break;+ }+ const float V_ki = __half2float(V_h[(k_VKQ_0 + k)*stride_KV + tid]);+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ VKQ[j] += V_ki*KQ[j*D + k];+ }}- tmp += dall * sum1 - dmin * sum2;+ __syncthreads();}-#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3- const int offset = tid * K_QUANTS_PER_ITERATION;- uint32_t uaux[2];- const uint8_t * d = (const uint8_t *)uaux;+#pragma unroll+ for (int j = 0; j < ncols; ++j) {+ kqsum[j] = warp_reduce_sum(kqsum[j]);+ if (threadIdx.x == 0) {+ kqsum_shared[j][threadIdx.y] = kqsum[j];+ }+ }- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ __syncthreads();- const float * y = yy + i * QK_K + offset;- const uint8_t * q = x[i].qs + offset;- const uint32_t * s = (const uint32_t *)x[i].scales;+#pragma unroll+ for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {+ kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x];+ kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]);- uaux[0] = s[0] & 0x0f0f0f0f;- uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;+ float dst_val = VKQ[j_VKQ];+ if (parallel_blocks == 1) {+ dst_val /= kqsum[j_VKQ];+ }+ const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;+ dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val;+ }- const float2 dall = __half22float2(x[i].dm);+ if (parallel_blocks != 1 && tid < ncols) {+ dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]);+ }+}- float sum1 = 0, sum2 = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- const uint8_t ql = q[l];- sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)- + y[l+16] * d[1] * ((ql >> 2) & 3)- + y[l+32] * d[2] * ((ql >> 4) & 3)- + y[l+48] * d[3] * ((ql >> 6) & 3);- sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];- }- tmp += dall.x * sum1 - dall.y * sum2;+template <int cols_per_block, int parallel_blocks>+void launch_fattn_vec_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];+ switch (Q->ne[0]) {+ case 64: {+ constexpr int D = 64;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ case 128: {+ constexpr int D = 128;+ constexpr int nwarps = D/WARP_SIZE;+ fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32<D, cols_per_block, parallel_blocks>;+ launch_fattn<D, parallel_blocks>(ctx, dst, fattn_kernel, nwarps, cols_per_block);+ } break;+ default: {+ GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128.");+ } break;+ }+}++void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * Q = dst->src[0];++ if (Q->ne[1] == 1) {+ constexpr int cols_per_block = 1;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;}-#endif- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ if (Q->ne[1] == 2) {+ constexpr int cols_per_block = 2;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }- if (threadIdx.x == 0) {- dst[row] = tmp;+ if (Q->ne[1] <= 4) {+ constexpr int cols_per_block = 4;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;+ }++ if (Q->ne[1] <= 8) {+ constexpr int cols_per_block = 8;+ constexpr int parallel_blocks = 4;+ launch_fattn_vec_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);+ return;}++ constexpr int cols_per_block = 8;+ constexpr int parallel_blocks = 1;+ launch_fattn_vec_f32_64_128<cols_per_block, parallel_blocks>(ctx, dst);}-static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP getrows.cu+//+////////////////////////////////////////////////////////////////////////////////- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP getrows.cuh+//+////////////////////////////////////////////////////////////////////////////////- const block_q3_K * x = (const block_q3_K *)vx + ib0;- float tmp = 0; // partial sum for thread in warp+#define CUDA_GET_ROWS_BLOCK_SIZE 256-#if QK_K == 256+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);- const uint16_t kmask1 = 0x0303;- const uint16_t kmask2 = 0x0f0f;+template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void k_get_rows(+ const void * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+ const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;- const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop- const int step = 16/K_QUANTS_PER_ITERATION;- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0....15 or 0...7+ if (i00 >= ne00) {+ return;+ }- const uint8_t m = 1 << (4*im);+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];- const int l0 = n*in; // 0...15 or 0...14 in steps of 2- const int q_offset = 32*im + l0;- const int y_offset = 128*im + l0;+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;- uint16_t utmp[4];- const int8_t * s = (const int8_t *)utmp;+ const int ib = i00/qk; // block index+ const int iqs = (i00%qk)/qr; // quant index+ const int iybs = i00 - i00%qk; // dst block start index+ const int y_offset = qr == 1 ? 1 : qk/2;++ // dequantize+ dfloat2 v;+ dequantize_kernel(src0_row, ib, iqs, v);++ dst_row[iybs + iqs + 0] = v.x;+ dst_row[iybs + iqs + y_offset] = v.y;+}- const uint16_t s_shift = 4*im;+template<typename src0_t, typename dst_t>+static __global__ void k_get_rows_float(+ const src0_t * src0, const int32_t * src1, dst_t * dst,+ int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/+ /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/+ /*size_t s0,*/ size_t s1, size_t s2, size_t s3,+ /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,+ size_t s10, size_t s11, size_t s12/*, size_t s13*/) {- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+ const int i00 = blockIdx.x*blockDim.x + threadIdx.x;+ const int i10 = blockDim.y*blockIdx.y + threadIdx.y;+ const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;+ const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;- const float * y = yy + i * QK_K + y_offset;- const uint8_t * q = x[i].qs + q_offset;- const uint8_t * h = x[i].hmask + l0;+ if (i00 >= ne00) {+ return;+ }- const uint16_t * a = (const uint16_t *)x[i].scales;- utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);- utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);- utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);- utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);+ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];- const float d = x[i].d;+ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);- float sum = 0;- for (int l = 0; l < n; ++l) {- sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))- + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))- + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))- + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));- sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))- + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))- + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))- + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));- }- tmp += d * sum;+ dst_row[i00] = src0_row[i00];+}- }-#else+template<int qk, int qr, dequantize_kernel_t dq>+static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3- const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14- const int in = offset/8; // 0 or 1- const int im = offset%8; // 0...7+ GGML_TENSOR_BINARY_OP_LOCALS- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);- const float * y = yy + i * QK_K + offset;- const uint8_t * q = x[i].qs + offset;- const uint8_t * s = x[i].scales;+ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);- const float dall = (float)x[i].d;+ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);- float sum = 0;- for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {- const uint8_t hl = x[i].hmask[im+l] >> in;- const uint8_t ql = q[l];- sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))- + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))- + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))- + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));- }- tmp += sum;- }-#endif+ GGML_ASSERT(ne00 % 2 == 0);- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);- if (threadIdx.x == 0) {- dst[row] = tmp;- }+ GGML_UNUSED(dst);}-static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {-- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;-- const block_q4_K * x = (const block_q4_K *)vx + ib0;+template<typename src0_t>+static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,+ const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {-#if QK_K == 256- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;+ GGML_TENSOR_BINARY_OP_LOCALS- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;+ const dim3 block_nums(block_num_x, ne10, ne11*ne12);- const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4+ // strides in elements+ //const size_t s0 = nb0 / ggml_element_size(dst);+ const size_t s1 = nb1 / ggml_element_size(dst);+ const size_t s2 = nb2 / ggml_element_size(dst);+ const size_t s3 = nb3 / ggml_element_size(dst);- const int il = tid/step; // 0...3- const int ir = tid - step*il; // 0...7 or 0...3- const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4+ const size_t s10 = nb10 / ggml_element_size(src1);+ const size_t s11 = nb11 / ggml_element_size(src1);+ const size_t s12 = nb12 / ggml_element_size(src1);+ //const size_t s13 = nb13 / ggml_element_size(src1);- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;+ k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(+ src0_dd, src1_dd, dst_dd,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10, ne11,*/ ne12, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;+ GGML_UNUSED(dst);+}- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;+void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ const float * src0_d = (const float *)src0->data;+ const float * src1_d = (const float *)src1->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();-#if K_QUANTS_PER_ITERATION == 2- uint32_t q32[4];- const uint8_t * q4 = (const uint8_t *)q32;-#else- uint16_t q16[4];- const uint8_t * q4 = (const uint8_t *)q16;-#endif- float tmp = 0; // partial sum for thread in warp+ GGML_ASSERT(src1->type == GGML_TYPE_I32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));- const float * y1 = yy + i*QK_K + y_offset;- const float * y2 = y1 + 128;+ const int32_t * src1_i32 = (const int32_t *) src1_d;- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ switch (src0->type) {+ case GGML_TYPE_F16:+ get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_F32:+ get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q4_0:+ get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q4_1:+ get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q5_0:+ get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q5_1:+ get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ case GGML_TYPE_Q8_0:+ get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);+ break;+ default:+ // TODO: k-quants+ fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));+ GGML_ASSERT(false);+ break;+ }+}- const uint16_t * a = (const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP im2col.cu+//+////////////////////////////////////////////////////////////////////////////////-#if K_QUANTS_PER_ITERATION == 2- const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);- const uint32_t * q2 = q1 + 16;- q32[0] = q1[0] & 0x0f0f0f0f;- q32[1] = q1[0] & 0xf0f0f0f0;- q32[2] = q2[0] & 0x0f0f0f0f;- q32[3] = q2[0] & 0xf0f0f0f0;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP im2col.cuh+//+////////////////////////////////////////////////////////////////////////////////- float4 s = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- for (int l = 0; l < 4; ++l) {- s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+ 4];- s.z += y2[l] * q4[l+8]; s.w += y2[l+32] * q4[l+12];- smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];- }- tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;-#else- const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);- const uint16_t * q2 = q1 + 32;- q16[0] = q1[0] & 0x0f0f;- q16[1] = q1[0] & 0xf0f0;- q16[2] = q2[0] & 0x0f0f;- q16[3] = q2[0] & 0xf0f0;+#define CUDA_IM2COL_BLOCK_SIZE 256- float4 s = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- for (int l = 0; l < 2; ++l) {- s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2];- s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6];- smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];- }- tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin;-#endif+void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template <typename T>+static __global__ void im2col_kernel(+ const float * x, T * dst, int64_t batch_offset,+ int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,+ int s0, int s1, int p0, int p1, int d0, int d1) {+ const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;+ if (i >= pelements) {+ return;}-#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);-- const int step = tid * K_QUANTS_PER_ITERATION;-- uint16_t aux16[2];- const uint8_t * s = (const uint8_t *)aux16;- float tmp = 0;+ const int64_t ksize = OW * (KH > 1 ? KW : 1);+ const int64_t kx = i / ksize;+ const int64_t kd = kx * ksize;+ const int64_t ky = (i - kd) / OW;+ const int64_t ix = i % OW;- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- const uint8_t * q = x[i].qs + step;- const float * y = yy + i*QK_K + step;- const uint16_t * a = (const uint16_t *)x[i].scales;- aux16[0] = a[0] & 0x0f0f;- aux16[1] = (a[0] >> 4) & 0x0f0f;- const float d = (float)x[i].dm[0];- const float m = (float)x[i].dm[1];- float sum = 0.f;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])- + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])- + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])- + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);- }- tmp += sum;- }+ const int64_t oh = blockIdx.y;+ const int64_t batch = blockIdx.z / IC;+ const int64_t ic = blockIdx.z % IC;-#endif+ const int64_t iiw = ix * s0 + kx * d0 - p0;+ const int64_t iih = oh * s1 + ky * d1 - p1;- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ const int64_t offset_dst =+ ((batch * OH + oh) * OW + ix) * CHW ++ (ic * (KW * KH) + ky * KW + kx);- if (tid == 0) {- dst[row] = tmp;+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst[offset_dst] = 0.0f;+ } else {+ const int64_t offset_src = ic * offset_delta + batch * batch_offset;+ dst[offset_dst] = x[offset_src + iih * IW + iiw];}}-static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols) {-- const int row = blockIdx.x;- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+template <typename T>+static void im2col_cuda(const float * x, T* dst,+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,+ int64_t batch, int64_t batch_offset, int64_t offset_delta,+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {+ const int parallel_elements = OW * KW * KH;+ const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;+ dim3 block_nums(num_blocks, OH, batch * IC);+ im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);+}- const block_q5_K * x = (const block_q5_K *)vx + ib0;+static void im2col_cuda_f16(const float * x, half * dst,+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,+ int64_t batch, int64_t batch_offset, int64_t offset_delta,+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {- float tmp = 0; // partial sum for thread in warp+ im2col_cuda<half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);+}-#if QK_K == 256- const uint16_t kmask1 = 0x3f3f;- const uint16_t kmask2 = 0x0f0f;- const uint16_t kmask3 = 0xc0c0;+static void im2col_cuda_f32(const float * x, float * dst,+ int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,+ int64_t batch, int64_t batch_offset, int64_t offset_delta,+ int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {- const int tid = threadIdx.x/2; // 0...15- const int ix = threadIdx.x%2;+ im2col_cuda<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);+}- const int il = tid/4; // 0...3- const int ir = tid - 4*il;// 0...3- const int n = 2;+void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];+ const float * src1_d = (const float *)src1->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();- const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224- const int in = il%2;+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);- const int l0 = n*(2*ir + in);- const int q_offset = 32*im + l0;- const int y_offset = 64*im + l0;+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t*)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t*)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t*)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t*)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t*)(dst->op_params))[5];- const uint8_t hm1 = 1 << (2*im);- const uint8_t hm2 = hm1 << 4;+ const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;- uint16_t aux[4];- const uint8_t * sc = (const uint8_t *)aux;+ const int64_t IC = src1->ne[is_2D ? 2 : 1];+ const int64_t IH = is_2D ? src1->ne[1] : 1;+ const int64_t IW = src1->ne[0];- uint16_t q16[8];- const uint8_t * q4 = (const uint8_t *)q16;+ const int64_t KH = is_2D ? src0->ne[1] : 1;+ const int64_t KW = src0->ne[0];- for (int i = ix; i < num_blocks_per_row; i += 2) {+ const int64_t OH = is_2D ? dst->ne[2] : 1;+ const int64_t OW = dst->ne[1];- const uint8_t * ql1 = x[i].qs + q_offset;- const uint8_t * qh = x[i].qh + l0;- const float * y1 = yy + i*QK_K + y_offset;- const float * y2 = y1 + 128;+ const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32+ const int64_t batch = src1->ne[3];+ const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32- const float dall = __low2half(x[i].dm);- const float dmin = __high2half(x[i].dm);+ if(dst->type == GGML_TYPE_F16) {+ im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);+ } else {+ im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);+ }+}- const uint16_t * a = (const uint16_t *)x[i].scales;- aux[0] = a[im+0] & kmask1;- aux[1] = a[im+2] & kmask1;- aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);- aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP mmq.cu+//+////////////////////////////////////////////////////////////////////////////////- float4 sum = {0.f, 0.f, 0.f, 0.f};- float smin = 0;- const uint16_t * q1 = (const uint16_t *)ql1;- const uint16_t * q2 = q1 + 32;- q16[0] = q1[0] & 0x0f0f;- q16[1] = q1[8] & 0x0f0f;- q16[2] = (q1[0] >> 4) & 0x0f0f;- q16[3] = (q1[8] >> 4) & 0x0f0f;- q16[4] = q2[0] & 0x0f0f;- q16[5] = q2[8] & 0x0f0f;- q16[6] = (q2[0] >> 4) & 0x0f0f;- q16[7] = (q2[8] >> 4) & 0x0f0f;- for (int l = 0; l < n; ++l) {- sum.x += y1[l+ 0] * (q4[l +0] + (qh[l+ 0] & (hm1 << 0) ? 16 : 0))- + y1[l+16] * (q4[l +2] + (qh[l+16] & (hm1 << 0) ? 16 : 0));- sum.y += y1[l+32] * (q4[l +4] + (qh[l+ 0] & (hm1 << 1) ? 16 : 0))- + y1[l+48] * (q4[l +6] + (qh[l+16] & (hm1 << 1) ? 16 : 0));- sum.z += y2[l+ 0] * (q4[l +8] + (qh[l+ 0] & (hm2 << 0) ? 16 : 0))- + y2[l+16] * (q4[l+10] + (qh[l+16] & (hm2 << 0) ? 16 : 0));- sum.w += y2[l+32] * (q4[l+12] + (qh[l+ 0] & (hm2 << 1) ? 16 : 0))- + y2[l+48] * (q4[l+14] + (qh[l+16] & (hm2 << 1) ? 16 : 0));- smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]- + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];- }- tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin;- }-#else- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION);- const int step = tid * K_QUANTS_PER_ITERATION;- const int im = step/8;- const int in = step%8;+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP mmq.cuh+//+////////////////////////////////////////////////////////////////////////////////- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {- const uint8_t * q = x[i].qs + step;- const int8_t * s = x[i].scales;- const float * y = yy + i*QK_K + step;- const float d = x[i].d;- float sum = 0.f;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- const uint8_t h = x[i].qh[in+j] >> im;- sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))- + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))- + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))- + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));- }- tmp += sum;- }-#endif- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+void ggml_cuda_op_mul_mat_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);- if (threadIdx.x == 0) {- dst[row] = tmp;- }-}+bool ggml_cuda_supports_mmq(enum ggml_type type);-static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP vecdotq.cuh+//+////////////////////////////////////////////////////////////////////////////////- static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");- const int row = blockIdx.x*blockDim.y + threadIdx.y;- if (row > nrows) return;+static __device__ __forceinline__ int get_int_from_int8(const int8_t * x8, const int & i32) {+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment- const int num_blocks_per_row = ncols / QK_K;- const int ib0 = row*num_blocks_per_row;+ int x32 = 0;+ x32 |= x16[0] << 0;+ x32 |= x16[1] << 16;- const block_q6_K * x = (const block_q6_K *)vx + ib0;+ return x32;+}-#if QK_K == 256+static __device__ __forceinline__ int get_int_from_uint8(const uint8_t * x8, const int & i32) {+ const uint16_t * x16 = (const uint16_t *) (x8 + sizeof(int) * i32); // assume at least 2 byte alignment- const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16- const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1+ int x32 = 0;+ x32 |= x16[0] << 0;+ x32 |= x16[1] << 16;- const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8+ return x32;+}- const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128...- const int in = tid - step*im; // 0...15 or 0...7+static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t * x8, const int & i32) {+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+}-#if K_QUANTS_PER_ITERATION == 1- const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15- const int is = 0;-#else- const int l0 = 4 * in; // 0, 4, 8, ..., 28- const int is = in / 4;-#endif- const int ql_offset = 64*im + l0;- const int qh_offset = 32*im + l0;- const int s_offset = 8*im + is;- const int y_offset = 128*im + l0;+static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t * x8, const int & i32) {+ return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment+}- float tmp = 0; // partial sum for thread in warp- for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q- const float * y = yy + i * QK_K + y_offset;- const uint8_t * ql = x[i].ql + ql_offset;- const uint8_t * qh = x[i].qh + qh_offset;- const int8_t * s = x[i].scales + s_offset;+#define VDR_Q4_0_Q8_1_MMVQ 2+#define VDR_Q4_0_Q8_1_MMQ 4- const float d = x[i].d;+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(+ const int * v, const int * u, const float & d4, const half2 & ds8) {-#if K_QUANTS_PER_ITERATION == 1- float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)- + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)- + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)- + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)- + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)- + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)- + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)- +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);- tmp += sum;-#else- float sum = 0;- for (int l = 0; l < 4; ++l) {- sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)- + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)- + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)- + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);- }- tmp += sum;-#endif+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;+ // SIMD dot product of quantized values+ sumi = __dp4a(vi0, u[2*i+0], sumi);+ sumi = __dp4a(vi1, u[2*i+1], sumi);}+ const float2 ds8f = __half22float2(ds8);++ // second part effectively subtracts 8 from each quant value+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7- const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3+#define VDR_Q4_1_Q8_1_MMVQ 2+#define VDR_Q4_1_Q8_1_MMQ 4- const int step = tid * K_QUANTS_PER_ITERATION;+template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {- float tmp = 0; // partial sum for thread in warp+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {+#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;- const float * y = yy + i * QK_K + step;- const uint8_t * ql = x[i].ql + step;- const uint8_t * qh = x[i].qh + step;- const int8_t * s = x[i].scales;+ // SIMD dot product of quantized values+ sumi = __dp4a(vi0, u[2*i+0], sumi);+ sumi = __dp4a(vi1, u[2*i+1], sumi);+ }- const float d = x[i+0].d;+#ifdef GGML_CUDA_F16+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));+ const float d4d8 = tmp.x;+ const float m4s8 = tmp.y;+#else+ const float2 dm4f = __half22float2(dm4);+ const float2 ds8f = __half22float2(ds8);+ const float d4d8 = dm4f.x * ds8f.x;+ const float m4s8 = dm4f.y * ds8f.y;+#endif // GGML_CUDA_F16- float sum = 0;- for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {- sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)- + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)- + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)- + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);- }- tmp += sum;+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- }+#define VDR_Q5_0_Q8_1_MMVQ 2+#define VDR_Q5_0_Q8_1_MMQ 4-#endif+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- if (tid == 0) {- dst[row] = tmp;+#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values++ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values}-}-static __device__ void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){- const half * x = (const half *) vx;+ const float2 ds8f = __half22float2(ds8);- // automatic half -> float type cast if dfloat == float- v.x = x[ib + iqs + 0];- v.y = x[ib + iqs + 1];+ // second part effectively subtracts 16 from each quant value+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-template <int qk, int qr, dequantize_kernel_t dequantize_kernel>-static __global__ void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows) {- // qk = quantized weights per x block- // qr = number of quantized weights per data value in x block- const int64_t row = (int64_t)blockIdx.x*blockDim.y + threadIdx.y;+#define VDR_Q5_1_Q8_1_MMVQ 2+#define VDR_Q5_1_Q8_1_MMQ 4- if (row >= nrows) {- return;- }+template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {- const int tid = threadIdx.x;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- const int iter_stride = 2*GGML_CUDA_DMMV_X;- const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter- const int y_offset = qr == 1 ? 1 : qk/2;+#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = __dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values++ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = __dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+ }-// partial sum for each thread#ifdef GGML_CUDA_F16- half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));+ const float d5d8 = tmp.x;+ const float m5s8 = tmp.y;#else- float tmp = 0.0f;+ const float2 dm5f = __half22float2(dm5);+ const float2 ds8f = __half22float2(ds8);+ const float d5d8 = dm5f.x * ds8f.x;+ const float m5s8 = dm5f.y * ds8f.y;#endif // GGML_CUDA_F16- for (int i = 0; i < ncols; i += iter_stride) {- const int col = i + vals_per_iter*tid;- const int64_t ib = ((int64_t)row*ncols + col)/qk; // x block index- const int iqs = (col%qk)/qr; // x quant index- const int iybs = col - col%qk; // y block start index+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);-// processing >2 values per i iter is faster for fast GPUs-#pragma unroll- for (int j = 0; j < vals_per_iter; j += 2) {- // process 2 vals per j iter+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- // dequantize- // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val- dfloat2 v;- dequantize_kernel(vx, ib, iqs + j/qr, v);+#define VDR_Q8_0_Q8_1_MMVQ 2+#define VDR_Q8_0_Q8_1_MMQ 8- // matrix multiplication- // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2-#ifdef GGML_CUDA_F16- tmp += __hmul2(v, {- y[iybs + iqs + j/qr + 0],- y[iybs + iqs + j/qr + y_offset]- });-#else- tmp += v.x * y[iybs + iqs + j/qr + 0];- tmp += v.y * y[iybs + iqs + j/qr + y_offset];-#endif // GGML_CUDA_F16- }+template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impl(+ const int * v, const int * u, const float & d8_0, const float & d8_1) {++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = __dp4a(v[i], u[i], sumi);}- // sum up partial sums and write back result- tmp = warp_reduce_sum(tmp);+ return d8_0*d8_1 * sumi;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}++template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {++#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = __dp4a(v[i], u[i], sumi);+ }- if (tid == 0) {#ifdef GGML_CUDA_F16- dst[row] = tmp.x + tmp.y;+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));+ const float d8d8 = tmp.x;+ const float m8s8 = tmp.y;#else- dst[row] = tmp;+ const float2 dm8f = __half22float2(dm8);+ const float2 ds8f = __half22float2(ds8);+ const float d8d8 = dm8f.x * ds8f.x;+ const float m8s8 = dm8f.y * ds8f.y;#endif // GGML_CUDA_F16- }-}-static void dequantize_mul_mat_vec_q4_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static void dequantize_mul_mat_vec_q4_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+#define VDR_Q2_K_Q8_1_MMVQ 1+#define VDR_Q2_K_Q8_1_MMQ 2-static void dequantize_mul_mat_vec_q5_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const half2 & dm2, const float * __restrict__ d8) {-static void dequantize_mul_mat_vec_q5_1_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;-static void dequantize_mul_mat_vec_q8_0_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+#pragma unroll+ for (int i = 0; i < QR2_K; ++i) {+ const int sc = scales[2*i];-static void dequantize_mul_mat_vec_q2_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q2_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ const int vi = (v >> (2*i)) & 0x03030303;-static void dequantize_mul_mat_vec_q3_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q3_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product-static void dequantize_mul_mat_vec_q4_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q4_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+ // fill int with 4x m+ int m = sc >> 4;+ m |= m << 8;+ m |= m << 16;+ sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values+ }-static void dequantize_mul_mat_vec_q5_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const dim3 block_dims(32, 1, 1);- dequantize_mul_mat_vec_q5_k<<<nrows, block_dims, 0, stream>>>(vx, y, dst, ncols);-}+ const float2 dm2f = __half22float2(dm2);-static void dequantize_mul_mat_vec_q6_K_cuda(const void * vx, const float * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % QK_K == 0);- const int ny = 2 / K_QUANTS_PER_ITERATION;- const int block_num_y = (nrows + ny - 1) / ny;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(32, ny, 1);- dequantize_mul_mat_vec_q6_k<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);+ return dm2f.x*sumf_d - dm2f.y*sumf_m;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static void convert_mul_mat_vec_f16_cuda(const void * vx, const dfloat * y, float * dst, const int ncols, const int nrows, cudaStream_t stream) {- GGML_ASSERT(ncols % GGML_CUDA_DMMV_X == 0);- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- dequantize_mul_mat_vec<1, 1, convert_f16>- <<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols, nrows);-}+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const half2 & dm2, const float & d8) {-void ggml_cuda_op_dequantize_mul_mat_vec(- ggml_backend_cuda_context & ctx,- const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,- const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, cudaStream_t stream) {- GGML_UNUSED(ctx);- const int64_t ne00 = src0->ne[0];- const int64_t row_diff = row_high - row_low;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi_d = 0;+ int sumi_m = 0;- GGML_ASSERT(src1->type == GGML_TYPE_F32);+#pragma unroll+ for (int i0 = 0; i0 < QI8_1; i0 += QI8_1/2) {+ int sumi_d_sc = 0;- // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics-#ifdef GGML_CUDA_F16- ggml_cuda_pool_alloc<half> src1_dfloat_a(ctx.pool());- half * src1_dfloat = nullptr; // dfloat == half+ const int sc = scales[i0 / (QI8_1/2)];- bool src1_convert_f16 =- src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||- src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||- src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;+ // fill int with 4x m+ int m = sc >> 4;+ m |= m << 8;+ m |= m << 16;- if (src1_convert_f16) {- src1_dfloat = src1_dfloat_a.alloc(ne00);- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- GGML_ASSERT(to_fp16_cuda != nullptr);- to_fp16_cuda(src1_ddf_i, src1_dfloat, ne00, stream);- }-#else- const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion-#endif // GGML_CUDA_F16+#pragma unroll+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product+ sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m+ }- switch (src0->type) {- case GGML_TYPE_Q4_0:- dequantize_mul_mat_vec_q4_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q4_1:- dequantize_mul_mat_vec_q4_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_0:- dequantize_mul_mat_vec_q5_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_1:- dequantize_mul_mat_vec_q5_1_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q8_0:- dequantize_mul_mat_vec_q8_0_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q2_K:- dequantize_mul_mat_vec_q2_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q3_K:- dequantize_mul_mat_vec_q3_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q4_K:- dequantize_mul_mat_vec_q4_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q5_K:- dequantize_mul_mat_vec_q5_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_Q6_K:- dequantize_mul_mat_vec_q6_K_cuda(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);- break;- case GGML_TYPE_F16:- convert_mul_mat_vec_f16_cuda(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);- break;- default:- GGML_ASSERT(false);- break;+ sumi_d += sumi_d_sc * (sc & 0xF);}- GGML_UNUSED(src1);- GGML_UNUSED(dst);- GGML_UNUSED(src1_ddq_i);- GGML_UNUSED(src1_ncols);- GGML_UNUSED(src1_padded_row_size);+ const float2 dm2f = __half22float2(dm2);++ return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define FATTN_KQ_STRIDE 256-#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction.-#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs.+#define VDR_Q3_K_Q8_1_MMVQ 1+#define VDR_Q3_K_Q8_1_MMQ 2-template<int D, int parallel_blocks> // D == head size-__launch_bounds__(((D + WARP_SIZE - 1) / WARP_SIZE)*WARP_SIZE, 1)-static __global__ void flash_attn_vec_ext_f16(- const char * __restrict__ Q,- const char * __restrict__ K,- const char * __restrict__ V,- const char * __restrict__ mask,- float * __restrict__ dst,- float2 * __restrict__ dst_meta,- const float scale,- const int ne00,- const int ne01,- const int ne02,- const int ne03,- const int ne10,- const int ne11,- const int ne12,- const int ne13,- const int ne31,- const int nb31,- const int nb01,- const int nb02,- const int nb03,- const int nb11,- const int nb12,- const int nb13,- const int ne0,- const int ne1,- const int ne2,- const int ne3) {-#if FP16_AVAILABLE- //In this kernel Q, K, V are matrices while i, j, k are matrix indices.+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {- const int ic = blockIdx.x / parallel_blocks; // Index of the Q/QKV column to work on.- const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf = 0.0f;- const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.- const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic);- const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio));- const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape- const half * maskh = (const half *) mask + ne11*ic;+#pragma unroll+ for (int i = 0; i < QR3_K; ++i) {+ const int isc = scale_offset + 2*i;- const int stride_KV = nb11 / sizeof(half);- const int stride_KV2 = nb11 / sizeof(half2);+ const int isc_low = isc % (QK_K/32);+ const int sc_shift_low = 4 * (isc / (QK_K/32));+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;- constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;- const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;- __builtin_assume(tid < nwarps*WARP_SIZE);+ const int isc_high = isc % (QK_K/64);+ const int sc_shift_high = 2 * (isc / (QK_K/64));+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;- __shared__ half KQ[nwarps*WARP_SIZE];- KQ[tid] = -INFINITY;- half2 * KQ2 = (half2 *) KQ;+ const int sc = (sc_low | sc_high) - 32;- half kqmax = -HALF_MAX_HALF;- half kqsum = 0.0f;+ const int vil = (vl >> (2*i)) & 0x03030303;- __shared__ half kqmax_shared[WARP_SIZE];- __shared__ half kqsum_shared[WARP_SIZE];- if (threadIdx.y == 0) {- kqmax_shared[threadIdx.x] = -HALF_MAX_HALF;- kqsum_shared[threadIdx.x] = 0.0f;- }- __syncthreads();+ const int vih = ((vh >> i) << 2) & 0x04040404;- // Convert Q to half2 and store in registers:- half2 Q_h2[(D/2 + WARP_SIZE - 1) / WARP_SIZE];-#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {- const int i = i0 + threadIdx.x;- if (i0 + WARP_SIZE > D/2 && i >= D/2) {- break;- }+ const int vi = __vsubss4(vil, vih);- Q_h2[i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(Q_f2[i].x, Q_f2[i].y);+ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product}- half2 VKQ = make_half2(0.0f, 0.0f); // Each thread calculates a single VKQ value.+ return d3 * sumf;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const int k_start = parallel_blocks == 1 ? 0 : ip*D;- for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) {- // Calculate KQ tile and keep track of new maximum KQ values:- half kqmax_new = kqmax;-#pragma unroll- for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) {- const int i_KQ = i_KQ_0 + threadIdx.y;+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d3, const float & d8) {- if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) {- break;- }+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ int sumi = 0;- half2 sum2 = make_half2(0.0f, 0.0f);#pragma unroll- for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) {- const int k_KQ = k_KQ_0 + threadIdx.x;- if (k_KQ_0 + WARP_SIZE > D/2 && k_KQ >= D/2) {- break;- }-- const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ];- sum2 += K_ik * Q_h2[k_KQ_0/WARP_SIZE];- }+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {+ int sumi_sc = 0;- sum2 = warp_reduce_sum(sum2);- half sum = __low2half(sum2) + __high2half(sum2);- sum += mask ? maskh[k_VKQ_0 + i_KQ] : __float2half(0.0f);- kqmax_new = ggml_cuda_hmax(kqmax_new, sum);- if (threadIdx.x == 0) {- KQ[i_KQ] = sum;- }+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product}- kqmax_new = warp_reduce_max(kqmax_new);- if (threadIdx.x == 0) {- kqmax_shared[threadIdx.y] = kqmax_new;- }- __syncthreads();- kqmax_new = kqmax_shared[threadIdx.x];- kqmax_new = warp_reduce_max(kqmax_new);+ sumi += sumi_sc * scales[i0 / (QI8_1/2)];+ }- const half KQ_max_scale = hexp(kqmax - kqmax_new);- kqmax = kqmax_new;+ return d3*d8 * sumi;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const half val = hexp(KQ[tid] - kqmax);- kqsum = kqsum*KQ_max_scale + val;- KQ[tid] = val;+#define VDR_Q4_K_Q8_1_MMVQ 2+#define VDR_Q4_K_Q8_1_MMQ 8- VKQ *= __half2half2(KQ_max_scale);+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {- __syncthreads();+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- if (tid < D) {#pragma unroll- for (int k0 = 0; k0 < D; k0 += 2) {- if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) {- break;- }-- half2 V_k;- reinterpret_cast<half&>(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid];- reinterpret_cast<half&>(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid];- VKQ += V_k*KQ2[k0/2];- }- }-- __syncthreads();- }+ for (int i = 0; i < QR4_K; ++i) {+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;- if (tid >= D) {- kqsum = 0.0f;- }+ const int dot1 = __dp4a(v1i, u[2*i+1], __dp4a(v0i, u[2*i+0], 0)); // SIMD dot product+ const int dot2 = __dp4a(0x01010101, u[2*i+1], __dp4a(0x01010101, u[2*i+0], 0)); // sum of u- kqsum = warp_reduce_sum(kqsum);- if (threadIdx.x == 0) {- kqsum_shared[threadIdx.y] = kqsum;+ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values}- __syncthreads();- kqsum = kqsum_shared[threadIdx.x];- kqsum = warp_reduce_sum(kqsum);- if (tid >= D) {- return;- }+ const float2 dm4f = __half22float2(dm4);- half dst_val = (__low2half(VKQ) + __high2half(VKQ));- if (parallel_blocks == 1) {- dst_val /= kqsum;- }- dst[D*gridDim.y*blockIdx.x + D*blockIdx.y + tid] = dst_val;+ return dm4f.x*sumf_d - dm4f.y*sumf_m;- if (parallel_blocks == 1 || tid != 0) {- return;- }- dst_meta[ic*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax, kqsum);#else- NO_DEVICE_CODE;-#endif // FP16_AVAILABLE+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-// D == head size, VKQ_stride == num VKQ rows calculated in parallel:-template<int D, int ncols, int nwarps, int VKQ_stride, int parallel_blocks, typename KQ_acc_t>-__launch_bounds__(nwarps*WARP_SIZE, 1)-static __global__ void flash_attn_ext_f16(- const char * __restrict__ Q,- const char * __restrict__ K,- const char * __restrict__ V,- const char * __restrict__ mask,- float * __restrict__ dst,- float2 * __restrict__ dst_meta,- const float scale,- const int ne00,- const int ne01,- const int ne02,- const int ne03,- const int ne10,- const int ne11,- const int ne12,- const int ne13,- const int ne31,- const int nb31,- const int nb01,- const int nb02,- const int nb03,- const int nb11,- const int nb12,- const int nb13,- const int ne0,- const int ne1,- const int ne2,- const int ne3) {-#if FP16_MMA_AVAILABLE- //In this kernel Q, K, V are matrices while i, j, k are matrix indices.+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {- const int ic0 = ncols*(blockIdx.x / parallel_blocks); // Index of the first Q/QKV column to work on.- const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel.+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");- static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");- constexpr int frag_m = ncols == 8 ? 32 : 16;- constexpr int frag_n = ncols == 8 ? 8 : 16;- static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");- typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::row_major> frag_a_K;- typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_a_V;- typedef nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, frag_m, frag_n, 16, half, nvcuda::wmma::col_major> frag_b;- typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;- typedef nvcuda::wmma::fragment<nvcuda::wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;+#pragma unroll+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;- constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.- constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.- static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");+#pragma unroll+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = __dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }- // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:- constexpr int D_padded = D + 8;- constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;- constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);+ const float2 ds8f = __half22float2(ds8[i]);- const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.- const float * Q_f = (const float *) (Q + nb02* blockIdx.y + nb01*ic0);- const half * K_h = (const half *) (K + nb12*(blockIdx.y / gqa_ratio));- const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape- const half * maskh = (const half *) mask + (nb31/sizeof(half))* ic0;- const half2 * mask2 = (const half2 *) mask + (nb31/sizeof(half))*(ic0/2);+ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ }- const int stride_Q = nb01 / sizeof(float);- const int stride_KV = nb11 / sizeof(half);+ const float2 dm4f = __half22float2(dm4);- frag_b Q_b[D/16][ncols/frag_n];+ return dm4f.x*sumf_d - dm4f.y*sumf_m;- // A single buffer for temporarily holding tiles of KQ and VKQ parts:- constexpr int mem_KQ = ncols*kqs_padded*kqar;- constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;- __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];- float * KQ_f = (float *) KQ;- half2 * KQ2 = (half2 *) KQ;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- float KQ_rowsum_f[ncols/nwarps] = {0.0f};- float KQ_max_f[ncols/nwarps];- float KQ_max_scale_f[ncols/nwarps] = {0.0f};+#define VDR_Q5_K_Q8_1_MMVQ 2+#define VDR_Q5_K_Q8_1_MMQ 8-#pragma unroll- for (int j = 0; j < ncols/nwarps; ++j) {- KQ_max_f[j] = -FLT_MAX/2.0f;- }+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {- half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};- half2 KQ_max_h2[ncols/nwarps];- half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;#pragma unroll- for (int j = 0; j < ncols/nwarps; ++j) {- KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);- }+ for (int i = 0; i < QR5_K; ++i) {+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;- __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.- half2 * VKQ2 = (half2 *) VKQ;-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {- const int i = i0 + threadIdx.x;- if (i0 + WARP_SIZE > D/2 && i >= D/2) {- break;- }- VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);- }- }+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;++ const int v0i = vl0i | vh0i;+ const int v1i = vl1i | vh1i;++ const int dot1 = __dp4a(v0i, u[2*i+0], __dp4a(v1i, u[2*i+1], 0)); // SIMD dot product+ const int dot2 = __dp4a(0x01010101, u[2*i+0], __dp4a(0x01010101, u[2*i+1], 0)); // sum of u++ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]);- // Convert Q to half and apply scale, temporarily store in KQ:-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-#pragma unroll- for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {- const int i = i0 + threadIdx.x;- if (i0 + WARP_SIZE > D && i >= D) {- break;- }- KQ[j*D_padded + i] = ic0 + j < ne01 ? Q_f[j*stride_Q + i] * scale : 0.0f;- }}- __syncthreads();+ const float2 dm5f = __half22float2(dm5);- // Load Q into tensor core fragments/registers since it will be used frequently:-#pragma unroll- for (int i0 = 0; i0 < D; i0 += 16) {-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- nvcuda::wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ + j0*D_padded + i0, D_padded);- }- }+ return dm5f.x*sumf_d - dm5f.y*sumf_m;- __syncthreads();+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- // Iterate over ne11 == previous tokens:- for (int k_VKQ_0 = ip*FATTN_KQ_STRIDE; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE) {- // Calculate tile of KQ:-#pragma unroll- for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {- frag_c_KQ KQ_c[ncols/frag_n];-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- nvcuda::wmma::fill_fragment(KQ_c[j], 0.0f);- }-#pragma unroll- for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {- frag_a_K K_a;- nvcuda::wmma::load_matrix_sync(K_a, K_h + (k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- nvcuda::wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);- }- }-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- nvcuda::wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, nvcuda::wmma::mem_col_major);- }- }+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {- __syncthreads();+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- // Calculate softmax for each KQ column using the current max. value.- // The divisor is stored in KQ_rowsum and will be applied at the end.#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;- if (std::is_same<KQ_acc_t, float>::value) {- float KQ_f_tmp[FATTN_KQ_STRIDE / WARP_SIZE];#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = __dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }- KQ_f_tmp[k0/WARP_SIZE] = KQ_f[j*kqs_padded + k];- }+ const float2 ds8f = __half22float2(ds8[i]);- float KQ_max_new = KQ_max_f[j0/nwarps];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ }- KQ_f_tmp[k0/WARP_SIZE] += mask ? __half2float(maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;- KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/WARP_SIZE]);- }- KQ_max_new = warp_reduce_max(KQ_max_new);+ const float2 dm4f = __half22float2(dm4);- const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;- KQ_max_scale_f[j0/nwarps] = expf(diff);- if (diff <= SOFTMAX_FTZ_THRESHOLD) {- KQ_max_scale_f[j0/nwarps] = 0.0f;- }- KQ_max_f[j0/nwarps] = KQ_max_new;+ return dm4f.x*sumf_d - dm4f.y*sumf_m;- float KQ_rowsum_add = 0.0f;-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- const float diff = KQ_f_tmp[k0/WARP_SIZE] - KQ_max_f[j0/nwarps];- KQ_f_tmp[k0/WARP_SIZE] = expf(diff);- if (diff <= SOFTMAX_FTZ_THRESHOLD) {- KQ_f_tmp[k0/WARP_SIZE] = 0.0f;- }- KQ_rowsum_add += KQ_f_tmp[k0/WARP_SIZE];- KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/WARP_SIZE];- }- KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);+#define VDR_Q6_K_Q8_1_MMVQ 1+#define VDR_Q6_K_Q8_1_MMQ 8- // Scale previous KQ_rowsum to account for a potential increase in KQ_max:- KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;- } else {- half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*WARP_SIZE)];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d, const float * __restrict__ d8) {- KQ2_tmp[k0/WARP_SIZE] = KQ2[j*(kqs_padded/2) + k];- }+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf = 0.0f;- half2 KQ_max_new = KQ_max_h2[j0/nwarps];#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+ for (int i = 0; i < QR6_K; ++i) {+ const int sc = scales[4*i];- KQ2_tmp[k0/WARP_SIZE] += mask ? mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);- KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/WARP_SIZE]);- }- KQ_max_new = __half2half2(warp_reduce_max(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));- const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;- KQ_max_scale_h2[j0/nwarps] = h2exp(diff);- const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));- *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;- KQ_max_h2[j0/nwarps] = KQ_max_new;+ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;- half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += WARP_SIZE) {- const int k = k0 + threadIdx.x;+ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;- const half2 diff = KQ2_tmp[k0/WARP_SIZE] - KQ_max_h2[j0/nwarps];- KQ2_tmp[k0/WARP_SIZE] = h2exp(diff);- const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));- *((uint32_t *) &KQ2_tmp[k0/WARP_SIZE]) &= ftz_mask;- KQ_rowsum_add += KQ2_tmp[k0/WARP_SIZE];- KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/WARP_SIZE];- }- KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add);+ const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32- // Scale previous KQ_rowsum to account for a potential increase in KQ_max:- KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;- }- }+ sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product+ }- __syncthreads();+ return d*sumf;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}- frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {- const int k = k0 + (threadIdx.y % VKQ_ratio)*16;- nvcuda::wmma::load_matrix_sync(- KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],- KQ + j0*(kqar*kqs_padded) + k,- kqar*kqs_padded);- }- }+// contiguous u/y values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,+ const float & d6, const float * __restrict__ d8) {- frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];-#pragma unroll- for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- nvcuda::wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], 0.0f);- }+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ float sumf_d = 0.0f;#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {- const int k = k0 + (threadIdx.y % VKQ_ratio)*16;+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale- frag_a_V v_a;- nvcuda::wmma::load_matrix_sync(v_a, V_h + (k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- nvcuda::wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);- }- }+ for (int i = i0; i < i0 + 2; ++i) {+ sumi_d.x = __dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product+ sumi_d.x = __dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product++ sumi_d.y = __dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product+ sumi_d.y = __dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product}- __syncthreads();+ sumf_d += d8[i0/4] * (sc[i0/2+0]*sumi_d.x + sc[i0/2+1]*sumi_d.y);+ }- const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);-#pragma unroll- for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- nvcuda::wmma::store_matrix_sync(- KQ + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),- VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],- D_padded, nvcuda::wmma::mem_col_major);- }- }+ return d6 * sumf_d;- __syncthreads();+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;+static __device__ __forceinline__ float vec_dot_q4_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- half2 VKQ_scale;- if (std::is_same<KQ_acc_t, float>::value) {- VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);- } else {- VKQ_scale = KQ_max_scale_h2[j0/nwarps];- }+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;++ int v[VDR_Q4_0_Q8_1_MMVQ];+ int u[2*VDR_Q4_0_Q8_1_MMVQ];#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) {- const int i = i0 + threadIdx.x;- if (i0 + WARP_SIZE > D/2 && i >= D/2) {- break;- }+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);+ }++ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);+}+++static __device__ __forceinline__ float vec_dot_q4_1_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {++ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;++ int v[VDR_Q4_1_Q8_1_MMVQ];+ int u[2*VDR_Q4_1_Q8_1_MMVQ];- half2 VKQ_add = make_half2(0.0f, 0.0f);#pragma unroll- for (int l = 0; l < VKQ_ratio; ++l) {- VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];- }- VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;- }- }-- __syncthreads();+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);}-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j_VKQ = j0 + threadIdx.y;- if (ic0 + j_VKQ >= ne01) {- return;- }- const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip;+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);+}- float KQ_rowsum_j;- if (std::is_same<KQ_acc_t, float>::value) {- KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];- } else {- KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);- }+static __device__ __forceinline__ float vec_dot_q5_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-#pragma unroll- for (int i0 = 0; i0 < D; i0 += WARP_SIZE) {- const int i = i0 + threadIdx.x;- if (i0 + WARP_SIZE > D && i >= D) {- break;- }- float dst_val = VKQ[j_VKQ*D_padded + i];- if (parallel_blocks == 1) {- dst_val /= KQ_rowsum_j;- }- dst[j_dst*gridDim.y*D + blockIdx.y*D + i] = dst_val;- }+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;- if (parallel_blocks == 1 || threadIdx.x != 0) {- continue;- }+ int vl[VDR_Q5_0_Q8_1_MMVQ];+ int vh[VDR_Q5_0_Q8_1_MMVQ];+ int u[2*VDR_Q5_0_Q8_1_MMVQ];- float2 dst_meta_val;- if (std::is_same<KQ_acc_t, float>::value) {- dst_meta_val.x = KQ_max_f[j0/nwarps];- } else {- dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);- }- dst_meta_val.y = KQ_rowsum_j;- dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = dst_meta_val;+#pragma unroll+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);+ vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);}-#else- NO_DEVICE_CODE;-#endif // FP16_MMA_AVAILABLE-}-template<int D, int parallel_blocks> // D == head size-__launch_bounds__(D, 1)-static __global__ void flash_attn_combine_results(- const float * __restrict__ VKQ_parts,- const float2 * __restrict__ VKQ_meta,- float * __restrict__ dst) {-#if FP16_AVAILABLE- VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x;- VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x;- dst += D * gridDim.y*blockIdx.x;+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);+}- const int tid = threadIdx.x;- __builtin_assume(tid < D);+static __device__ __forceinline__ float vec_dot_q5_1_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- __shared__ float2 meta[parallel_blocks];- if (tid < 2*parallel_blocks) {- ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid];- }+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;- __syncthreads();+ int vl[VDR_Q5_1_Q8_1_MMVQ];+ int vh[VDR_Q5_1_Q8_1_MMVQ];+ int u[2*VDR_Q5_1_Q8_1_MMVQ];- float kqmax = meta[0].x;#pragma unroll- for (int l = 1; l < parallel_blocks; ++l) {- kqmax = max(kqmax, meta[l].x);+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);+ vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);}- float VKQ_numerator = 0.0f;- float VKQ_denominator = 0.0f;-#pragma unroll- for (int l = 0; l < parallel_blocks; ++l) {- const float diff = meta[l].x - kqmax;- const float KQ_max_scale = expf(diff);- const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD);- *((uint32_t *) &KQ_max_scale) &= ftz_mask;+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);+}- VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid];- VKQ_denominator += KQ_max_scale * meta[l].y;+static __device__ __forceinline__ float vec_dot_q8_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {++ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;++ int v[VDR_Q8_0_Q8_1_MMVQ];+ int u[VDR_Q8_0_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_from_int8(bq8_0->qs, iqs + i);+ u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);}- dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator;-#else- NO_DEVICE_CODE;-#endif // FP16_AVAILABLE+ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));}-constexpr int get_max_power_of_2(int x) {- return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;-}+static __device__ __forceinline__ float vec_dot_q2_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {-static_assert(get_max_power_of_2(1) == 1, "Test failed.");-static_assert(get_max_power_of_2(2) == 2, "Test failed.");-static_assert(get_max_power_of_2(4) == 4, "Test failed.");-static_assert(get_max_power_of_2(6) == 2, "Test failed.");+ const block_q2_K * bq2_K = (const block_q2_K *) vbq;-// Number of VKQ rows calculated in parallel:-constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {- return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;-}+ const int bq8_offset = QR2_K * (iqs / QI8_1);+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);-static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");-static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");-static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");-static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");-static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");-static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");+ const uint8_t * scales = bq2_K->scales + scale_offset;-template <int D, int parallel_blocks> void launch_fattn_vec_f16(- const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,- ggml_cuda_pool & pool, cudaStream_t main_stream-) {- ggml_cuda_pool_alloc<float> dst_tmp(pool);- ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);+ const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);+ int u[QR2_K];+ float d8[QR2_K];- if (parallel_blocks > 1) {- dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));- dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));+#pragma unroll+ for (int i = 0; i < QR2_K; ++ i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);}- constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE;- const dim3 block_dim(WARP_SIZE, nwarps, 1);- const dim3 blocks_num(parallel_blocks*Q->ne[1], Q->ne[2], Q->ne[3]);- const int shmem = 0;+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);+}- float scale;- memcpy(&scale, KQV->op_params, sizeof(float));-- flash_attn_vec_ext_f16<D, parallel_blocks>- <<<blocks_num, block_dim, shmem, main_stream>>> (- (const char *) Q->data,- (const char *) K->data,- (const char *) V->data,- mask ? ((const char *) mask->data) : nullptr,- parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,- scale,- Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],- K->ne[0], K->ne[1], K->ne[2], K->ne[3],- mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,- Q->nb[1], Q->nb[2], Q->nb[3],- K->nb[1], K->nb[2], K->nb[3],- KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]- );- CUDA_CHECK(cudaGetLastError());+static __device__ __forceinline__ float vec_dot_q3_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- if (parallel_blocks == 1) {- return;- }+ const block_q3_K * bq3_K = (const block_q3_K *) vbq;- const dim3 block_dim_combine(D, 1, 1);- const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);- const int shmem_combine = 0;+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);- flash_attn_combine_results<D, parallel_blocks>- <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>- (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);- CUDA_CHECK(cudaGetLastError());-}+ const float d = bq3_K->d;-template <int D, int cols_per_block, int nwarps, int parallel_blocks, typename KQ_acc_t> void launch_fattn_f16_impl(- const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,- ggml_cuda_pool & pool, cudaStream_t main_stream-) {- ggml_cuda_pool_alloc<float> dst_tmp(pool);- ggml_cuda_pool_alloc<float2> dst_tmp_meta(pool);+ const int vl = get_int_from_uint8(bq3_K->qs, iqs);- if (parallel_blocks > 1) {- dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV));- dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV));+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted+ const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;++ int u[QR3_K];+ float d8[QR3_K];++#pragma unroll+ for (int i = 0; i < QR3_K; ++i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);}- constexpr int frag_m = (cols_per_block) == 8 && (D) % 32 == 0 ? 32 : 16;- const dim3 block_dim(WARP_SIZE, nwarps, 1);- const dim3 blocks_num(parallel_blocks*(Q->ne[1] + cols_per_block - 1) / cols_per_block, Q->ne[2], Q->ne[3]);- const int shmem = 0;+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);+}- float scale;- memcpy(&scale, KQV->op_params, sizeof(float));-- flash_attn_ext_f16<D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), parallel_blocks, KQ_acc_t>- <<<blocks_num, block_dim, shmem, main_stream>>> (- (const char *) Q->data,- (const char *) K->data,- (const char *) V->data,- mask ? ((const char *) mask->data) : nullptr,- (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr,- scale,- Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3],- K->ne[0], K->ne[1], K->ne[2], K->ne[3],- mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0,- Q->nb[1], Q->nb[2], Q->nb[3],- K->nb[1], K->nb[2], K->nb[3],- KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3]- );- CUDA_CHECK(cudaGetLastError());+static __device__ __forceinline__ float vec_dot_q4_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- if ((parallel_blocks) == 1) {- return;- }+#ifndef GGML_QKK_64+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;- const dim3 block_dim_combine(D, 1, 1);- const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z);- const int shmem_combine = 0;+ int v[2];+ int u[2*QR4_K];+ float d8[QR4_K];- flash_attn_combine_results<D, parallel_blocks>- <<<blocks_num_combine, block_dim_combine, shmem_combine, main_stream>>>- (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data);- CUDA_CHECK(cudaGetLastError());-}+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));-template <int D, int cols_per_block, int nwarps, typename KQ_acc_t> void launch_fattn_f16(- const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask,- const int nsm, ggml_cuda_pool & pool, cudaStream_t main_stream-) {- const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3];+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108- if (4*blocks_num_pb1 < 2*nsm) {- launch_fattn_f16_impl<D, cols_per_block, nwarps, 4, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);- return;+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ v[0] = q4[0];+ v[1] = q4[4];++ const uint16_t * scales = (const uint16_t *)bq4_K->scales;+ uint16_t aux[2];+ const int j = bq8_offset/2;+ if (j < 2) {+ aux[0] = scales[j+0] & 0x3f3f;+ aux[1] = scales[j+2] & 0x3f3f;+ } else {+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);}- if (2*blocks_num_pb1 < 2*nsm) {- launch_fattn_f16_impl<D, cols_per_block, nwarps, 2, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);- return;+ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;++ for (int i = 0; i < QR4_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);++ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];}- launch_fattn_f16_impl<D, cols_per_block, nwarps, 1, KQ_acc_t>(Q, K, V, KQV, mask, pool, main_stream);-}-#ifndef GGML_MINIMIZE_CODE_SIZE // [jart]-void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * Q = dst->src[0];- const ggml_tensor * K = dst->src[1];- const ggml_tensor * V = dst->src[2];+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);- const ggml_tensor * mask = dst->src[3];+#else- ggml_tensor * KQV = dst;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;- GGML_ASSERT(Q->type == GGML_TYPE_F32);- GGML_ASSERT(K->type == GGML_TYPE_F16);- GGML_ASSERT(V->type == GGML_TYPE_F16);- GGML_ASSERT(KQV->type == GGML_TYPE_F32);+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;- GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16);- GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) &&- "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big");+ uint16_t aux16[2];+ const uint8_t * s = (const uint8_t *)aux16;- GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding.");+ const uint16_t * a = (const uint16_t *)bq4_K->scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;- ggml_cuda_set_device(ctx.device);+ const float dall = bq4_K->dm[0];+ const float dmin = bq4_K->dm[1];- const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;+ const float d8_1 = __low2float(bq8_1[0].ds);+ const float d8_2 = __low2float(bq8_1[1].ds);- const int32_t precision = KQV->op_params[1];+ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);- if (precision != GGML_PREC_DEFAULT) {- if (Q->ne[1] <= 32 || Q->ne[0] > 128) {- constexpr int cols_per_block = 16;- constexpr int nwarps = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 80:- launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 96:- launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 112:- launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 256:- launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- default:- GGML_ASSERT(false);- break;- }- } else {- constexpr int cols_per_block = 32;- constexpr int nwarps = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 80:- launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 96:- launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 112:- launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- // case 256:- // launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- // break;- default:- GGML_ASSERT(false);- break;- }- }- return;- }+ const int * q4 = (const int *)bq4_K->qs + (iqs/2);+ const int v1 = q4[0];+ const int v2 = q4[4];- if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) {- constexpr int parallel_blocks = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_vec_f16< 64, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_vec_f16<128, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());- break;- case 256:- launch_fattn_vec_f16<256, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream());- break;- default:- GGML_ASSERT(false);- break;- }- return;- }+ const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0));+ const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0));+ const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0));+ const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0));- if (Q->ne[1] <= 8 && Q->ne[0] % WARP_SIZE == 0) {- constexpr int cols_per_block = 8;- constexpr int nwarps = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 96:- launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 256:- launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- default:- GGML_ASSERT(false);- break;- }- return;- }+ sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]);+ sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]);- if (Q->ne[1] <= 32) {- constexpr int cols_per_block = 16;- constexpr int nwarps = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 80:- launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 96:- launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 112:- launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 256:- launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- default:- GGML_ASSERT(false);- break;- }- return;- }+ return dall * sumf_d - dmin * sumf_m;- constexpr int cols_per_block = 32;- constexpr int nwarps = 4;- switch (Q->ne[0]) {- case 64:- launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 80:- launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 96:- launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 112:- launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 128:- launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- case 256:- launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream());- break;- default:- GGML_ASSERT(false);- break;- }- return;+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A++#endif}-#endif // GGML_MINIMIZE_CODE_SIZE [jart]-template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void k_get_rows(- const void * src0, const int32_t * src1, dst_t * dst,- int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/- /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/- /*size_t s0,*/ size_t s1, size_t s2, size_t s3,- /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,- size_t s10, size_t s11, size_t s12/*, size_t s13*/) {+static __device__ __forceinline__ float vec_dot_q5_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;- const int i10 = blockDim.y*blockIdx.y + threadIdx.y;- const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;- const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;+#ifndef GGML_QKK_64+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;- if (i00 >= ne00) {- return;- }+ int vl[2];+ int vh[2];+ int u[2*QR5_K];+ float d8[QR5_K];- const int i01 = src1[i10*s10 + i11*s11 + i12*s12];+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));- dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;- const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;+ vl[0] = ql[0];+ vl[1] = ql[4];- const int ib = i00/qk; // block index- const int iqs = (i00%qk)/qr; // quant index- const int iybs = i00 - i00%qk; // dst block start index- const int y_offset = qr == 1 ? 1 : qk/2;+ vh[0] = qh[0] >> bq8_offset;+ vh[1] = qh[4] >> bq8_offset;- // dequantize- dfloat2 v;- dequantize_kernel(src0_row, ib, iqs, v);+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;+ uint16_t aux[2];+ const int j = bq8_offset/2;+ if (j < 2) {+ aux[0] = scales[j+0] & 0x3f3f;+ aux[1] = scales[j+2] & 0x3f3f;+ } else {+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);+ }+ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;- dst_row[iybs + iqs + 0] = v.x;- dst_row[iybs + iqs + y_offset] = v.y;-}+#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);-template<typename src0_t, typename dst_t>-static __global__ void k_get_rows_float(- const src0_t * src0, const int32_t * src1, dst_t * dst,- int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/- /*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/- /*size_t s0,*/ size_t s1, size_t s2, size_t s3,- /*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,- size_t s10, size_t s11, size_t s12/*, size_t s13*/) {+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];+ }- const int i00 = blockIdx.x*blockDim.x + threadIdx.x;- const int i10 = blockDim.y*blockIdx.y + threadIdx.y;- const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;- const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);- if (i00 >= ne00) {- return;- }+#else- const int i01 = src1[i10*s10 + i11*s11 + i12*s12];+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;- dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;- const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);+ const int8_t * s = bq5_K->scales;- dst_row[i00] = src0_row[i00];-}+ const float d = bq5_K->d;-template<int qk, int qr, dequantize_kernel_t dq>-static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {+ const float d8_1 = __low2half(bq8_1[0].ds);+ const float d8_2 = __low2half(bq8_1[1].ds);- GGML_TENSOR_BINARY_OP_LOCALS+ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);- const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);- const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);- const dim3 block_nums(block_num_x, ne10, ne11*ne12);+ const int * ql = (const int *)bq5_K->qs + (iqs/2);+ const int vl1 = ql[0];+ const int vl2 = ql[4];- // strides in elements- //const size_t s0 = nb0 / ggml_element_size(dst);- const size_t s1 = nb1 / ggml_element_size(dst);- const size_t s2 = nb2 / ggml_element_size(dst);- const size_t s3 = nb3 / ggml_element_size(dst);+ const int step = 4 * (iqs/2); // 0, 4, 8, 12+ const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6+ const int in = step%8; // 0, 4, 0, 4+ const int vh = (*((const int *)(bq5_K->qh + in))) >> im;- const size_t s10 = nb10 / ggml_element_size(src1);- const size_t s11 = nb11 / ggml_element_size(src1);- const size_t s12 = nb12 / ggml_element_size(src1);- //const size_t s13 = nb13 / ggml_element_size(src1);+ const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f);+ const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f);+ const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f);+ const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f);- GGML_ASSERT(ne00 % 2 == 0);+ const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1])+ + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]);- k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne00, /*ne01, ne02, ne03,*/- /*ne10, ne11,*/ ne12, /*ne13,*/- /* s0,*/ s1, s2, s3,- /* nb00,*/ nb01, nb02, nb03,- s10, s11, s12/*, s13*/);+ return d * sumf_d;- GGML_UNUSED(dst);+#else+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A++#endif}-template<typename src0_t>-static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {+static __device__ __forceinline__ float vec_dot_q6_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- GGML_TENSOR_BINARY_OP_LOCALS+ const block_q6_K * bq6_K = (const block_q6_K *) vbq;- const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);- const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;- const dim3 block_nums(block_num_x, ne10, ne11*ne12);+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));- // strides in elements- //const size_t s0 = nb0 / ggml_element_size(dst);- const size_t s1 = nb1 / ggml_element_size(dst);- const size_t s2 = nb2 / ggml_element_size(dst);- const size_t s3 = nb3 / ggml_element_size(dst);+ const int vl = get_int_from_uint8(bq6_K->ql, iqs);+ const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;- const size_t s10 = nb10 / ggml_element_size(src1);- const size_t s11 = nb11 / ggml_element_size(src1);- const size_t s12 = nb12 / ggml_element_size(src1);- //const size_t s13 = nb13 / ggml_element_size(src1);+ const int8_t * scales = bq6_K->scales + scale_offset;- k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(- src0_dd, src1_dd, dst_dd,- ne00, /*ne01, ne02, ne03,*/- /*ne10, ne11,*/ ne12, /*ne13,*/- /* s0,*/ s1, s2, s3,- /* nb00,*/ nb01, nb02, nb03,- s10, s11, s12/*, s13*/);+ int u[QR6_K];+ float d8[QR6_K];- GGML_UNUSED(dst);+#pragma unroll+ for (int i = 0; i < QR6_K; ++i) {+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);+ }++ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);}-void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const ggml_tensor * src1 = dst->src[1];- const float * src0_d = (const float *)src0->data;- const float * src1_d = (const float *)src1->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;+#if QR2_XXS == 8+ const int ib32 = iqs;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const int8_t * q8 = bq8_1[ib32].qs;+ uint32_t aux32 = q2[2] | (q2[3] << 16);+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);+ const uint8_t signs = ksigns_iq2xs[aux32 & 127];+ for (int j = 0; j < 8; ++j) {+ sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ aux32 >>= 7;+ }+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * sumi;+#else+ // iqs is 0...15+ const int ib32 = iqs/2;+ const int il = iqs%2;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]);+ const uint32_t aux32 = q2[2] | (q2[3] << 16);+ const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * __low2float(bq8_1[ib32].ds) * 0.25f;+ const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127];+ const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127];+ const int8_t * q8 = bq8_1[ib32].qs + 16*il;+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < 8; ++j) {+ sumi1 += q8[j+0] * grid1[j] * (signs1 & kmask_iq2xs[j] ? -1 : 1);+ sumi2 += q8[j+8] * grid2[j] * (signs2 & kmask_iq2xs[j] ? -1 : 1);+ }+ return d * (sumi1 + sumi2);+#endif+#else+ NO_DEVICE_CODE;+#endif+}- GGML_ASSERT(src1->type == GGML_TYPE_I32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;- GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));- GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));- GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));+ const int ib32 = iqs;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;+ const uint8_t ls2 = bq2->scales[ib32] >> 4;+ int sumi1 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);+ q8 += 8;+ }+ int sumi2 = 0;+ for (int l = 2; l < 4; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));+ const int grid_l = __vsub4(grid[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid[1] ^ signs[1], signs[1]);+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);+ q8 += 8;+ }+ const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ GGML_UNUSED(ksigns64);+ NO_DEVICE_CODE;+#endif+#else+ GGML_UNUSED(ksigns64);+ NO_DEVICE_CODE;+#endif+}- const int32_t * src1_i32 = (const int32_t *) src1_d;+// TODO+static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq;- switch (src0->type) {- case GGML_TYPE_F16:- get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_F32:- get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q4_0:- get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q4_1:- get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q5_0:- get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q5_1:- get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- case GGML_TYPE_Q8_0:- get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);- break;- default:- // TODO: k-quants- fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));- GGML_ASSERT(false);- break;+ const int ib32 = iqs;+ const int8_t * q8 = bq8_1[ib32].qs;+ const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;+ const uint8_t ls2 = bq2->scales[ib32] >> 4;+ int sumi1 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);+ sumi1 = __dp4a(grid_l, *((const int *)q8 + 0), sumi1);+ sumi1 = __dp4a(grid_h, *((const int *)q8 + 1), sumi1);+ q8 += 8;+ }+ int sumi2 = 0;+ for (int l = 2; l < 4; ++l) {+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));+ const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid[1] ^ signs1, signs1);+ sumi2 = __dp4a(grid_l, *((const int *)q8 + 0), sumi2);+ sumi2 = __dp4a(grid_h, *((const int *)q8 + 1), sumi2);+ q8 += 8;}+ const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ GGML_UNUSED(ksigns64);+ NO_DEVICE_CODE;+#endif+#else+ GGML_UNUSED(ksigns64);+ NO_DEVICE_CODE;+#endif}-template <typename T>-static __global__ void im2col_kernel(- const float * x, T * dst, int64_t batch_offset,- int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,- int s0, int s1, int p0, int p1, int d0, int d1) {- const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;- if (i >= pelements) {- return;- }-- const int64_t ksize = OW * (KH > 1 ? KW : 1);- const int64_t kx = i / ksize;- const int64_t kd = kx * ksize;- const int64_t ky = (i - kd) / OW;- const int64_t ix = i % OW;-- const int64_t oh = blockIdx.y;- const int64_t batch = blockIdx.z / IC;- const int64_t ic = blockIdx.z % IC;+static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;- const int64_t iiw = ix * s0 + kx * d0 - p0;- const int64_t iih = oh * s1 + ky * d1 - p1;+ const int ib32 = iqs;+ const uint8_t * q3 = bq2->qs + 8*ib32;+ const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ uint32_t aux32 = gas[0] | (gas[1] << 16);+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];+ const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));+ const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);+ const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);+ q8 += 8;+ aux32 >>= 7;+ }+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;+ return d * sumi;+#else+ NO_DEVICE_CODE;+#endif+#else+ NO_DEVICE_CODE;+#endif+}- const int64_t offset_dst =- ((batch * OH + oh) * OW + ix) * CHW +- (ic * (KW * KH) + ky * KW + kx);+// TODO: don't use lookup table for signs+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+#if QK_K == 256+ const block_iq3_s * bq2 = (const block_iq3_s *) vbq;- if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- dst[offset_dst] = 0.0f;- } else {- const int64_t offset_src = ic * offset_delta + batch * batch_offset;- dst[offset_dst] = x[offset_src + iih * IW + iiw];+ const int ib32 = iqs;+ const uint8_t * qs = bq2->qs + 8*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ int sumi = 0;+ for (int l = 0; l < 4; ++l) {+ const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));+ const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));+ uint32_t signs0 = __vcmpeq4(((bq2->signs[4*ib32+l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);+ uint32_t signs1 = __vcmpeq4(((bq2->signs[4*ib32+l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);+ const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);+ sumi = __dp4a(grid_l, *((int *)q8+0), sumi);+ sumi = __dp4a(grid_h, *((int *)q8+1), sumi);+ q8 += 8;}+ const float d = (float)bq2->d * (1 + 2*((bq2->scales[ib32/2] >> 4*(ib32%2)) & 0xf)) * __low2float(bq8_1[ib32].ds);+ return d * sumi;+#else+ NO_DEVICE_CODE;+#endif+#else+ NO_DEVICE_CODE;+#endif}-template <typename T>-static void im2col_cuda(const float * x, T* dst,- int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,- int64_t batch, int64_t batch_offset, int64_t offset_delta,- int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {- const int parallel_elements = OW * KW * KH;- const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;- dim3 block_nums(num_blocks, OH, batch * IC);- im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);+static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq;++ const int ib32 = iqs;+ int sumi = 0;+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const int * q8 = (const int *)bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));+ int grid0 = grid[0] & 0x0f0f0f0f;+ int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;+ sumi = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi));+ }+#else+ const int8_t * q8 = bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));+ for (int j = 0; j < 4; ++j) {+ sumi += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4);+ }+ q8 += 8;+ }+#endif+ const float delta = bq1->qh[ib32] & 0x8000 ? -1-IQ1S_DELTA : -1+IQ1S_DELTA;+ const float d1q = (float)bq1->d * (2*((bq1->qh[ib32] >> 12) & 7) + 1);+ const float d = d1q * __low2float (bq8_1[ib32].ds);+ const float m = d1q * __high2float(bq8_1[ib32].ds);+ return d * sumi + m * delta;+#else+ NO_DEVICE_CODE;+#endif}-static void im2col_cuda_f16(const float * x, half * dst,- int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,- int64_t batch, int64_t batch_offset, int64_t offset_delta,- int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {+static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq1_m * bq1 = (const block_iq1_m *) vbq;- im2col_cuda<half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);+ const int ib32 = iqs;+ int sumi[2] = {0, 0};+ float sumf[2] = {0.f, 0.f};+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const int * q8 = (const int *)bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 7) << 8)));+ int grid0 = grid[0] & 0x0f0f0f0f;+ int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;+ sumi[l/2] = __dp4a(q8[2*l+1], grid1, __dp4a(q8[2*l+0], grid0, sumi[l/2]));+ const float delta = (bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 0x08 ? -1-IQ1M_DELTA : -1+IQ1M_DELTA;+ const int sumy = __dp4a(q8[2*l+1], 0x01010101, __dp4a(q8[2*l+0], 0x01010101, 0));+ sumf[l/2] += delta*sumy;+ }+#else+ const int8_t * q8 = bq8_1[ib32].qs;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));+ int sumy = 0;+ for (int j = 0; j < 4; ++j) {+ sumi[l/2] += q8[j] * (grid[j] & 0xf) + q8[j+4] * (grid[j] >> 4);+ sumy += q8[j] + q8[j+4];+ }+ const float delta = (bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 0x08 ? -1-IQ1M_DELTA : -1+IQ1M_DELTA;+ sumf[l/2] += delta*sumy;+ q8 += 8;+ }+#endif+ iq1m_scale_t scale;+ const uint16_t * sc = (const uint16_t *)bq1->scales;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ const float d = (float)scale.f16 * __low2float (bq8_1[ib32].ds);+ return d * ((sumi[0] + sumf[0]) * (2*((sc[ib32/2] >> 6*(ib32%2)) & 0x7) + 1) + (sumi[1] + sumf[1]) * (2*((sc[ib32/2] >> (6*(ib32%2)+3)) & 0x7) + 1));+#else+ NO_DEVICE_CODE;+#endif}-static void im2col_cuda_f32(const float * x, float * dst,- int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,- int64_t batch, int64_t batch_offset, int64_t offset_delta,- int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,+ int & val1, int & val2) {- im2col_cuda<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);+ uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;+ aux32 = q4 & 0x0f0f0f0f;+ uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);+ uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);+ val1 = v1 | (v2 << 16);+ aux32 = (q4 >> 4) & 0x0f0f0f0f;+ v1 = values[q8[0]] | (values[q8[1]] << 8);+ v2 = values[q8[2]] | (values[q8[3]] << 8);+ val2 = v1 | (v2 << 16);}+#endif-void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * src0 = dst->src[0];- const ggml_tensor * src1 = dst->src[1];- const float * src1_d = (const float *)src1->data;- float * dst_d = (float *)dst->data;- cudaStream_t stream = ctx.stream();+static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);+ const block_iq4_nl * bq = (const block_iq4_nl *) vbq;- const int32_t s0 = ((const int32_t*)(dst->op_params))[0];- const int32_t s1 = ((const int32_t*)(dst->op_params))[1];- const int32_t p0 = ((const int32_t*)(dst->op_params))[2];- const int32_t p1 = ((const int32_t*)(dst->op_params))[3];- const int32_t d0 = ((const int32_t*)(dst->op_params))[4];- const int32_t d1 = ((const int32_t*)(dst->op_params))[5];+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics+ const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;+ const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;- const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;- const int64_t IC = src1->ne[is_2D ? 2 : 1];- const int64_t IH = is_2D ? src1->ne[1] : 1;- const int64_t IW = src1->ne[0];+ int v1, v2;+ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {+ const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);+ get_int_from_table_16(aux, values, v1, v2);+ sumi1 = __dp4a(v1, q8[l+0], sumi1);+ sumi2 = __dp4a(v2, q8[l+4], sumi2);+ }- const int64_t KH = is_2D ? src0->ne[1] : 1;- const int64_t KW = src0->ne[0];+#else+ const uint8_t * q4 = bq->qs + 4*iqs;+ const int8_t * q8 = bq8_1->qs + 4*iqs;++ int sumi1 = 0, sumi2 = 0;+ for (int l = 0; l < 4*VDR_Q4_0_Q8_1_MMVQ; ++l) {+ sumi1 += q8[l+ 0] * kvalues_iq4nl[q4[l] & 0xf];+ sumi2 += q8[l+16] * kvalues_iq4nl[q4[l] >> 4];+ }+#endif+ const float d = (float)bq->d * __low2float(bq8_1->ds);+ return d * (sumi1 + sumi2);+}- const int64_t OH = is_2D ? dst->ne[2] : 1;- const int64_t OW = dst->ne[1];+static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {- const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32- const int64_t batch = src1->ne[3];- const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32+#if QK_K == 256+#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics- if(dst->type == GGML_TYPE_F16) {- im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);- } else {- im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;++ // iqs is 0...7+ const int ib32 = iqs;+ const int32_t * q8 = (const int *)bq8_1[ib32].qs;+ const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;+ const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);+ const float d = (float)bq4->d * (ls - 32) * __low2float(bq8_1[ib32].ds);+ int v1, v2;+ int sumi1 = 0, sumi2 = 0;+ for (int j = 0; j < 4; ++j) {+ get_int_from_table_16(q4[j], values, v1, v2);+ sumi1 = __dp4a(v1, q8[j+0], sumi1);+ sumi2 = __dp4a(v2, q8[j+4], sumi2);}+ return d * (sumi1 + sumi2);++#else+ NO_DEVICE_CODE;+#endif+#else+ return vec_dot_iq4_xs_q8_1(vbq, bq8_1, iqs);+#endif}typedef void (*allocate_tiles_cuda_t)(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc);@@ -6272,6 +7596,135 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)(const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc,const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k);typedef void (*dot_kernel_k_t)(const void * __restrict__ vx, const int ib, const int iqs, const float * __restrict__ y, float & v);+typedef void (mul_mat_q_t)(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst);++struct mmq_arch_config_t {+ int x;+ int y;+ int nwarps;+};++struct mmq_config_t {+ mmq_arch_config_t rdna2;+ mmq_arch_config_t rdna1;+ mmq_arch_config_t ampere;+ mmq_arch_config_t pascal;+};++constexpr mmq_config_t MMQ_CONFIG_Q4_0 = {+// x y nwarps+ { 64, 128, 8},+ { 64, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q4_1 = {+// x y nwarps+ { 64, 128, 8},+ { 64, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q5_0 = {+// x y nwarps+ { 64, 128, 8},+ { 64, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ {128, 64, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q5_1 = {+// x y nwarps+ { 64, 128, 8},+ { 64, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ {128, 64, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q8_0 = {+// x y nwarps+ { 64, 128, 8},+ { 64, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ {128, 64, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q2_K = {+// x y nwarps+ { 64, 128, 8},+ {128, 32, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q3_K = {+// x y nwarps+ {128, 64, 8},+ { 32, 128, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ {128, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q4_K = {+// x y nwarps+ { 64, 128, 8},+ { 32, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q5_K = {+// x y nwarps+ { 64, 128, 8},+ { 32, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 128, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};+constexpr mmq_config_t MMQ_CONFIG_Q6_K = {+// x y nwarps+ { 64, 128, 8},+ { 32, 64, 8},+#ifdef CUDA_USE_TENSOR_CORES+ { 4, 32, 4},+#else+ { 64, 64, 4},+#endif // CUDA_USE_TENSOR_CORES+ { 64, 64, 8},+};++// ------------------------------------------------------------template <int mmq_y> static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) {GGML_UNUSED(x_qh);@@ -7206,25 +8659,6 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]);}-#define MMQ_X_Q4_0_RDNA2 64-#define MMQ_Y_Q4_0_RDNA2 128-#define NWARPS_Q4_0_RDNA2 8-#define MMQ_X_Q4_0_RDNA1 64-#define MMQ_Y_Q4_0_RDNA1 64-#define NWARPS_Q4_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_0_AMPERE 4-#define MMQ_Y_Q4_0_AMPERE 32-#define NWARPS_Q4_0_AMPERE 4-#else-#define MMQ_X_Q4_0_AMPERE 64-#define MMQ_Y_Q4_0_AMPERE 128-#define NWARPS_Q4_0_AMPERE 4-#endif-#define MMQ_X_Q4_0_PASCAL 64-#define MMQ_Y_Q4_0_PASCAL 64-#define NWARPS_Q4_0_PASCAL 8-template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>static __device__ __forceinline__ void mul_mat_q(@@ -7335,1118 +8769,276 @@ static __device__ __forceinline__ void mul_mat_q(}}-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_0_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q4_0(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {--#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_0_RDNA2;- const int mmq_y = MMQ_Y_Q4_0_RDNA2;- const int nwarps = NWARPS_Q4_0_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_0_RDNA1;- const int mmq_y = MMQ_Y_Q4_0_RDNA1;- const int nwarps = NWARPS_Q4_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_0_AMPERE;- const int mmq_y = MMQ_Y_Q4_0_AMPERE;- const int nwarps = NWARPS_Q4_0_AMPERE;-- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_0_PASCAL;- const int mmq_y = MMQ_Y_Q4_0_PASCAL;- const int nwarps = NWARPS_Q4_0_PASCAL;-- mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat);- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}--#define MMQ_X_Q4_1_RDNA2 64-#define MMQ_Y_Q4_1_RDNA2 128-#define NWARPS_Q4_1_RDNA2 8-#define MMQ_X_Q4_1_RDNA1 64-#define MMQ_Y_Q4_1_RDNA1 64-#define NWARPS_Q4_1_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_1_AMPERE 4-#define MMQ_Y_Q4_1_AMPERE 32-#define NWARPS_Q4_1_AMPERE 4-#else-#define MMQ_X_Q4_1_AMPERE 64-#define MMQ_Y_Q4_1_AMPERE 128-#define NWARPS_Q4_1_AMPERE 4-#endif-#define MMQ_X_Q4_1_PASCAL 64-#define MMQ_Y_Q4_1_PASCAL 64-#define NWARPS_Q4_1_PASCAL 8+static constexpr __device__ mmq_arch_config_t get_arch_config_device(mmq_config_t mmq_config) {-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q4_1(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_1_RDNA2;- const int mmq_y = MMQ_Y_Q4_1_RDNA2;- const int nwarps = NWARPS_Q4_1_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_1_RDNA1;- const int mmq_y = MMQ_Y_Q4_1_RDNA1;- const int nwarps = NWARPS_Q4_1_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_1_AMPERE;- const int mmq_y = MMQ_Y_Q4_1_AMPERE;- const int nwarps = NWARPS_Q4_1_AMPERE;-- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_1_PASCAL;- const int mmq_y = MMQ_Y_Q4_1_PASCAL;- const int nwarps = NWARPS_Q4_1_PASCAL;-- mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+ return mmq_config.rdna2;#else- GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat);- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}--#define MMQ_X_Q5_0_RDNA2 64-#define MMQ_Y_Q5_0_RDNA2 128-#define NWARPS_Q5_0_RDNA2 8-#define MMQ_X_Q5_0_RDNA1 64-#define MMQ_Y_Q5_0_RDNA1 64-#define NWARPS_Q5_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_0_AMPERE 4-#define MMQ_Y_Q5_0_AMPERE 32-#define NWARPS_Q5_0_AMPERE 4-#else-#define MMQ_X_Q5_0_AMPERE 128-#define MMQ_Y_Q5_0_AMPERE 64-#define NWARPS_Q5_0_AMPERE 4-#endif-#define MMQ_X_Q5_0_PASCAL 64-#define MMQ_Y_Q5_0_PASCAL 64-#define NWARPS_Q5_0_PASCAL 8--template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_0_RDNA2, 2)+ return mmq_config.rdna1;#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q5_0(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_0_RDNA2;- const int mmq_y = MMQ_Y_Q5_0_RDNA2;- const int nwarps = NWARPS_Q5_0_RDNA2;#else- const int mmq_x = MMQ_X_Q5_0_RDNA1;- const int mmq_y = MMQ_Y_Q5_0_RDNA1;- const int nwarps = NWARPS_Q5_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_0_AMPERE;- const int mmq_y = MMQ_Y_Q5_0_AMPERE;- const int nwarps = NWARPS_Q5_0_AMPERE;-- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_0_PASCAL;- const int mmq_y = MMQ_Y_Q5_0_PASCAL;- const int nwarps = NWARPS_Q5_0_PASCAL;-- mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#if __CUDA_ARCH__ >= CC_VOLTA+ return mmq_config.ampere;#else- GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat);- NO_DEVICE_CODE;+ return mmq_config.pascal;#endif // __CUDA_ARCH__ >= CC_VOLTA-}--#define MMQ_X_Q5_1_RDNA2 64-#define MMQ_Y_Q5_1_RDNA2 128-#define NWARPS_Q5_1_RDNA2 8-#define MMQ_X_Q5_1_RDNA1 64-#define MMQ_Y_Q5_1_RDNA1 64-#define NWARPS_Q5_1_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_1_AMPERE 4-#define MMQ_Y_Q5_1_AMPERE 32-#define NWARPS_Q5_1_AMPERE 4-#else-#define MMQ_X_Q5_1_AMPERE 128-#define MMQ_Y_Q5_1_AMPERE 64-#define NWARPS_Q5_1_AMPERE 4-#endif-#define MMQ_X_Q5_1_PASCAL 64-#define MMQ_Y_Q5_1_PASCAL 64-#define NWARPS_Q5_1_PASCAL 8-template <bool need_check> static __global__ void-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_1_RDNA2, 2)-#endif // defined(RDNA3) || defined(RDNA2)#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q5_1(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {--#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_1_RDNA2;- const int mmq_y = MMQ_Y_Q5_1_RDNA2;- const int nwarps = NWARPS_Q5_1_RDNA2;-#else- const int mmq_x = MMQ_X_Q5_1_RDNA1;- const int mmq_y = MMQ_Y_Q5_1_RDNA1;- const int nwarps = NWARPS_Q5_1_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_1_AMPERE;- const int mmq_y = MMQ_Y_Q5_1_AMPERE;- const int nwarps = NWARPS_Q5_1_AMPERE;-- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_1_PASCAL;- const int mmq_y = MMQ_Y_Q5_1_PASCAL;- const int nwarps = NWARPS_Q5_1_PASCAL;-- mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-#else- GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat);- NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA}-#define MMQ_X_Q8_0_RDNA2 64-#define MMQ_Y_Q8_0_RDNA2 128-#define NWARPS_Q8_0_RDNA2 8-#define MMQ_X_Q8_0_RDNA1 64-#define MMQ_Y_Q8_0_RDNA1 64-#define NWARPS_Q8_0_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q8_0_AMPERE 4-#define MMQ_Y_Q8_0_AMPERE 32-#define NWARPS_Q8_0_AMPERE 4-#else-#define MMQ_X_Q8_0_AMPERE 128-#define MMQ_Y_Q8_0_AMPERE 64-#define NWARPS_Q8_0_AMPERE 4-#endif-#define MMQ_X_Q8_0_PASCAL 64-#define MMQ_Y_Q8_0_PASCAL 64-#define NWARPS_Q8_0_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q8_0_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_0.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- mul_mat_q8_0(+ mul_mat_q4_0(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q8_0_RDNA2;- const int mmq_y = MMQ_Y_Q8_0_RDNA2;- const int nwarps = NWARPS_Q8_0_RDNA2;-#else- const int mmq_x = MMQ_X_Q8_0_RDNA1;- const int mmq_y = MMQ_Y_Q8_0_RDNA1;- const int nwarps = NWARPS_Q8_0_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q8_0_AMPERE;- const int mmq_y = MMQ_Y_Q8_0_AMPERE;- const int nwarps = NWARPS_Q8_0_AMPERE;-- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q8_0_PASCAL;- const int mmq_y = MMQ_Y_Q8_0_PASCAL;- const int nwarps = NWARPS_Q8_0_PASCAL;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_0);- mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>+ mul_mat_q<QK4_0, QR4_0, QI4_0, true, block_q4_0, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q4_0<arch_config.y>,+ load_tiles_q4_0<arch_config.y, arch_config.nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define MMQ_X_Q2_K_RDNA2 64-#define MMQ_Y_Q2_K_RDNA2 128-#define NWARPS_Q2_K_RDNA2 8-#define MMQ_X_Q2_K_RDNA1 128-#define MMQ_Y_Q2_K_RDNA1 32-#define NWARPS_Q2_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q2_K_AMPERE 4-#define MMQ_Y_Q2_K_AMPERE 32-#define NWARPS_Q2_K_AMPERE 4-#else-#define MMQ_X_Q2_K_AMPERE 64-#define MMQ_Y_Q2_K_AMPERE 128-#define NWARPS_Q2_K_AMPERE 4-#endif-#define MMQ_X_Q2_K_PASCAL 64-#define MMQ_Y_Q2_K_PASCAL 64-#define NWARPS_Q2_K_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q2_K_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_1.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q2_K(+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_1.pascal.nwarps, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q4_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q2_K_RDNA2;- const int mmq_y = MMQ_Y_Q2_K_RDNA2;- const int nwarps = NWARPS_Q2_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q2_K_RDNA1;- const int mmq_y = MMQ_Y_Q2_K_RDNA1;- const int nwarps = NWARPS_Q2_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q2_K_AMPERE;- const int mmq_y = MMQ_Y_Q2_K_AMPERE;- const int nwarps = NWARPS_Q2_K_AMPERE;-- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q2_K_PASCAL;- const int mmq_y = MMQ_Y_Q2_K_PASCAL;- const int nwarps = NWARPS_Q2_K_PASCAL;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_1);- mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>+ mul_mat_q<QK4_1, QR4_1, QI4_1, true, block_q4_1, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q4_1<arch_config.y>,+ load_tiles_q4_1<arch_config.y, arch_config.nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define MMQ_X_Q3_K_RDNA2 128-#define MMQ_Y_Q3_K_RDNA2 64-#define NWARPS_Q3_K_RDNA2 8-#define MMQ_X_Q3_K_RDNA1 32-#define MMQ_Y_Q3_K_RDNA1 128-#define NWARPS_Q3_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q3_K_AMPERE 4-#define MMQ_Y_Q3_K_AMPERE 32-#define NWARPS_Q3_K_AMPERE 4-#else-#define MMQ_X_Q3_K_AMPERE 128-#define MMQ_Y_Q3_K_AMPERE 128-#define NWARPS_Q3_K_AMPERE 4-#endif-#define MMQ_X_Q3_K_PASCAL 64-#define MMQ_Y_Q3_K_PASCAL 64-#define NWARPS_Q3_K_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_0.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q3_K(- const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,- const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {--#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q3_K_RDNA2;- const int mmq_y = MMQ_Y_Q3_K_RDNA2;- const int nwarps = NWARPS_Q3_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q3_K_RDNA1;- const int mmq_y = MMQ_Y_Q3_K_RDNA1;- const int nwarps = NWARPS_Q3_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q3_K_AMPERE;- const int mmq_y = MMQ_Y_Q3_K_AMPERE;- const int nwarps = NWARPS_Q3_K_AMPERE;-- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ mul_mat_q5_0(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q3_K_PASCAL;- const int mmq_y = MMQ_Y_Q3_K_PASCAL;- const int nwarps = NWARPS_Q3_K_PASCAL;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_0);- mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>+ mul_mat_q<QK5_0, QR5_0, QI5_0, false, block_q5_0, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q5_0<arch_config.y>,+ load_tiles_q5_0<arch_config.y, arch_config.nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define MMQ_X_Q4_K_RDNA2 64-#define MMQ_Y_Q4_K_RDNA2 128-#define NWARPS_Q4_K_RDNA2 8-#define MMQ_X_Q4_K_RDNA1 32-#define MMQ_Y_Q4_K_RDNA1 64-#define NWARPS_Q4_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q4_K_AMPERE 4-#define MMQ_Y_Q4_K_AMPERE 32-#define NWARPS_Q4_K_AMPERE 4-#else-#define MMQ_X_Q4_K_AMPERE 64-#define MMQ_Y_Q4_K_AMPERE 128-#define NWARPS_Q4_K_AMPERE 4-#endif-#define MMQ_X_Q4_K_PASCAL 64-#define MMQ_Y_Q4_K_PASCAL 64-#define NWARPS_Q4_K_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_1.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)-#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_PASCAL, 2)-#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q4_K(+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q5_1(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q4_K_RDNA2;- const int mmq_y = MMQ_Y_Q4_K_RDNA2;- const int nwarps = NWARPS_Q4_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q4_K_RDNA1;- const int mmq_y = MMQ_Y_Q4_K_RDNA1;- const int nwarps = NWARPS_Q4_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q4_K_AMPERE;- const int mmq_y = MMQ_Y_Q4_K_AMPERE;- const int nwarps = NWARPS_Q4_K_AMPERE;-- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q4_K_PASCAL;- const int mmq_y = MMQ_Y_Q4_K_PASCAL;- const int nwarps = NWARPS_Q4_K_PASCAL;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_1);- mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>+ mul_mat_q<QK5_1, QR5_1, QI5_1, true, block_q5_1, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q5_1<arch_config.y>,+ load_tiles_q5_1<arch_config.y, arch_config.nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define MMQ_X_Q5_K_RDNA2 64-#define MMQ_Y_Q5_K_RDNA2 128-#define NWARPS_Q5_K_RDNA2 8-#define MMQ_X_Q5_K_RDNA1 32-#define MMQ_Y_Q5_K_RDNA1 64-#define NWARPS_Q5_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q5_K_AMPERE 4-#define MMQ_Y_Q5_K_AMPERE 32-#define NWARPS_Q5_K_AMPERE 4-#else-#define MMQ_X_Q5_K_AMPERE 64-#define MMQ_Y_Q5_K_AMPERE 128-#define NWARPS_Q5_K_AMPERE 4-#endif-#define MMQ_X_Q5_K_PASCAL 64-#define MMQ_Y_Q5_K_PASCAL 64-#define NWARPS_Q5_K_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q5_K_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q8_0.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-mul_mat_q5_K(+ mul_mat_q8_0(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q5_K_RDNA2;- const int mmq_y = MMQ_Y_Q5_K_RDNA2;- const int nwarps = NWARPS_Q5_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q5_K_RDNA1;- const int mmq_y = MMQ_Y_Q5_K_RDNA1;- const int nwarps = NWARPS_Q5_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q8_0);- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ mul_mat_q<QK8_0, QR8_0, QI8_0, false, block_q8_0, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q8_0<arch_config.y>,+ load_tiles_q8_0<arch_config.y, arch_config.nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A+}-#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q5_K_AMPERE;- const int mmq_y = MMQ_Y_Q5_K_AMPERE;- const int nwarps = NWARPS_Q5_K_AMPERE;-- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q2_K.rdna2.nwarps, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q2_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q5_K_PASCAL;- const int mmq_y = MMQ_Y_Q5_K_PASCAL;- const int nwarps = NWARPS_Q5_K_PASCAL;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q2_K);- mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ mul_mat_q<QK_K, QR2_K, QI2_K, false, block_q2_K, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q2_K<arch_config.y>,+ load_tiles_q2_K<arch_config.y, arch_config.nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-#define MMQ_X_Q6_K_RDNA2 64-#define MMQ_Y_Q6_K_RDNA2 128-#define NWARPS_Q6_K_RDNA2 8-#define MMQ_X_Q6_K_RDNA1 32-#define MMQ_Y_Q6_K_RDNA1 64-#define NWARPS_Q6_K_RDNA1 8-#if defined(CUDA_USE_TENSOR_CORES)-#define MMQ_X_Q6_K_AMPERE 4-#define MMQ_Y_Q6_K_AMPERE 32-#define NWARPS_Q6_K_AMPERE 4-#else-#define MMQ_X_Q6_K_AMPERE 64-#define MMQ_Y_Q6_K_AMPERE 64-#define NWARPS_Q6_K_AMPERE 4-#endif-#define MMQ_X_Q6_K_PASCAL 64-#define MMQ_Y_Q6_K_PASCAL 64-#define NWARPS_Q6_K_PASCAL 8-template <bool need_check> static __global__ void#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)#if defined(RDNA3) || defined(RDNA2)- __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_RDNA2, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q3_K.rdna2.nwarps, 2)#endif // defined(RDNA3) || defined(RDNA2)#elif __CUDA_ARCH__ < CC_VOLTA- __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_PASCAL, 2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q3_K.pascal.nwarps, 2)#endif // __CUDA_ARCH__ < CC_VOLTA- mul_mat_q6_K(+ mul_mat_q3_K(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)-#if defined(RDNA3) || defined(RDNA2)- const int mmq_x = MMQ_X_Q6_K_RDNA2;- const int mmq_y = MMQ_Y_Q6_K_RDNA2;- const int nwarps = NWARPS_Q6_K_RDNA2;-#else- const int mmq_x = MMQ_X_Q6_K_RDNA1;- const int mmq_y = MMQ_Y_Q6_K_RDNA1;- const int nwarps = NWARPS_Q6_K_RDNA1;-#endif // defined(RDNA3) || defined(RDNA2)-- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= CC_VOLTA- const int mmq_x = MMQ_X_Q6_K_AMPERE;- const int mmq_y = MMQ_Y_Q6_K_AMPERE;- const int nwarps = NWARPS_Q6_K_AMPERE;+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q3_K);- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);--#elif __CUDA_ARCH__ >= MIN_CC_DP4A- const int mmq_x = MMQ_X_Q6_K_PASCAL;- const int mmq_y = MMQ_Y_Q6_K_PASCAL;- const int nwarps = NWARPS_Q6_K_PASCAL;-- mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>+ mul_mat_q<QK_K, QR3_K, QI3_K, false, block_q3_K, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q3_K<arch_config.y>,+ load_tiles_q3_K<arch_config.y, arch_config.nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else- GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat);+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat);NO_DEVICE_CODE;-#endif // __CUDA_ARCH__ >= CC_VOLTA-}--static void ggml_mul_mat_q4_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_0_RDNA2;- mmq_y = MMQ_Y_Q4_0_RDNA2;- nwarps = NWARPS_Q4_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_0_RDNA1;- mmq_y = MMQ_Y_Q4_0_RDNA1;- nwarps = NWARPS_Q4_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_0_AMPERE;- mmq_y = MMQ_Y_Q4_0_AMPERE;- nwarps = NWARPS_Q4_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_0_PASCAL;- mmq_y = MMQ_Y_Q4_0_PASCAL;- nwarps = NWARPS_Q4_0_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q4_1_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_1_RDNA2;- mmq_y = MMQ_Y_Q4_1_RDNA2;- nwarps = NWARPS_Q4_1_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_1_RDNA1;- mmq_y = MMQ_Y_Q4_1_RDNA1;- nwarps = NWARPS_Q4_1_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_1_AMPERE;- mmq_y = MMQ_Y_Q4_1_AMPERE;- nwarps = NWARPS_Q4_1_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_1_PASCAL;- mmq_y = MMQ_Y_Q4_1_PASCAL;- nwarps = NWARPS_Q4_1_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q5_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_0_RDNA2;- mmq_y = MMQ_Y_Q5_0_RDNA2;- nwarps = NWARPS_Q5_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_0_RDNA1;- mmq_y = MMQ_Y_Q5_0_RDNA1;- nwarps = NWARPS_Q5_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_0_AMPERE;- mmq_y = MMQ_Y_Q5_0_AMPERE;- nwarps = NWARPS_Q5_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_0_PASCAL;- mmq_y = MMQ_Y_Q5_0_PASCAL;- nwarps = NWARPS_Q5_0_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q5_1_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_1_RDNA2;- mmq_y = MMQ_Y_Q5_1_RDNA2;- nwarps = NWARPS_Q5_1_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_1_RDNA1;- mmq_y = MMQ_Y_Q5_1_RDNA1;- nwarps = NWARPS_Q5_1_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_1_AMPERE;- mmq_y = MMQ_Y_Q5_1_AMPERE;- nwarps = NWARPS_Q5_1_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_1_PASCAL;- mmq_y = MMQ_Y_Q5_1_PASCAL;- nwarps = NWARPS_Q5_1_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_1<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q8_0_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q8_0_RDNA2;- mmq_y = MMQ_Y_Q8_0_RDNA2;- nwarps = NWARPS_Q8_0_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q8_0_RDNA1;- mmq_y = MMQ_Y_Q8_0_RDNA1;- nwarps = NWARPS_Q8_0_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q8_0_AMPERE;- mmq_y = MMQ_Y_Q8_0_AMPERE;- nwarps = NWARPS_Q8_0_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q8_0_PASCAL;- mmq_y = MMQ_Y_Q8_0_PASCAL;- nwarps = NWARPS_Q8_0_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q8_0<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q2_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q2_K_RDNA2;- mmq_y = MMQ_Y_Q2_K_RDNA2;- nwarps = NWARPS_Q2_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q2_K_RDNA1;- mmq_y = MMQ_Y_Q2_K_RDNA1;- nwarps = NWARPS_Q2_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q2_K_AMPERE;- mmq_y = MMQ_Y_Q2_K_AMPERE;- nwarps = NWARPS_Q2_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q2_K_PASCAL;- mmq_y = MMQ_Y_Q2_K_PASCAL;- nwarps = NWARPS_Q2_K_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q2_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--static void ggml_mul_mat_q3_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {--#if QK_K == 256-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q3_K_RDNA2;- mmq_y = MMQ_Y_Q3_K_RDNA2;- nwarps = NWARPS_Q3_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q3_K_RDNA1;- mmq_y = MMQ_Y_Q3_K_RDNA1;- nwarps = NWARPS_Q3_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q3_K_AMPERE;- mmq_y = MMQ_Y_Q3_K_AMPERE;- nwarps = NWARPS_Q3_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q3_K_PASCAL;- mmq_y = MMQ_Y_Q3_K_PASCAL;- nwarps = NWARPS_Q3_K_PASCAL;- } else {- GGML_ASSERT(false);- }-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q3_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-#endif+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static void ggml_mul_mat_q4_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q4_K_RDNA2;- mmq_y = MMQ_Y_Q4_K_RDNA2;- nwarps = NWARPS_Q4_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q4_K_RDNA1;- mmq_y = MMQ_Y_Q4_K_RDNA1;- nwarps = NWARPS_Q4_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q4_K_AMPERE;- mmq_y = MMQ_Y_Q4_K_AMPERE;- nwarps = NWARPS_Q4_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q4_K_PASCAL;- mmq_y = MMQ_Y_Q4_K_PASCAL;- nwarps = NWARPS_Q4_K_PASCAL;- } else {- GGML_ASSERT(false);- }+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.rdna2.nwarps, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.pascal.nwarps, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q4_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_K);- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }+ mul_mat_q<QK_K, QR4_K, QI4_K, true, block_q4_K, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q4_K<arch_config.y>,+ load_tiles_q4_K<arch_config.y, arch_config.nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static void ggml_mul_mat_q5_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q5_K_RDNA2;- mmq_y = MMQ_Y_Q5_K_RDNA2;- nwarps = NWARPS_Q5_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q5_K_RDNA1;- mmq_y = MMQ_Y_Q5_K_RDNA1;- nwarps = NWARPS_Q5_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q5_K_AMPERE;- mmq_y = MMQ_Y_Q5_K_AMPERE;- nwarps = NWARPS_Q5_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q5_K_PASCAL;- mmq_y = MMQ_Y_Q5_K_PASCAL;- nwarps = NWARPS_Q5_K_PASCAL;- } else {- GGML_ASSERT(false);- }+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_K.rdna2.nwarps, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+mul_mat_q5_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_K);- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }+ mul_mat_q<QK_K, QR5_K, QI5_K, true, block_q5_K, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q5_K<arch_config.y>,+ load_tiles_q5_K<arch_config.y, arch_config.nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}-static void ggml_mul_mat_q6_K_q8_1_cuda(- const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,- const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {-- int id;- CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = ggml_cuda_info().devices[id].cc;-- int mmq_x, mmq_y, nwarps;- if (compute_capability >= CC_RDNA2) {- mmq_x = MMQ_X_Q6_K_RDNA2;- mmq_y = MMQ_Y_Q6_K_RDNA2;- nwarps = NWARPS_Q6_K_RDNA2;- } else if (compute_capability >= CC_OFFSET_AMD) {- mmq_x = MMQ_X_Q6_K_RDNA1;- mmq_y = MMQ_Y_Q6_K_RDNA1;- nwarps = NWARPS_Q6_K_RDNA1;- } else if (compute_capability >= CC_VOLTA) {- mmq_x = MMQ_X_Q6_K_AMPERE;- mmq_y = MMQ_Y_Q6_K_AMPERE;- nwarps = NWARPS_Q6_K_AMPERE;- } else if (compute_capability >= MIN_CC_DP4A) {- mmq_x = MMQ_X_Q6_K_PASCAL;- mmq_y = MMQ_Y_Q6_K_PASCAL;- nwarps = NWARPS_Q6_K_PASCAL;- } else {- GGML_ASSERT(false);- }+template <bool need_check> static __global__ void+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#if defined(RDNA3) || defined(RDNA2)+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q6_K.rdna2.nwarps, 2)+#endif // defined(RDNA3) || defined(RDNA2)+#elif __CUDA_ARCH__ < CC_VOLTA+ __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.pascal.nwarps, 2)+#endif // __CUDA_ARCH__ < CC_VOLTA+ mul_mat_q6_K(+ const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,+ const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE, nwarps, 1);+#if __CUDA_ARCH__ >= MIN_CC_DP4A+ constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q6_K);- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q6_K<need_check><<<block_nums, block_dims, 0, stream>>>- (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }+ mul_mat_q<QK_K, QR6_K, QI6_K, false, block_q6_K, arch_config.x, arch_config.y, arch_config.nwarps, allocate_tiles_q6_K<arch_config.y>,+ load_tiles_q6_K<arch_config.y, arch_config.nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>+ (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);+#else+ GGML_UNUSED(get_arch_config_device);+ GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}+#define MMQ_SWITCH_CASE(type_suffix) \+ case GGML_TYPE_Q##type_suffix: if (row_diff % arch_config.y == 0) { \+ const bool need_check = false; \+ mul_mat_q##type_suffix<need_check><<<block_nums, block_dims, 0, stream>>> \+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst); \+ } else { \+ const bool need_check = true; \+ mul_mat_q##type_suffix<need_check><<<block_nums, block_dims, 0, stream>>> \+ (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst); \+ } break; \+void ggml_cuda_op_mul_mat_q(ggml_backend_cuda_context & ctx,const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,@@ -8463,42 +9055,79 @@ void ggml_cuda_op_mul_mat_q(const int64_t row_diff = row_high - row_low;int id = ggml_cuda_get_device();+ const int compute_capability = ggml_cuda_info().devices[id].cc;// the main device has a larger memory buffer to hold the results from all GPUs// nrows_dst == nrows of the matrix that the kernel writes intoconst int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;+ mmq_config_t mmq_config;+switch (src0->type) {case GGML_TYPE_Q4_0:- ggml_mul_mat_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q4_0;break;case GGML_TYPE_Q4_1:- ggml_mul_mat_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q4_1;break;case GGML_TYPE_Q5_0:- ggml_mul_mat_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q5_0;break;case GGML_TYPE_Q5_1:- ggml_mul_mat_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q5_1;break;case GGML_TYPE_Q8_0:- ggml_mul_mat_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q8_0;break;case GGML_TYPE_Q2_K:- ggml_mul_mat_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q2_K;break;case GGML_TYPE_Q3_K:- ggml_mul_mat_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q3_K;break;case GGML_TYPE_Q4_K:- ggml_mul_mat_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q4_K;break;case GGML_TYPE_Q5_K:- ggml_mul_mat_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q5_K;break;case GGML_TYPE_Q6_K:- ggml_mul_mat_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream);+ mmq_config = MMQ_CONFIG_Q6_K;+ break;+ default:+ GGML_ASSERT(false);break;+ }++ mmq_arch_config_t arch_config;+ if (compute_capability >= CC_RDNA2) {+ arch_config = mmq_config.rdna2;+ } else if (compute_capability >= CC_OFFSET_AMD) {+ arch_config = mmq_config.rdna1;+ } else if (compute_capability >= CC_VOLTA) {+ arch_config = mmq_config.ampere;+ } else if (compute_capability >= MIN_CC_DP4A) {+ arch_config = mmq_config.pascal;+ } else {+ GGML_ASSERT(false);+ }++ const int block_num_x = (row_diff + arch_config.y - 1) / arch_config.y;+ const int block_num_y = (src1_ncols + arch_config.x - 1) / arch_config.x;+ const dim3 block_nums(block_num_x, block_num_y, 1);+ const dim3 block_dims(WARP_SIZE, arch_config.nwarps, 1);++ switch (src0->type) {+ MMQ_SWITCH_CASE(4_0)+ MMQ_SWITCH_CASE(4_1)+ MMQ_SWITCH_CASE(5_0)+ MMQ_SWITCH_CASE(5_1)+ MMQ_SWITCH_CASE(8_0)+ MMQ_SWITCH_CASE(2_K)+ MMQ_SWITCH_CASE(3_K)+ MMQ_SWITCH_CASE(4_K)+ MMQ_SWITCH_CASE(5_K)+ MMQ_SWITCH_CASE(6_K)default:GGML_ASSERT(false);break;@@ -8527,6 +9156,26 @@ bool ggml_cuda_supports_mmq(enum ggml_type type) {}}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP mmvq.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP mmvq.cuh+//+////////////////////////////////////////////////////////////////////////////////+++void ggml_cuda_op_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);+typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs);template <int ncols_y, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>@@ -8615,8 +9264,7 @@ static void mul_mat_vec_q_cuda(GGML_ASSERT(ncols_x % qk == 0);GGML_ASSERT(ncols_y <= MMVQ_MAX_BATCH_SIZE);- int id;- CUDA_CHECK(cudaGetDevice(&id));+ int id = ggml_cuda_get_device();int64_t nwarps = 1;int64_t rows_per_cuda_block = 1;@@ -8854,8 +9502,7 @@ void ggml_cuda_op_mul_mat_vec_q(const int64_t ne0 = dst->ne[0];- int id;- CUDA_CHECK(cudaGetDevice(&id));+ int id = ggml_cuda_get_device();// the main device has a larger memory buffer to hold the results from all GPUs// nrows_dst == nrows of the matrix that the kernel writes into@@ -8931,6 +9578,26 @@ void ggml_cuda_op_mul_mat_vec_q(GGML_UNUSED(src1_padded_row_size);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP norm.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP norm.cuh+//+////////////////////////////////////////////////////////////////////////////////+++void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template <int block_size>static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {const int row = blockIdx.x*blockDim.y + threadIdx.y;@@ -9145,6 +9812,24 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {rms_norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP pad.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP pad.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_PAD_BLOCK_SIZE 256++void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) {// blockIdx.z: idx of ne2*ne3, aka ne02*ne03// blockIdx.y: idx of ne1@@ -9193,6 +9878,24 @@ void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], stream);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP pool2d.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP pool2d.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_POOL2D_BLOCK_SIZE 256++void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template <typename Ti, typename To>static __global__ void pool2d_nchw_kernel(const int ih, const int iw, const int oh, const int ow,@@ -9286,6 +9989,24 @@ void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {pool2d_nchw_kernel_f32_f32_cuda(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_d, dst_d, op, stream);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP quantize.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP quantize.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_QUANTIZE_BLOCK_SIZE 256++void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream);+static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int64_t kx, const int64_t kx_padded) {const int64_t ix = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;@@ -9330,6 +10051,24 @@ void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP rope.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP rope.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_ROPE_BLOCK_SIZE 256++void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+struct rope_corr_dims {float v[4];};@@ -9637,6 +10376,24 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {}}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP scale.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP scale.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_SCALE_BLOCK_SIZE 256++void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __global__ void scale_f32(const float * x, float * dst, const float scale, const int k) {const int i = blockDim.x*blockIdx.x + threadIdx.x;@@ -9665,9 +10422,26 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {memcpy(&scale, dst->op_params, sizeof(float));scale_f32_cuda(src0_d, dst_d, scale, ggml_nelements(src0), stream);- CUDA_CHECK(cudaGetLastError());}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP softmax.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP softmax.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_SOFT_MAX_BLOCK_SIZE 1024++void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template <typename T>static __device__ __forceinline__ float t2f32(T val) {return (float) val;@@ -9679,7 +10453,7 @@ __device__ float __forceinline__ t2f32<half>(half val) {}template <bool vals_smem, int ncols_template, int block_size_template, typename T>-static __global__ void soft_max_f32(const float * x, const T * mask, const T * pos, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {+static __global__ void soft_max_f32(const float * x, const T * mask, float * dst, const int ncols_par, const int nrows_y, const float scale, const float max_bias, const float m0, const float m1, uint32_t n_head_log2) {const int ncols = ncols_template == 0 ? ncols_par : ncols_template;const int tid = threadIdx.x;@@ -9691,17 +10465,7 @@ static __global__ void soft_max_f32(const float * x, const T * mask, const T * pconst int warp_id = threadIdx.x / WARP_SIZE;const int lane_id = threadIdx.x % WARP_SIZE;- float slope = 0.0f;-- // ALiBi- if (max_bias > 0.0f) {- const int h = rowx/nrows_y; // head index-- const float base = h < n_head_log2 ? m0 : m1;- const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;-- slope = powf(base, exp);- }+ const float slope = get_alibi_slope(max_bias, rowx/nrows_y, n_head_log2, m0, m1);extern __shared__ float data_soft_max_f32[];float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication@@ -9721,7 +10485,7 @@ static __global__ void soft_max_f32(const float * x, const T * mask, const T * pconst int64_t ix = (int64_t)rowx*ncols + col;const int64_t iy = (int64_t)rowy*ncols + col;- const float val = x[ix]*scale + (mask ? t2f32(mask[iy]) : 0.0f) + (pos ? slope*t2f32(pos[col]) : 0.0f);+ const float val = x[ix]*scale + (mask ? slope*t2f32(mask[iy]) : 0.0f);vals[col] = val;max_val = max(max_val, val);@@ -9793,7 +10557,7 @@ static __global__ void soft_max_f32(const float * x, const T * mask, const T * p}template<typename T>-static void soft_max_f32_cuda(const float * x, const T * mask, const T * pos, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {+static void soft_max_f32_cuda(const float * x, const T * mask, float * dst, const int ncols_x, const int nrows_x, const int nrows_y, const float scale, const float max_bias, cudaStream_t stream) {int nth = WARP_SIZE;while (nth < ncols_x && nth < CUDA_SOFT_MAX_BLOCK_SIZE) nth *= 2;const dim3 block_dims(nth, 1, 1);@@ -9801,8 +10565,8 @@ static void soft_max_f32_cuda(const float * x, const T * mask, const T * pos, flconst size_t shmem = (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE)*sizeof(float);static_assert(CUDA_SOFT_MAX_BLOCK_SIZE == 1024, "These values need to be adjusted.");- const uint32_t n_head_kv = nrows_x/nrows_y;- const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));+ const uint32_t n_head = nrows_x/nrows_y;+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);@@ -9810,43 +10574,42 @@ static void soft_max_f32_cuda(const float * x, const T * mask, const T * pos, flif (shmem < ggml_cuda_info().devices[ggml_cuda_get_device()].smpb) {switch (ncols_x) {case 32:- soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 32, 32><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 64:- soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 64, 64><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 128:- soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 128, 128><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 256:- soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 256, 256><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 512:- soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 512, 512><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 1024:- soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 1024, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 2048:- soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 2048, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;case 4096:- soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 4096, 1024><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;default:- soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<true, 0, 0><<<block_nums, block_dims, shmem, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);break;}} else {const size_t shmem_low = WARP_SIZE*sizeof(float);- soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, pos, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);+ soft_max_f32<false, 0, 0><<<block_nums, block_dims, shmem_low, stream>>>(x, mask, dst, ncols_x, nrows_y, scale, max_bias, m0, m1, n_head_log2);}}void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];- const ggml_tensor * src2 = dst->src[2];const float * src0_d = (const float *)src0->data;const void * src1_d = src1 ? (const void *)src1->data : nullptr;@@ -9858,7 +10621,6 @@ void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {GGML_ASSERT( dst->type == GGML_TYPE_F32);GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); // src1 contains mask and it is optional- GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32); // src2 contains positions and it is optionalconst int64_t ne00 = src0->ne[0];const int64_t nrows_x = ggml_nrows(src0);@@ -9870,29 +10632,34 @@ void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));- // positions tensor- void * src2_d = nullptr;-- const bool use_src2 = src2 != nullptr;-- if (use_src2) {- src2_d = (void *)src2->data;- }-- const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);if (use_f16) {const half * src1_dd = (const half *)src1_d;- const half * src2_dd = (const half *)src2_d;- soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);+ soft_max_f32_cuda(src0_d, src1_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);} else {const float * src1_dd = (const float *)src1_d;- const float * src2_dd = (const float *)src2_d;- soft_max_f32_cuda(src0_d, src1_dd, src2_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);- }-}+ soft_max_f32_cuda(src0_d, src1_dd, dst_d, ne00, nrows_x, nrows_y, scale, max_bias, stream);+ }+}++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP sumrows.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP sumrows.cuh+//+////////////////////////////////////////////////////////////////////////////////+++void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {const int row = blockIdx.x;@@ -9933,6 +10700,24 @@ void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP tsembd.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP tsembd.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256++void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {// blockIDx.y: idx of timesteps->ne[0]// blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE@@ -9979,6 +10764,49 @@ void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensotimestep_embedding_f32_cuda(src0_d, dst_d, src0->ne[0], dst->nb[1], dim, max_period, stream);}+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP unary.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP unary.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_GELU_BLOCK_SIZE 256+#define CUDA_SILU_BLOCK_SIZE 256+#define CUDA_TANH_BLOCK_SIZE 256+#define CUDA_RELU_BLOCK_SIZE 256+#define CUDA_SIGMOID_BLOCK_SIZE 256+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256+#define CUDA_HARDSWISH_BLOCK_SIZE 256+#define CUDA_SQR_BLOCK_SIZE 256++void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+static __global__ void gelu_f32(const float * x, float * dst, const int k) {const float GELU_COEF_A = 0.044715f;const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;@@ -10027,6 +10855,15 @@ static __global__ void relu_f32(const float * x, float * dst, const int k) {dst[i] = fmaxf(x[i], 0);}+static __global__ void sigmoid_f32(const float * x, float * dst, const int k) {+ const int i = blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k) {+ return;+ }+ dst[i] = 1.0f / (1.0f + expf(-x[i]));+}+static __global__ void hardsigmoid_f32(const float * x, float * dst, const int k) {const int i = blockDim.x*blockIdx.x + threadIdx.x;@@ -10087,6 +10924,11 @@ static void relu_f32_cuda(const float * x, float * dst, const int k, cudaStream_relu_f32<<<num_blocks, CUDA_RELU_BLOCK_SIZE, 0, stream>>>(x, dst, k);}+static void sigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_SIGMOID_BLOCK_SIZE - 1) / CUDA_SIGMOID_BLOCK_SIZE;+ sigmoid_f32<<<num_blocks, CUDA_SIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);+}+static void hardsigmoid_f32_cuda(const float * x, float * dst, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_HARDSIGMOID_BLOCK_SIZE - 1) / CUDA_HARDSIGMOID_BLOCK_SIZE;hardsigmoid_f32<<<num_blocks, CUDA_HARDSIGMOID_BLOCK_SIZE, 0, stream>>>(x, dst, k);@@ -10167,6 +11009,18 @@ void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {relu_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);}+void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const float * src0_d = (const float *)src0->data;+ float * dst_d = (float *)dst->data;+ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ sigmoid_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);+}+void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const float * src0_d = (const float *)src0->data;@@ -10218,36 +11072,55 @@ void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {sqr_f32_cuda(src0_d, dst_d, ggml_nelements(src0), stream);}-static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int ne00xne01, const int scale_factor) {- // blockIdx.z: idx of ne02*ne03- // blockIdx.y: idx of ne01*scale_factor, aka ne1- // blockIDx.x: idx of ne00*scale_factor / BLOCK_SIZE- // ne00xne01: ne00 * ne01- int ne0 = ne00 * scale_factor;- int nidx = threadIdx.x + blockIdx.x * blockDim.x;- if (nidx >= ne0) {+////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP upscale.cu+//+////////////////////////////////////////////////////////////////////////////////+++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP upscale.cuh+//+////////////////////////////////////////////////////////////////////////////////+++#define CUDA_UPSCALE_BLOCK_SIZE 256++void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++static __global__ void upscale_f32(const float * x, float * dst,+ const int nb00, const int nb01, const int nb02, const int nb03,+ const int ne10, const int ne11, const int ne12, const int ne13,+ const float sf0, const float sf1, const float sf2, const float sf3) {+ int index = threadIdx.x + blockIdx.x * blockDim.x;+ if (index >= ne10 * ne11 * ne12 * ne13) {return;}- // operation- int i00 = nidx / scale_factor;- int i01 = blockIdx.y / scale_factor;- int offset_src =- i00 +- i01 * ne00 +- blockIdx.z * ne00xne01;- int offset_dst =- nidx +- blockIdx.y * ne0 +- blockIdx.z * ne0 * gridDim.y;- dst[offset_dst] = x[offset_src];++ int i10 = index % ne10;+ int i11 = (index / ne10) % ne11;+ int i12 = (index / (ne10 * ne11)) % ne12;+ int i13 = (index / (ne10 * ne11 * ne12)) % ne13;++ int i00 = i10 / sf0;+ int i01 = i11 / sf1;+ int i02 = i12 / sf2;+ int i03 = i13 / sf3;++ dst[index] = *(float *)((char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00);}-static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int ne03,- const int scale_factor, cudaStream_t stream) {- int ne0 = (ne00 * scale_factor);- int num_blocks = (ne0 + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;- dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02*ne03);- upscale_f32<<<gridDim, CUDA_UPSCALE_BLOCK_SIZE, 0, stream>>>(x, dst, ne00, ne00 * ne01, scale_factor);+static void upscale_f32_cuda(const float * x, float * dst,+ const int nb00, const int nb01, const int nb02, const int nb03,+ const int ne10, const int ne11, const int ne12, const int ne13,+ const float sf0, const float sf1, const float sf2, const float sf3,+ cudaStream_t stream) {+ int dst_size = ne10 * ne11 * ne12 * ne13;+ int num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE;++ upscale_f32<<<num_blocks, CUDA_UPSCALE_BLOCK_SIZE,0,stream>>>(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3);}void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {@@ -10257,27 +11130,77 @@ void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {cudaStream_t stream = ctx.stream();GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);- GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors+ GGML_ASSERT( dst->type == GGML_TYPE_F32);- const int scale_factor = dst->op_params[0];+ const float sf0 = (float)dst->ne[0]/src0->ne[0];+ const float sf1 = (float)dst->ne[1]/src0->ne[1];+ const float sf2 = (float)dst->ne[2]/src0->ne[2];+ const float sf3 = (float)dst->ne[3]/src0->ne[3];- upscale_f32_cuda(src0_d, dst_d, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], scale_factor, stream);+ upscale_f32_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream);}++////////////////////////////////////////////////////////////////////////////////+//+// ROLLUP ../ggml-cuda.cu+//+////////////////////////////////////////////////////////////////////////////////+#include "ggml-cuda.h"#include "ggml.h"#include "ggml-backend-impl.h"++static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");+static void ggml_cuda_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {+ if (FLAG_log_disable) return; // [jart]+ GGML_UNUSED(level);+ GGML_UNUSED(user_data);+ fprintf(stderr, "%s", msg);+}++ggml_log_callback ggml_cuda_log_callback = ggml_cuda_default_log_callback;+void * ggml_cuda_log_user_data = NULL;++GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data) {+ ggml_cuda_log_callback = log_callback;+ ggml_cuda_log_user_data = user_data;+}++#define GGML_CUDA_LOG_INFO(...) ggml_cuda_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)+#define GGML_CUDA_LOG_WARN(...) ggml_cuda_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)+#define GGML_CUDA_LOG_ERROR(...) ggml_cuda_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)++GGML_ATTRIBUTE_FORMAT(2, 3)+static void ggml_cuda_log(enum ggml_log_level level, const char * format, ...) {+ if (ggml_cuda_log_callback != NULL) {+ va_list args;+ va_start(args, format);+ char buffer[128];+ int len = vsnprintf(buffer, 128, format, args);+ if (len < 128) {+ ggml_cuda_log_callback(level, buffer, ggml_cuda_log_user_data);+ } else {+ std::vector<char> buffer2(len + 1); // vsnprintf adds a null terminator+ va_end(args);+ va_start(args, format);+ vsnprintf(&buffer2[0], buffer2.size(), format, args);+ ggml_cuda_log_callback(level, buffer2.data(), ggml_cuda_log_user_data);+ }+ va_end(args);+ }+}+[[noreturn]]void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {int id = -1; // in case cudaGetDevice failscudaGetDevice(&id);- fprintf(stderr, "CUDA error: %s\n", msg);- fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);- fprintf(stderr, " %s\n", stmt);+ GGML_CUDA_LOG_ERROR("CUDA error: %s\n", msg);+ GGML_CUDA_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func, file, line);+ GGML_CUDA_LOG_ERROR(" %s\n", stmt);// abort with GGML_ASSERT to get a stack traceGGML_ASSERT(!"CUDA error");}@@ -10315,31 +11238,28 @@ static ggml_cuda_device_info ggml_cuda_init() {cudaError_t err = cudaGetDeviceCount(&info.device_count);if (err != cudaSuccess) {- fprintf(stderr, "%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));+ GGML_CUDA_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));return info;}GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES);int64_t total_vram = 0;- if (!FLAG_log_disable)#if defined(GGML_CUDA_FORCE_MMQ)- fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);#else- fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);+ GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);#endif- if (!FLAG_log_disable)#if defined(CUDA_USE_TENSOR_CORES)- fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__);+ GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: yes\n", __func__);#else- fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__);+ GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: no\n", __func__);#endif- if (!FLAG_log_disable)- fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count);+ GGML_CUDA_LOG_INFO("%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count);for (int id = 0; id < info.device_count; ++id) {int device_vmm = 0;-#if !defined(GGML_USE_HIPBLAS)+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)CUdevice device;CU_CHECK(cuDeviceGet(&device, id));CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));@@ -10356,8 +11276,7 @@ static ggml_cuda_device_info ggml_cuda_init() {cudaDeviceProp prop;CUDA_CHECK(cudaGetDeviceProperties(&prop, id));- if (!FLAG_log_disable)- fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");+ GGML_CUDA_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");info.default_tensor_split[id] = total_vram;total_vram += prop.totalGlobalMem;@@ -10463,8 +11382,8 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {*actual_size = look_ahead_size;pool_size += look_ahead_size;#ifdef DEBUG_CUDA_MALLOC- fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,- (uint32_t)(max_size/1024/1024), (uint32_t)(pool_size/1024/1024), (uint32_t)(size/1024/1024));+ GGML_CUDA_LOG_INFO("%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,+ (uint32_t)(max_size / 1024 / 1024), (uint32_t)(pool_size / 1024 / 1024), (uint32_t)(size / 1024 / 1024));#endifreturn ptr;}@@ -10478,7 +11397,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {return;}}- fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");+ GGML_CUDA_LOG_WARN("Cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");ggml_cuda_set_device(device);CUDA_CHECK(cudaFree(ptr));pool_size -= size;@@ -10486,7 +11405,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {};// pool with virtual memory-#if !defined(GGML_USE_HIPBLAS)+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)struct ggml_cuda_pool_vmm : public ggml_cuda_pool {static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB@@ -10583,7 +11502,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {#endif // !defined(GGML_USE_HIPBLAS)std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device) {-#if !defined(GGML_USE_HIPBLAS)+#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)if (ggml_cuda_info().devices[device].vmm) {return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));}@@ -10727,7 +11646,9 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffevoid * dev_ptr;cudaError_t err = cudaMalloc(&dev_ptr, size);if (err != cudaSuccess) {- fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err));+ // clear the error+ cudaGetLastError();+ GGML_CUDA_LOG_ERROR("%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err));return nullptr;}@@ -11230,8 +12151,8 @@ static void * ggml_cuda_host_malloc(size_t size) {if (err != cudaSuccess) {// clear the errorcudaGetLastError();- fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,- size/1024.0/1024.0, cudaGetErrorString(err));+ GGML_CUDA_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,+ size / 1024.0 / 1024.0, cudaGetErrorString(err));return nullptr;}@@ -11874,7 +12795,7 @@ static void ggml_cuda_op_mul_mat(}}-static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){+static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1));GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation@@ -11897,7 +12818,7 @@ static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream);}-static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst){+static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(!ggml_is_transposed(src0));GGML_ASSERT(!ggml_is_transposed(src1));GGML_ASSERT(!ggml_is_permuted(src0));@@ -12432,6 +13353,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_UNARY_OP_RELU:ggml_cuda_op_relu(ctx, dst);break;+ case GGML_UNARY_OP_SIGMOID:+ ggml_cuda_op_sigmoid(ctx, dst);+ break;case GGML_UNARY_OP_HARDSIGMOID:ggml_cuda_op_hardsigmoid(ctx, dst);break;@@ -12471,7 +13395,7 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggbreak;case GGML_OP_MUL_MAT:if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) {- fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]);+ GGML_CUDA_LOG_ERROR("%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]);return false;} else {ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst);@@ -12504,9 +13428,6 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_OP_ROPE:ggml_cuda_op_rope(ctx, dst);break;- case GGML_OP_ALIBI:- ggml_cuda_op_alibi(ctx, dst);- break;case GGML_OP_IM2COL:ggml_cuda_op_im2col(ctx, dst);break;@@ -12519,18 +13440,16 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggcase GGML_OP_ARGSORT:ggml_cuda_op_argsort(ctx, dst);break;-#ifndef GGML_MINIMIZE_CODE_SIZE // [jart]case GGML_OP_FLASH_ATTN_EXT:ggml_cuda_flash_attn_ext(ctx, dst);break;-#endifdefault:return false;}cudaError_t err = cudaGetLastError();if (err != cudaSuccess) {- fprintf(stderr, "%s: %s failed\n", __func__, ggml_op_desc(dst));+ GGML_CUDA_LOG_ERROR("%s: %s failed\n", __func__, ggml_op_desc(dst));CUDA_CHECK(err);}@@ -12639,44 +13558,318 @@ GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {GGML_UNUSED(backend);}+static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {+ graph_node_properties->node_address = node->data;+ graph_node_properties->node_op = node->op;+ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ graph_node_properties->ne[i] = node->ne[i];+ graph_node_properties->nb[i] = node->nb[i];+ }+ for (int i = 0; i < GGML_MAX_SRC; i++) {+ graph_node_properties->src_address[i] = node->src[i] ? node->src[i]->data : nullptr;+ }+}++static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {+ if (node->data != graph_node_properties->node_address &&+ node->op != GGML_OP_CPY &&+ node->op != GGML_OP_VIEW) {+ return false;+ }++ if (node->op != graph_node_properties->node_op) {+ return false;+ }++ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ if (node->ne[i] != graph_node_properties->ne[i]) {+ return false;+ }+ if (node->nb[i] != graph_node_properties->nb[i]) {+ return false;+ }+ }++ for (int i = 0; i < GGML_MAX_SRC; i++) {+ if (node->src[i] &&+ node->src[i]->data != graph_node_properties->src_address[i] &&+ node->op != GGML_OP_CPY &&+ node->op != GGML_OP_VIEW+ ) {+ return false;+ }+ }+ return true;+}+GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;ggml_cuda_set_device(cuda_ctx->device);- for (int i = 0; i < cgraph->n_nodes; i++) {- ggml_tensor * node = cgraph->nodes[i];+#ifdef USE_CUDA_GRAPH+ static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);- if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {- continue;+ // Objects required for CUDA Graph+ if (cuda_ctx->cuda_graph == nullptr) {+ cuda_ctx->cuda_graph.reset(new ggml_cuda_graph());+ }++ bool use_cuda_graph = true;+ bool cuda_graph_update_required = false;+ // pointer to CUDA cpy kernel, which is required to identify+ // kernel parameters which need updated in the graph for each token+ void * ggml_cuda_cpy_fn_ptr = nullptr;++ if (cuda_ctx->cuda_graph->graph == nullptr) {+ if (ggml_cuda_info().devices[cuda_ctx->device].cc < CC_AMPERE) {+ cuda_ctx->cuda_graph->disable_due_to_gpu_arch = true;+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to GPU architecture\n", __func__);+#endif+ }+ }++ // Disable CUDA graphs in presence of env var, old GPU, use-case which is changing too rapidly,+ // or previous graph capture failure.+ // Also disable for multi-gpu for now. TO DO investigate+ if (disable_cuda_graphs_due_to_env+ || cuda_ctx->cuda_graph->disable_due_to_gpu_arch+ || cuda_ctx->cuda_graph->disable_due_to_too_many_updates+ || cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture) {+ use_cuda_graph = false;+ }++ if (use_cuda_graph) {+ if (cuda_ctx->cuda_graph->instance == nullptr) {+ cuda_graph_update_required = true;+ }++ // Check if the graph size has changed+ if (cuda_ctx->cuda_graph->ggml_graph_properties.size() != (size_t)cgraph->n_nodes) {+ cuda_graph_update_required = true;+ cuda_ctx->cuda_graph->ggml_graph_properties.resize(cgraph->n_nodes);+ }++ // Loop over nodes in GGML graph to determine if CUDA graph update is required+ // and store properties to allow this comparison for the next token+ for (int i = 0; i < cgraph->n_nodes; i++) {+ bool has_matching_properties = true;+ if (!cuda_graph_update_required) {+ has_matching_properties = ggml_graph_node_has_matching_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);+ }+ if (!has_matching_properties) {+ cuda_graph_update_required = true;+ }+ set_ggml_graph_node_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);+ }++ // Loop over nodes in GGML graph to obtain info needed for CUDA graph+ cuda_ctx->cuda_graph->updated_kernel_arg.clear();+ for (int i = 0; i < cgraph->n_nodes; i++) {+ ggml_tensor * node = cgraph->nodes[i];++ if (node->src[0] && ggml_backend_buffer_is_cuda_split(node->src[0]->buffer)) {+ use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to split buffer\n", __func__);+#endif+ }++ if (node->op == GGML_OP_MUL_MAT_ID) {+ use_cuda_graph = false; // This node type is not supported by CUDA graph capture+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to mul_mat_id\n", __func__);+#endif+ }++ if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) {+ // disable CUDA graphs for batch size > 1 for now.+ // Changes in batch size or context size can cause changes to the grid size of some kernels.+ use_cuda_graph = false;+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);+#endif+ }++ if (node->op == GGML_OP_CPY) {+ // store the copy op parameter which changes with each token.+ cuda_ctx->cuda_graph->updated_kernel_arg.push_back((char **) &(node->src[1]->data));+ if (ggml_cuda_cpy_fn_ptr == nullptr) {+ // store a pointer to the copy op CUDA kernel to identify it later+ ggml_cuda_cpy_fn_ptr = ggml_cuda_cpy_fn(node->src[0], node->src[1]);+ }+ }++ if (!use_cuda_graph) {+ break;+ }+ }++ // Disable CUDA graphs (from the next token) if the use-case is demanding too many consecutive graph updates.+ if (use_cuda_graph && cuda_graph_update_required) {+ cuda_ctx->cuda_graph->number_consecutive_updates++;+ } else {+ cuda_ctx->cuda_graph->number_consecutive_updates = 0;}+ if (cuda_ctx->cuda_graph->number_consecutive_updates >= 4) {+ cuda_ctx->cuda_graph->disable_due_to_too_many_updates = true;+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__);+#endif+ }+ }++ if (use_cuda_graph && cuda_graph_update_required) { // Start CUDA graph capture+ CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed));+ }++#else+ bool use_cuda_graph = false;+ bool cuda_graph_update_required = false;+#endif // USE_CUDA_GRAPH++ bool graph_evaluated_or_captured = false;++ while (!graph_evaluated_or_captured) {+ // Only perform the graph execution if CUDA graphs are not enabled, or we are capturing the graph.+ // With the use of CUDA graphs, the execution will be performed by the graph launch.+ if (!use_cuda_graph || cuda_graph_update_required) {+ for (int i = 0; i < cgraph->n_nodes; i++) {+ ggml_tensor * node = cgraph->nodes[i];++ if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {+ continue;+ }+#ifndef NDEBUG- assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));- for (int j = 0; j < GGML_MAX_SRC; j++) {- if (node->src[j] != nullptr) {- assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));+ assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ if (node->src[j] != nullptr) {+ assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));+ }+ }+#endif++ bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);+ if (!ok) {+ GGML_CUDA_LOG_ERROR("%s: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));+ }+ GGML_ASSERT(ok);}}++#ifdef USE_CUDA_GRAPH+ if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture+ if (cuda_ctx->cuda_graph->graph != nullptr) {+ CUDA_CHECK(cudaGraphDestroy(cuda_ctx->cuda_graph->graph));+ cuda_ctx->cuda_graph->graph = nullptr;+ }+ CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_ctx->cuda_graph->graph));++#if 0+ if (disable_cuda_graphs_due_to_failed_capture) {+ use_cuda_graph = false;+ cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture = true;+#ifndef NDEBUG+ GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to failed graph capture\n", __func__);+#endif+ } else {+ graph_evaluated_or_captured = true; // CUDA graph has been captured+ }#endif+ graph_evaluated_or_captured = true; // CUDA graph has been captured+ } else {+ graph_evaluated_or_captured = true; // ggml graph has been directly evaluated+ }+ }++ if (use_cuda_graph) {+ if (cuda_ctx->cuda_graph->instance == nullptr) { // Create executable graph from captured graph.+ CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));+ }++ // Perform update to graph (if required for this token), and change copy parameter (required for every token)++ if (cuda_graph_update_required) {+ // Extract nodes from graph+ if (cuda_ctx->cuda_graph->num_nodes == 0) {+ // First call with null argument gets number of nodes in graph+ CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, nullptr, &cuda_ctx->cuda_graph->num_nodes));+ }+ // Subsequent call with non-null argument gets nodes+ cuda_ctx->cuda_graph->nodes.resize(cuda_ctx->cuda_graph->num_nodes);+ cuda_ctx->cuda_graph->params.resize(cuda_ctx->cuda_graph->num_nodes);+ if (cuda_ctx->cuda_graph->num_nodes > 0) {+ CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, cuda_ctx->cuda_graph->nodes.data(), &cuda_ctx->cuda_graph->num_nodes));++ // Loop over nodes, and extract kernel parameters from each node+ for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {+ cudaGraphNodeType node_type;+ CUDA_CHECK(cudaGraphNodeGetType(cuda_ctx->cuda_graph->nodes[i], &node_type));+ if (node_type == cudaGraphNodeTypeKernel) {+ cudaError_t stat = cudaGraphKernelNodeGetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]); // Get params using runtime+ if (stat == cudaErrorInvalidDeviceFunction) {+ // Fails due to incorrect handling by CUDA runtime of CUDA BLAS node.+ // We don't need to update blas nodes, so clear error and move on.+ cudaGetLastError();+ } else {+ GGML_ASSERT(stat == cudaSuccess);+ }+ }+ }+ }+ }- bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);- if (!ok) {- fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));+ // One of the arguments to the copy kernel is updated for each token, hence we need to+ // replace that argument with the updated value in the CUDA graph+ if (!cuda_graph_update_required) { // on update steps, the live parameters will already be captured+ int k = 0;+ for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {+ if (cuda_ctx->cuda_graph->params[i].func == ggml_cuda_cpy_fn_ptr) {+ char ** updated_kernel_arg_ptr = cuda_ctx->cuda_graph->updated_kernel_arg.at(k++);+ cuda_ctx->cuda_graph->params[i].kernelParams[1] = updated_kernel_arg_ptr;+ CUDA_CHECK(cudaGraphKernelNodeSetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]));+ }+ }+ }++ // Update graph executable+ cudaGraphExecUpdateResultInfo result_info;+ cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info);+ if (stat == cudaErrorGraphExecUpdateFailure) {+#ifndef NDEBUG+ GGML_CUDA_LOG_ERROR("%s: CUDA graph update failed\n", __func__);+#endif+ // The pre-existing graph exec cannot be updated due to violated constraints+ // so instead clear error and re-instantiate+ cudaGetLastError();+ CUDA_CHECK(cudaGraphExecDestroy(cuda_ctx->cuda_graph->instance));+ cuda_ctx->cuda_graph->instance = nullptr;+ CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));+ } else {+ GGML_ASSERT(stat == cudaSuccess);}- GGML_ASSERT(ok);+ // Launch graph+ CUDA_CHECK(cudaGraphLaunch(cuda_ctx->cuda_graph->instance, cuda_ctx->stream()));+#else+ graph_evaluated_or_captured = true;+#endif // USE_CUDA_GRAPH}return GGML_STATUS_SUCCESS;}GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;switch (op->op) {case GGML_OP_UNARY:switch (ggml_get_unary_op(op)) {case GGML_UNARY_OP_GELU:case GGML_UNARY_OP_SILU:case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_SIGMOID:case GGML_UNARY_OP_HARDSIGMOID:case GGML_UNARY_OP_HARDSWISH:case GGML_UNARY_OP_GELU_QUICK:@@ -12786,7 +13979,6 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_OP_DIAG_MASK_INF:case GGML_OP_SOFT_MAX:case GGML_OP_ROPE:- case GGML_OP_ALIBI:case GGML_OP_IM2COL:case GGML_OP_POOL_2D:case GGML_OP_SUM_ROWS:@@ -12798,10 +13990,16 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, conscase GGML_OP_ARANGE:case GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_LEAKY_RELU:-#ifndef GGML_MINIMIZE_CODE_SIZE // [jart]- case GGML_OP_FLASH_ATTN_EXT:-#endifreturn true;+ case GGML_OP_FLASH_ATTN_EXT:+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ return op->src[0]->ne[0] == 64 || op->src[0]->ne[0] == 128;+#else+ if (op->src[0]->ne[0] == 64 || op->src[0]->ne[0] == 128) {+ return true;+ }+ return ggml_cuda_info().devices[cuda_ctx->device].cc >= CC_VOLTA;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)default:return false;}@@ -12899,13 +14097,13 @@ GGML_CALL static ggml_guid_t ggml_backend_cuda_guid() {GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {if (device < 0 || device >= ggml_backend_cuda_get_device_count()) {- fprintf(stderr, "%s: error: invalid device %d\n", __func__, device);+ GGML_CUDA_LOG_ERROR("%s: invalid device %d\n", __func__, device);return nullptr;}ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device);if (ctx == nullptr) {- fprintf(stderr, "%s: error: failed to allocate context\n", __func__);+ GGML_CUDA_LOG_ERROR("%s: failed to allocate context\n", __func__);return nullptr;}@@ -12949,8 +14147,8 @@ GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size// clear the errorcudaGetLastError();- fprintf(stderr, "%s: warning: failed to register %.2f MiB of pinned memory: %s\n", __func__,- size/1024.0/1024.0, cudaGetErrorString(err));+ GGML_CUDA_LOG_WARN("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__,+ size / 1024.0 / 1024.0, cudaGetErrorString(err));return false;}return true;diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex c8f94ea8..90149072 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -40,6 +40,7 @@ GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);+GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data);#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex efa1762a..b6af39ea 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -19,6 +19,18 @@#define MIN(a, b) ((a) < (b) ? (a) : (b))#define MAX(a, b) ((a) > (b) ? (a) : (b))+#if defined(_WIN32)++#define m512bh(p) p+#define m512i(p) p++#else++#define m512bh(p) (__m512bh)(p)+#define m512i(p) (__m512i)(p)++#endif+/*** Converts brain16 to float32.*@@ -122,9 +134,16 @@ extern "C" {#ifndef __F16C__#define __F16C__#endif+#endif++// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available+#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))#ifndef __SSE3__#define __SSE3__#endif+#ifndef __SSSE3__+#define __SSSE3__+#endif#endif// 16-bit float@@ -438,6 +457,34 @@ static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {#include <riscv_vector.h>#endif+#if defined(__loongarch64)+#if defined(__loongarch_asx)+#include <lasxintrin.h>+#endif+#if defined(__loongarch_sx)+#include <lsxintrin.h>+#endif+#endif++#if defined(__loongarch_asx)++typedef union {+ int32_t i;+ float f;+} ft_union;++/* float type data load instructions */+static __m128 __lsx_vreplfr2vr_s(float val) {+ ft_union fi_tmpval = {.f = val};+ return (__m128)__lsx_vreplgr2vr_w(fi_tmpval.i);+}++static __m256 __lasx_xvreplfr2vr_s(float val) {+ ft_union fi_tmpval = {.f = val};+ return (__m256)__lasx_xvreplgr2vr_w(fi_tmpval.i);+}+#endif+#ifdef __F16C__#ifdef _MSC_VERdiff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex d32007a7..c519a797 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -68,6 +68,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_CLAMP,GGML_METAL_KERNEL_TYPE_TANH,GGML_METAL_KERNEL_TYPE_RELU,+ GGML_METAL_KERNEL_TYPE_SIGMOID,GGML_METAL_KERNEL_TYPE_GELU,GGML_METAL_KERNEL_TYPE_GELU_4,GGML_METAL_KERNEL_TYPE_GELU_QUICK,@@ -197,7 +198,6 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32,GGML_METAL_KERNEL_TYPE_ROPE_F32,GGML_METAL_KERNEL_TYPE_ROPE_F16,- GGML_METAL_KERNEL_TYPE_ALIBI_F32,GGML_METAL_KERNEL_TYPE_IM2COL_F16,GGML_METAL_KERNEL_TYPE_IM2COL_F32,GGML_METAL_KERNEL_TYPE_UPSCALE_F32,@@ -295,11 +295,20 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){static void * ggml_metal_host_malloc(size_t n) {void * data = NULL;++#if TARGET_OS_OSX+ kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);+ if (err != KERN_SUCCESS) {+ GGML_METAL_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);+ return NULL;+ }+#elseconst int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);if (result != 0) {GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);return NULL;}+#endifreturn data;}@@ -515,6 +524,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIGMOID, sigmoid, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);@@ -644,7 +654,6 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);@@ -654,14 +663,14 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, true);- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, ctx->support_simdgroup_reduction);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);@@ -753,6 +762,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constswitch (ggml_get_unary_op(op)) {case GGML_UNARY_OP_TANH:case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_SIGMOID:case GGML_UNARY_OP_GELU:case GGML_UNARY_OP_GELU_QUICK:case GGML_UNARY_OP_SILU:@@ -780,7 +790,6 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_OP_GROUP_NORM:return ctx->support_simdgroup_reduction;case GGML_OP_NORM:- case GGML_OP_ALIBI:case GGML_OP_ROPE:case GGML_OP_IM2COL:return true;@@ -793,8 +802,9 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_OP_TIMESTEP_EMBEDDING:case GGML_OP_ARGSORT:case GGML_OP_LEAKY_RELU:- case GGML_OP_FLASH_ATTN_EXT:return true;+ case GGML_OP_FLASH_ATTN_EXT:+ return ctx->support_simdgroup_mm; // TODO: over-restricted for vec-kernelscase GGML_OP_MUL_MAT:case GGML_OP_MUL_MAT_ID:return ctx->support_simdgroup_reduction &&@@ -833,7 +843,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, constcase GGML_OP_DIAG_MASK_INF:case GGML_OP_GET_ROWS:{- return op->ne[3] == 1;+ return op->src[0]->type != GGML_TYPE_BF16 && op->ne[3] == 1;}default:return false;@@ -947,22 +957,32 @@ static enum ggml_status ggml_metal_graph_compute(const int64_t ne10 = src1 ? src1->ne[0] : 0;const int64_t ne11 = src1 ? src1->ne[1] : 0;const int64_t ne12 = src1 ? src1->ne[2] : 0;- const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);+ const int64_t ne13 = src1 ? src1->ne[3] : 0;const uint64_t nb10 = src1 ? src1->nb[0] : 0;const uint64_t nb11 = src1 ? src1->nb[1] : 0;const uint64_t nb12 = src1 ? src1->nb[2] : 0;- const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13);+ const uint64_t nb13 = src1 ? src1->nb[3] : 0;++ const int64_t ne20 = src2 ? src2->ne[0] : 0;+ const int64_t ne21 = src2 ? src2->ne[1] : 0;+ const int64_t ne22 = src2 ? src2->ne[2] : 0; GGML_UNUSED(ne22);+ const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);++ const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);+ const uint64_t nb21 = src2 ? src2->nb[1] : 0;+ const uint64_t nb22 = src2 ? src2->nb[2] : 0;+ const uint64_t nb23 = src2 ? src2->nb[3] : 0;- const int64_t ne0 = dst ? dst->ne[0] : 0;- const int64_t ne1 = dst ? dst->ne[1] : 0;- const int64_t ne2 = dst ? dst->ne[2] : 0;- const int64_t ne3 = dst ? dst->ne[3] : 0;+ const int64_t ne0 = dst ? dst->ne[0] : 0;+ const int64_t ne1 = dst ? dst->ne[1] : 0;+ const int64_t ne2 = dst ? dst->ne[2] : 0;+ const int64_t ne3 = dst ? dst->ne[3] : 0;- const uint64_t nb0 = dst ? dst->nb[0] : 0;- const uint64_t nb1 = dst ? dst->nb[1] : 0;- const uint64_t nb2 = dst ? dst->nb[2] : 0;- const uint64_t nb3 = dst ? dst->nb[3] : 0;+ const uint64_t nb0 = dst ? dst->nb[0] : 0;+ const uint64_t nb1 = dst ? dst->nb[1] : 0;+ const uint64_t nb2 = dst ? dst->nb[2] : 0;+ const uint64_t nb3 = dst ? dst->nb[3] : 0;const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;@@ -1215,24 +1235,24 @@ static enum ggml_status ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} break;case GGML_OP_CLAMP:- {- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CLAMP].pipeline;+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CLAMP].pipeline;- float min;- float max;- memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));- memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));+ float min;+ float max;+ memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));+ memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));- [encoder setComputePipelineState:pipeline];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&min length:sizeof(min) atIndex:2];- [encoder setBytes:&max length:sizeof(max) atIndex:3];+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&min length:sizeof(min) atIndex:2];+ [encoder setBytes:&max length:sizeof(max) atIndex:3];- const int64_t n = ggml_nelements(dst);+ const int64_t n = ggml_nelements(dst);- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;case GGML_OP_UNARY:switch (ggml_get_unary_op(gf->nodes[i])) {// we are not taking into account the strides, so for now require contiguous tensors@@ -1260,6 +1280,18 @@ static enum ggml_status ggml_metal_graph_compute(const int64_t n = ggml_nelements(dst);+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_UNARY_OP_SIGMOID:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SIGMOID].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];++ const int64_t n = ggml_nelements(dst);+[encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];} break;case GGML_UNARY_OP_GELU:@@ -1378,16 +1410,15 @@ static enum ggml_status ggml_metal_graph_compute(case GGML_OP_SOFT_MAX:{GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);- GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F16 || src2->type == GGML_TYPE_F32);int nth = 32; // SIMD widthid<MTLComputePipelineState> pipeline = nil;- const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);if (ne00%4 == 0) {- while (nth < ne00/4 && nth < 256) {+ while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) {nth *= 2;}if (use_f16) {@@ -1396,7 +1427,7 @@ static enum ggml_status ggml_metal_graph_compute(pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;}} else {- while (nth < ne00 && nth < 1024) {+ while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {nth *= 2;}if (use_f16) {@@ -1415,8 +1446,8 @@ static enum ggml_status ggml_metal_graph_compute(const int64_t nrows_x = ggml_nrows(src0);const int64_t nrows_y = src0->ne[1];- const uint32_t n_head_kv = nrows_x/nrows_y;- const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv));+ const uint32_t n_head = nrows_x/nrows_y;+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);@@ -1428,20 +1459,15 @@ static enum ggml_status ggml_metal_graph_compute(} else {[encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];}- if (id_src2) {- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];- } else {- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2];- }- [encoder setBuffer:id_dst offset:offs_dst atIndex:3];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:4];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:5];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:6];- [encoder setBytes:&scale length:sizeof(scale) atIndex:7];- [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:8];- [encoder setBytes:&m0 length:sizeof(m0) atIndex:9];- [encoder setBytes:&m1 length:sizeof(m1) atIndex:10];- [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:11];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&scale length:sizeof(scale) atIndex:6];+ [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:7];+ [encoder setBytes:&m0 length:sizeof(m0) atIndex:8];+ [encoder setBytes:&m1 length:sizeof(m1) atIndex:9];+ [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:10];[encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];[encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];@@ -1799,16 +1825,6 @@ static enum ggml_status ggml_metal_graph_compute(const int n_as = src0->ne[2];// src2 = ids- const int64_t ne20 = src2->ne[0];- const int64_t ne21 = src2->ne[1];- const int64_t ne22 = src2->ne[2]; GGML_UNUSED(ne22);- const int64_t ne23 = src2->ne[3]; GGML_UNUSED(ne23);-- const uint64_t nb20 = src2->nb[0]; GGML_UNUSED(nb20);- const uint64_t nb21 = src2->nb[1];- const uint64_t nb22 = src2->nb[2]; GGML_UNUSED(nb22);- const uint64_t nb23 = src2->nb[3]; GGML_UNUSED(nb23);-const enum ggml_type src2t = src2->type; GGML_UNUSED(src2t);GGML_ASSERT(src2t == GGML_TYPE_I32);@@ -2246,49 +2262,6 @@ static enum ggml_status ggml_metal_graph_compute([encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;- case GGML_OP_ALIBI:- {- GGML_ASSERT((src0t == GGML_TYPE_F32));-- const int nth = MIN(1024, ne00);-- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];-- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));-- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);-- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline;-- [encoder setComputePipelineState:pipeline];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];- [encoder setBytes:&m0 length:sizeof( float) atIndex:18];- [encoder setBytes:&m1 length:sizeof( float) atIndex:19];- [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20];-- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;case GGML_OP_ROPE:{GGML_ASSERT(ne10 == ne02);@@ -2301,7 +2274,13 @@ static enum ggml_status ggml_metal_graph_compute(// skip 3, n_ctx, used in GLM RoPE, unimplemented in metalconst int n_orig_ctx = ((int32_t *) dst->op_params)[4];- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;+ float freq_base;+ float freq_scale;+ float ext_factor;+ float attn_factor;+ float beta_fast;+ float beta_slow;+memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));@@ -2309,6 +2288,15 @@ static enum ggml_status ggml_metal_graph_compute(memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));+ const bool is_neox = mode & 2;+ const bool is_glm = mode & 4;++ GGML_ASSERT(!is_glm && "GLM RoPE not implemented in Metal");++ if (!is_neox) {+ GGML_ASSERT(id_src2 == nil && "TODO: freq_factors not implemented for !is_neox");+ }+id<MTLComputePipelineState> pipeline = nil;switch (src0->type) {@@ -2320,33 +2308,38 @@ static enum ggml_status ggml_metal_graph_compute([encoder setComputePipelineState:pipeline];[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];- [encoder setBytes:&n_past length:sizeof( int) atIndex:19];- [encoder setBytes:&n_dims length:sizeof( int) atIndex:20];- [encoder setBytes:&mode length:sizeof( int) atIndex:21];- [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22];- [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];- [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];- [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];- [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];- [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];- [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];+ if (id_src2 != nil) {+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];+ } else {+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2];+ }+ [encoder setBuffer:id_dst offset:offs_dst atIndex:3];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:10];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:11];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:14];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:15];+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:17];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:18];+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:19];+ [encoder setBytes:&n_past length:sizeof( int) atIndex:20];+ [encoder setBytes:&n_dims length:sizeof( int) atIndex:21];+ [encoder setBytes:&mode length:sizeof( int) atIndex:22];+ [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:23];+ [encoder setBytes:&freq_base length:sizeof( float) atIndex:24];+ [encoder setBytes:&freq_scale length:sizeof( float) atIndex:25];+ [encoder setBytes:&ext_factor length:sizeof( float) atIndex:26];+ [encoder setBytes:&attn_factor length:sizeof( float) atIndex:27];+ [encoder setBytes:&beta_fast length:sizeof( float) atIndex:28];+ [encoder setBytes:&beta_slow length:sizeof( float) atIndex:29];[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;@@ -2410,7 +2403,10 @@ static enum ggml_status ggml_metal_graph_compute({GGML_ASSERT(src0->type == GGML_TYPE_F32);- const int sf = dst->op_params[0];+ const float sf0 = (float)ne0/src0->ne[0];+ const float sf1 = (float)ne1/src0->ne[1];+ const float sf2 = (float)ne2/src0->ne[2];+ const float sf3 = (float)ne3/src0->ne[3];const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline;@@ -2433,7 +2429,10 @@ static enum ggml_status ggml_metal_graph_compute([encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];[encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];[encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];- [encoder setBytes:&sf length:sizeof(sf) atIndex:18];+ [encoder setBytes:&sf0 length:sizeof(sf0) atIndex:18];+ [encoder setBytes:&sf1 length:sizeof(sf1) atIndex:19];+ [encoder setBytes:&sf2 length:sizeof(sf2) atIndex:20];+ [encoder setBytes:&sf3 length:sizeof(sf3) atIndex:21];const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);@@ -2569,13 +2568,14 @@ static enum ggml_status ggml_metal_graph_compute(} break;case GGML_OP_FLASH_ATTN_EXT:{- GGML_ASSERT(ne00 % 4 == 0);+ GGML_ASSERT(ne00 % 4 == 0);+ GGML_ASSERT(ne11 % 32 == 0);+GGML_ASSERT(src0->type == GGML_TYPE_F32);- struct ggml_tensor * src3 = gf->nodes[i]->src[3];+ GGML_ASSERT(ggml_are_same_shape (src1, src2));- GGML_ASSERT(ggml_are_same_shape(src1, src2));- GGML_ASSERT(src3);+ struct ggml_tensor * src3 = gf->nodes[i]->src[3];size_t offs_src3 = 0;@@ -2586,7 +2586,7 @@ static enum ggml_status ggml_metal_graph_compute("the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big");const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30);- const int64_t ne31 = src3 ? src3->ne[1] : 0;+ //const int64_t ne31 = src3 ? src3->ne[1] : 0;const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32);const int64_t ne33 = src3 ? src3->ne[3] : 0; GGML_UNUSED(ne33);@@ -2598,7 +2598,16 @@ static enum ggml_status ggml_metal_graph_compute(const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);float scale;- memcpy(&scale, dst->op_params, sizeof(float));+ float max_bias;++ memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));+ memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));++ const uint32_t n_head = src0->ne[2];+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);id<MTLComputePipelineState> pipeline = nil;@@ -2635,34 +2644,38 @@ static enum ggml_status ggml_metal_graph_compute(}[encoder setComputePipelineState:pipeline];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];- [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];- [encoder setBuffer:id_dst offset:offs_dst atIndex:4];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:6];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:7];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:8];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:10];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:11];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:12];- [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:14];- [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:15];- [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:16];- [encoder setBytes:&nb10 length:sizeof(uint64_t) atIndex:17];- [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:18];- [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:19];- [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:20];- [encoder setBytes:&ne31 length:sizeof( int64_t) atIndex:21];- [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:22];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:23];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:24];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:25];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:26];- [encoder setBytes:&scale length:sizeof( float) atIndex:27];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];+ if (id_src3) {+ [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];+ } else {+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:3];+ }+ [encoder setBuffer:id_dst offset:offs_dst atIndex:4];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];+ [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:11];+ [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:14];+ [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:15];+ [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb21 length:sizeof(uint64_t) atIndex:17];+ [encoder setBytes:&nb22 length:sizeof(uint64_t) atIndex:18];+ [encoder setBytes:&nb23 length:sizeof(uint64_t) atIndex:19];+ [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:20];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:21];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:22];+ [encoder setBytes:&scale length:sizeof( float) atIndex:23];+ [encoder setBytes:&max_bias length:sizeof( float) atIndex:24];+ [encoder setBytes:&m0 length:sizeof(m0) atIndex:25];+ [encoder setBytes:&m1 length:sizeof(m1) atIndex:26];+ [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:27];if (!use_vec_kernel) {// half8x8 kernel@@ -2870,7 +2883,11 @@ GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_ggml_backend_metal_free_device();if (ctx->owned) {+#if TARGET_OS_OSX+ vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ctx->all_data, ctx->all_size);+#elsefree(ctx->all_data);+#endif}free(ctx);@@ -2974,14 +2991,16 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffctx->owned = true;ctx->n_buffers = 1;- ctx->buffers[0].data = ctx->all_data;- ctx->buffers[0].size = size;- ctx->buffers[0].metal = [device newBufferWithBytesNoCopy:ctx->all_data- length:size_aligned- options:MTLResourceStorageModeShared- deallocator:nil];+ if (ctx->all_data != NULL) {+ ctx->buffers[0].data = ctx->all_data;+ ctx->buffers[0].size = size;+ ctx->buffers[0].metal = [device newBufferWithBytesNoCopy:ctx->all_data+ length:size_aligned+ options:MTLResourceStorageModeShared+ deallocator:nil];+ }- if (ctx->buffers[0].metal == nil) {+ if (ctx->all_data == NULL || ctx->buffers[0].metal == nil) {GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);free(ctx);ggml_backend_metal_free_device();diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex 382e4278..a124962c 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -2083,6 +2083,13 @@ kernel void kernel_relu(dst[tpig] = max(0.0f, src0[tpig]);}+kernel void kernel_sigmoid(+ device const float * src0,+ device float * dst,+ uint tpig[[thread_position_in_grid]]) {+ dst[tpig] = 1.0f / (1.0f + exp(-src0[tpig]));+}+kernel void kernel_tanh(device const float * src0,device float * dst,@@ -2210,7 +2217,6 @@ template<typename T>kernel void kernel_soft_max(device const char * src0,device const char * src1,- device const char * src2,device char * dst,constant int64_t & ne00,constant int64_t & ne01,@@ -2232,10 +2238,9 @@ kernel void kernel_soft_max(device const float * psrc0 = (device const float *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00 : nullptr;- device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;device float * pdst = (device float *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);- float slope = 0.0f;+ float slope = 1.0f;// ALiBiif (max_bias > 0.0f) {@@ -2251,7 +2256,7 @@ kernel void kernel_soft_max(float lmax = -INFINITY;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f));+ lmax = MAX(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));}// find the max value in the block@@ -2276,7 +2281,7 @@ kernel void kernel_soft_max(// parallel sumfloat lsum = 0.0f;for (int i00 = tpitg; i00 < ne00; i00 += ntg) {- const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)) - max_val);+ const float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val);lsum += exp_psrc0;pdst[i00] = exp_psrc0;}@@ -2315,7 +2320,6 @@ template<typename T>kernel void kernel_soft_max_4(device const char * src0,device const char * src1,- device const char * src2,device char * dst,constant int64_t & ne00,constant int64_t & ne01,@@ -2337,10 +2341,9 @@ kernel void kernel_soft_max_4(device const float4 * psrc4 = (device const float4 *) src0 + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;device const T * pmask = src1 != src0 ? (device const T *) src1 + i01*ne00/4 : nullptr;- device const T * ppos = src2 != src0 ? (device const T *) src2 : nullptr;device float4 * pdst4 = (device float4 *) dst + (i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00)/4;- float slope = 0.0f;+ float slope = 1.0f;if (max_bias > 0.0f) {const int64_t h = i02;@@ -2355,7 +2358,7 @@ kernel void kernel_soft_max_4(float4 lmax4 = -INFINITY;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f)));+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f)));}const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));@@ -2381,7 +2384,7 @@ kernel void kernel_soft_max_4(// parallel sumfloat4 lsum4 = 0.0f;for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {- const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? pmask[i00] : 0.0f) + (ppos ? slope*ppos[i00] : 0.0f))) - max_val);+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val);lsum4 += exp_psrc4;pdst4[i00] = exp_psrc4;}@@ -3449,60 +3452,6 @@ kernel void kernel_mul_mv_f16_f32_l4(}}-kernel void kernel_alibi_f32(- device const float * src0,- device float * dst,- constant int64_t & ne00,- constant int64_t & ne01,- constant int64_t & ne02,- constant int64_t & ne03,- constant uint64_t & nb00,- constant uint64_t & nb01,- constant uint64_t & nb02,- constant uint64_t & nb03,- constant int64_t & ne0,- constant int64_t & ne1,- constant int64_t & ne2,- constant int64_t & ne3,- constant uint64_t & nb0,- constant uint64_t & nb1,- constant uint64_t & nb2,- constant uint64_t & nb3,- constant float & m0,- constant float & m1,- constant int & n_heads_log2_floor,- uint3 tgpig[[threadgroup_position_in_grid]],- uint3 tpitg[[thread_position_in_threadgroup]],- uint3 ntg[[threads_per_threadgroup]]) {- const int64_t i03 = tgpig[2];- const int64_t i02 = tgpig[1];- const int64_t i01 = tgpig[0];-- const int64_t n = i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;-- const int64_t i3 = n / (ne2*ne1*ne0);- const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);- const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- //const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);-- const int64_t k = i3*ne3 + i2;-- float m_k;- if (k < n_heads_log2_floor) {- m_k = pow(m0, k + 1);- } else {- m_k = pow(m1, 2 * (k - n_heads_log2_floor) + 1);- }-- device char * dst_row = (device char *) dst + i3*nb3 + i2*nb2 + i1*nb1;- device const char * src_row = (device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01;- for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {- const float src_v = *(device float *)(src_row + i00*nb00);- device float * dst_v = (device float *)(dst_row + i00*nb0);- *dst_v = i00 * m_k + src_v;- }-}-static float rope_yarn_ramp(const float low, const float high, const int i0) {const float y = (i0 / 2 - low) / max(0.001f, high - low);return 1.0f - min(1.0f, max(0.0f, y));@@ -3545,6 +3494,7 @@ static void rope_yarn_corr_dims(typedef void (rope_t)(device const void * src0,device const int32_t * src1,+ device const float * src2,device float * dst,constant int64_t & ne00,constant int64_t & ne01,@@ -3580,6 +3530,7 @@ template<typename T>kernel void kernel_rope(device const void * src0,device const int32_t * src1,+ device const float * src2,device float * dst,constant int64_t & ne00,constant int64_t & ne01,@@ -3649,8 +3600,10 @@ kernel void kernel_rope(// simplified from `(ib * n_dims + ic) * inv_ndims`const float cur_rot = inv_ndims*ic - ib;+ const float freq_factor = src2 != src0 ? src2[ic/2] : 1.0f;++ const float theta = theta_0 * pow(freq_base, cur_rot) / freq_factor;- const float theta = theta_0 * pow(freq_base, cur_rot);float cos_theta, sin_theta;rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);@@ -3757,7 +3710,10 @@ kernel void kernel_upscale_f32(constant uint64_t & nb1,constant uint64_t & nb2,constant uint64_t & nb3,- constant int32_t & sf,+ constant float & sf0,+ constant float & sf1,+ constant float & sf2,+ constant float & sf3,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],uint3 ntg[[threads_per_threadgroup]]) {@@ -3766,15 +3722,17 @@ kernel void kernel_upscale_f32(const int64_t i2 = tgpig.y;const int64_t i1 = tgpig.x;- const int64_t i03 = i3;- const int64_t i02 = i2;- const int64_t i01 = i1/sf;-- device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);- device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1);+ const int64_t i03 = i3/sf3;+ const int64_t i02 = i2/sf2;+ const int64_t i01 = i1/sf1;for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {- dst_ptr[i0] = src0_ptr[i0/sf];+ const int64_t i00 = i0/sf0;++ device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);+ device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ dst_ptr[0] = src0_ptr[0];}}@@ -3954,29 +3912,29 @@ typedef void (flash_attn_ext_f16_t)(device const char * v,device const char * mask,device float * dst,- constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant uint64_t & nb00,constant uint64_t & nb01,constant uint64_t & nb02,constant uint64_t & nb03,- constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant uint64_t & nb10,constant uint64_t & nb11,constant uint64_t & nb12,constant uint64_t & nb13,- constant int64_t & ne31,+ constant uint64_t & nb21,+ constant uint64_t & nb22,+ constant uint64_t & nb23,constant uint64_t & nb31,- constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,- constant int64_t & ne3,constant float & scale,+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,threadgroup half * shared,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],@@ -3992,29 +3950,29 @@ kernel void kernel_flash_attn_ext_f16(device const char * v,device const char * mask,device float * dst,- constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant uint64_t & nb00,constant uint64_t & nb01,constant uint64_t & nb02,constant uint64_t & nb03,- constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant uint64_t & nb10,constant uint64_t & nb11,constant uint64_t & nb12,constant uint64_t & nb13,- constant int64_t & ne31,+ constant uint64_t & nb21,+ constant uint64_t & nb22,+ constant uint64_t & nb23,constant uint64_t & nb31,- constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,- constant int64_t & ne3,constant float & scale,+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,threadgroup half * shared [[threadgroup(0)]],uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],@@ -4029,7 +3987,7 @@ kernel void kernel_flash_attn_ext_f16(const short D4 = D/4;const short D8 = D/8;- const short Q8 = Q/8;+ //const short Q8 = Q/8;const short NW = N_SIMDWIDTH;const short SH = (C + Q); // shared memory per simdgroup in (half)@@ -4079,10 +4037,6 @@ kernel void kernel_flash_attn_ext_f16(const short ne22 = ne12;const short ne23 = ne13;- const uint nb21 = nb11;- const uint nb22 = nb12;- const uint nb23 = nb13;-// broadcastconst short rk2 = ne02/ne12;const short rk3 = ne03/ne13;@@ -4108,8 +4062,17 @@ kernel void kernel_flash_attn_ext_f16(// pointer to the maskdevice const half * mp = (device const half *) (mask + iq1*nb31);- // prepare diagonal scale matrix- simdgroup_float8x8 mscale(scale);+ float slope = 1.0f;++ // ALiBi+ if (max_bias > 0.0f) {+ const uint32_t h = iq2;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exph);+ }// loop over the KV cache// each simdgroup handles blocks of Q rows and C columns@@ -4133,12 +4096,20 @@ kernel void kernel_flash_attn_ext_f16(simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk);}- // mqk = mqk*scale + mask- simdgroup_half8x8 mm;- simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false);- simdgroup_multiply_accumulate(mqk, mqk, mscale, mm);-simdgroup_store(mqk, ss + 8*cc, TF, 0, false);++ const short tx = tiisg%4;+ const short ty = tiisg/4;++ if (mask != q) {+ // mqk = mqk*scale + mask*slope+ ss[8*cc + ty*TF + 2*tx + 0] = scale*ss[8*cc + ty*TF + 2*tx + 0] + slope*mp[ic + 8*cc + ty*nb31/sizeof(half) + 2*tx + 0];+ ss[8*cc + ty*TF + 2*tx + 1] = scale*ss[8*cc + ty*TF + 2*tx + 1] + slope*mp[ic + 8*cc + ty*nb31/sizeof(half) + 2*tx + 1];+ } else {+ // mqk = mqk*scale+ ss[8*cc + ty*TF + 2*tx + 0] *= scale;+ ss[8*cc + ty*TF + 2*tx + 1] *= scale;+ }}}@@ -4310,29 +4281,29 @@ kernel void kernel_flash_attn_ext_vec_f16(device const char * v,device const char * mask,device float * dst,- constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant uint64_t & nb00,constant uint64_t & nb01,constant uint64_t & nb02,constant uint64_t & nb03,- constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant uint64_t & nb10,constant uint64_t & nb11,constant uint64_t & nb12,constant uint64_t & nb13,- constant int64_t & ne31,+ constant uint64_t & nb21,+ constant uint64_t & nb22,+ constant uint64_t & nb23,constant uint64_t & nb31,- constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,- constant int64_t & ne3,constant float & scale,+ constant float & max_bias,+ constant float & m0,+ constant float & m1,+ constant uint32_t & n_head_log2,threadgroup half * shared [[threadgroup(0)]],uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],@@ -4351,6 +4322,18 @@ kernel void kernel_flash_attn_ext_vec_f16(const short T = D + 2*nsg*SH; // shared memory size per query in (half)+ float slope = 1.0f;++ // ALiBi+ if (max_bias > 0.0f) {+ const uint32_t h = iq2;++ const float base = h < n_head_log2 ? m0 : m1;+ const int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ slope = pow(base, exp);+ }+//threadgroup half * sq = (threadgroup half *) (shared + 0*D); // holds the query datathreadgroup half4 * sq4 = (threadgroup half4 *) (shared + 0*D); // same as above but in half4threadgroup float * ss = (threadgroup float *) (shared + 2*sgitg*SH + 1*D); // scratch buffer for attention and diagonal matrix@@ -4391,10 +4374,6 @@ kernel void kernel_flash_attn_ext_vec_f16(const short ne22 = ne12;const short ne23 = ne13;- const uint nb21 = nb11;- const uint nb22 = nb12;- const uint nb23 = nb13;-// broadcastconst short rk2 = ne02/ne12;const short rk3 = ne03/ne13;@@ -4457,10 +4436,9 @@ kernel void kernel_flash_attn_ext_vec_f16(mqk += simd_shuffle_down(mqk, 2);mqk += simd_shuffle_down(mqk, 1);- // mqk = mqk*scale + mask+ // mqk = mqk*scale + mask*slopeif (tiisg == 0) {- float4 mm = (float4) mp4[ic/4 + cc];- mqk = mqk*scale + mm;+ mqk = mqk*scale + ((mask != q) ? ((float4) mp4[ic/4 + cc])*slope : (float4) 0.0f);ss4[cc] = mqk;}diff --git a/llama.cpp/ggml-quants.inc b/llama.cpp/ggml-quants.incindex c33e21f1..ed40ca74 100644--- a/llama.cpp/ggml-quants.inc+++ b/llama.cpp/ggml-quants.inc@@ -14,6 +14,18 @@#include <stdlib.h> // for qsort#include <stdio.h> // for GGML_ASSERT+#define GROUP_MAX_EPS 1e-15f+#define GROUP_MAX_EPS_IQ3_XXS 1e-8f+#define GROUP_MAX_EPS_IQ2_S 1e-8f+#define GROUP_MAX_EPS_IQ1_M 1e-7f+#define GROUP_MAX_EPS_IQ1_S 1e-12f++#if defined(_MSC_VER)+// disable "possible loss of data" to avoid warnings for hundreds of casts+// we should just be careful :)+#pragma warning(disable: 4244 4267)+#endif+#define UNUSED GGML_UNUSED// some compilers don't provide _mm256_set_m128i, e.g. gcc 7@@ -235,7 +247,7 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128#endif // __AVX__ || __AVX2__ || __AVX512F__#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)-#if defined(__ARM_NEON) || defined(__wasm_simd128__)+#if defined(__ARM_NEON) || defined(__wasm_simd128__) || defined(__POWER9_VECTOR__)#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)@@ -250,6 +262,403 @@ static const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4static const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4#endif+#if defined(__loongarch_asx)++#ifdef __clang__+#define VREGS_PREFIX "$vr"+#define XREGS_PREFIX "$xr"+#else // GCC+#define VREGS_PREFIX "$f"+#define XREGS_PREFIX "$f"+#endif+#define __ALL_REGS "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"+// Convert __m128i to __m256i+static inline __m256i ____m256i(__m128i in) {+ __m256i out = __lasx_xvldi(0);+ __asm__ volatile (+ ".irp i," __ALL_REGS "\n\t"+ " .ifc %[out], " XREGS_PREFIX"\\i \n\t"+ " .irp j," __ALL_REGS "\n\t"+ " .ifc %[in], " VREGS_PREFIX "\\j \n\t"+ " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t"+ " .endif \n\t"+ " .endr \n\t"+ " .endif \n\t"+ ".endr \n\t"+ : [out] "+f" (out) : [in] "f" (in)+ );+ return out;+}+// Convert two __m128i to __m256i+static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) {+ __m256i out;+ __asm__ volatile (+ ".irp i," __ALL_REGS "\n\t"+ " .ifc %[hi], " VREGS_PREFIX "\\i \n\t"+ " .irp j," __ALL_REGS "\n\t"+ " .ifc %[lo], " VREGS_PREFIX "\\j \n\t"+ " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t"+ " .endif \n\t"+ " .endr \n\t"+ " .endif \n\t"+ ".endr \n\t"+ ".ifnc %[out], %[hi] \n\t"+ ".irp i," __ALL_REGS "\n\t"+ " .ifc %[out], " XREGS_PREFIX "\\i \n\t"+ " .irp j," __ALL_REGS "\n\t"+ " .ifc %[hi], " VREGS_PREFIX "\\j \n\t"+ " xvori.b $xr\\i, $xr\\j, 0 \n\t"+ " .endif \n\t"+ " .endr \n\t"+ " .endif \n\t"+ ".endr \n\t"+ ".endif \n\t"+ : [out] "=f" (out), [hi] "+f" (inhi)+ : [lo] "f" (inlo)+ );+ return out;+}+// Convert __m256i low part to __m128i+static inline __m128i lasx_extracti128_lo(__m256i in) {+ __m128i out;+ __asm__ volatile (+ ".ifnc %[out], %[in] \n\t"+ ".irp i," __ALL_REGS "\n\t"+ " .ifc %[out], " VREGS_PREFIX "\\i \n\t"+ " .irp j," __ALL_REGS "\n\t"+ " .ifc %[in], " XREGS_PREFIX "\\j \n\t"+ " vori.b $vr\\i, $vr\\j, 0 \n\t"+ " .endif \n\t"+ " .endr \n\t"+ " .endif \n\t"+ ".endr \n\t"+ ".endif \n\t"+ : [out] "=f" (out) : [in] "f" (in)+ );+ return out;+}+// Convert __m256i high part to __m128i+static inline __m128i lasx_extracti128_hi(__m256i in) {+ __m128i out;+ __asm__ volatile (+ ".irp i," __ALL_REGS "\n\t"+ " .ifc %[out], " VREGS_PREFIX "\\i \n\t"+ " .irp j," __ALL_REGS "\n\t"+ " .ifc %[in], " XREGS_PREFIX "\\j \n\t"+ " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t"+ " .endif \n\t"+ " .endr \n\t"+ " .endif \n\t"+ ".endr \n\t"+ : [out] "=f" (out) : [in] "f" (in)+ );+ return out;+}++static __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) {+ v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7};+ return (__m256i)__ret;+}++static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) {+ v4i32 __ret = {d, c, b, a};+ return (__m128i)__ret;+}++static __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) {+ v4i64 __ret = {d, c, b, a};+ return (__m256i)__ret;+}++static __m256i lasx_insertf128( __m128i x, __m128i y) {+ return lasx_set_q(x, y);+}++static __m128i lsx_shuffle_b(__m128i a, __m128i b) {+ __m128i mask_f, zero, tmp0, tmp2, mask;+ int f = 0x8f;+ mask_f = __lsx_vreplgr2vr_b(f);+ zero = __lsx_vldi(0);+ tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits+ tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive+ mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask+ tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones+ return __lsx_vshuf_b(a, zero, tmp2);+}++static __m256i lasx_shuffle_b(__m256i a, __m256i b) {+ __m256i mask_f, zero, tmp0, tmp2, mask;+ int f = 0x8f;+ mask_f = __lasx_xvreplgr2vr_b(f);+ zero = __lasx_xvldi(0);+ tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits+ tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive+ mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask+ tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones+ return __lasx_xvshuf_b(a, zero, tmp2);+}++static __m256i lasx_extu8_16(__m128i a) {+ __m128i zero = __lsx_vldi(0);+ __m128i vlo = __lsx_vilvl_b(zero, a);+ __m128i vhi = __lsx_vilvh_b(zero, a);+ return lasx_set_q(vhi, vlo);+}++static __m256i lasx_ext8_16(__m128i a) {+ __m128i sign = __lsx_vslti_b(a, 0);+ __m128i vlo = __lsx_vilvl_b(sign, a);+ __m128i vhi = __lsx_vilvh_b(sign, a);+ return lasx_set_q(vhi, vlo);+}++static __m256i lasx_ext16_32(__m128i a) {+ __m256i tmp1;+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 0), 0);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 1), 1);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 2), 2);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 3), 3);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 4), 4);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 5), 5);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 6), 6);+ tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 7), 7);+ return tmp1;+}++static __m128i lasx_extracti128( __m256i a, int pos) {+ __m128i ret;+ if( pos == 0)+ {+ ret = lasx_extracti128_lo(a);+ } else {+ ret = lasx_extracti128_hi(a);+ }+ return ret;+}++static __m128 lasx_extractf128( __m256 a, int pos) {+ __m128 ret;+ if( pos == 0)+ {+ ret = (__m128)lasx_extracti128_lo((__m256i)a);+ } else {+ ret = (__m128)lasx_extracti128_hi((__m256i)a);+ }+ return ret;+}++static __m128i lsx_hadd_h(__m128i a, __m128i b) {+ __m128i tmp1 = __lsx_vpickev_h(b, a);+ __m128i tmp2 = __lsx_vpickod_h(b, a);+ return __lsx_vadd_h(tmp1, tmp2);+}++static __m128i lsx_hadd_w(__m128i a, __m128i b) {+ __m128i tmp1 = __lsx_vpickev_w(b, a);+ __m128i tmp2 = __lsx_vpickod_w(b, a);+ return __lsx_vadd_w(tmp1, tmp2);+}++static __m128 lsx_hadd_s(__m128 a, __m128 b) {+ __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a);+ __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a);++ return __lsx_vfadd_s(tmp1, tmp2);+}++static __m256i lasx_maddubs_h(__m256i a, __m256i b) {+ __m256i tmp1, tmp2;+ tmp1 = __lasx_xvmulwev_h_b(a, b);+ tmp2 = __lasx_xvmulwod_h_b(a, b);+ return __lasx_xvsadd_h(tmp1, tmp2);+}++static __m256i lasx_madd_h(__m256i a, __m256i b) {+ __m256i tmp1, tmp2;+ tmp1 = __lasx_xvmulwev_w_h(a, b);+ tmp2 = __lasx_xvmulwod_w_h(a, b);+ return __lasx_xvadd_w(tmp1, tmp2);+}++static __m256i lasx_packs_w(__m256i a, __m256i b) {+ __m256i tmp, tmp1;+ tmp = __lasx_xvsat_w(a, 15);+ tmp1 = __lasx_xvsat_w(b, 15);+ return __lasx_xvpickev_h(tmp1, tmp);+}++static __m256i lasx_packs_h(__m256i a, __m256i b) {+ __m256i tmp, tmp1;+ tmp = __lasx_xvsat_h(a, 7);+ tmp1 = __lasx_xvsat_h(b, 7);+ return __lasx_xvpickev_b(tmp1, tmp);+}++static __m128i lsx_packs_w(__m128i a, __m128i b) {+ __m128i tmp, tmp1;+ tmp = __lsx_vsat_w(a, 15);+ tmp1 = __lsx_vsat_w(b, 15);+ return __lsx_vpickev_h(tmp1, tmp);+}++static __m128i lsx_packs_h(__m128i a, __m128i b) {+ __m128i tmp, tmp1;+ tmp = __lsx_vsat_h(a, 7);+ tmp1 = __lsx_vsat_h(b, 7);+ return __lsx_vpickev_b(tmp1, tmp);+}++static __m128i lsx_packus_h(__m128i a, __m128i b) {+ __m128i tmp, tmp1;+ tmp = __lsx_vsat_hu(a, 7);+ tmp1 = __lsx_vsat_hu(b, 7);+ return __lsx_vpickev_b(tmp1, tmp);+}+++static __m128i lsx_maddubs_h(__m128i a, __m128i b) {+ __m128i tmp1, tmp2;+ tmp1 = __lsx_vmulwev_h_b(a, b);+ tmp2 = __lsx_vmulwod_h_b(a, b);+ return __lsx_vsadd_h(tmp1, tmp2);+}++static __m128i lsx_madd_h(__m128i a, __m128i b) {+ __m128i tmp1, tmp2;+ tmp1 = __lsx_vmulwev_w_h(a, b);+ tmp2 = __lsx_vmulwod_w_h(a, b);+ return __lsx_vadd_w(tmp1, tmp2);+}++// multiply int8_t, add results pairwise twice+static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {+ // Get absolute values of x vectors+ const __m128i ax = __lsx_vsigncov_b(x, x);+ // Sign the values of the y vectors+ const __m128i sy = __lsx_vsigncov_b(x, y);+ // Perform multiplication and create 16-bit values+ const __m128i dot = lsx_maddubs_h(ax, sy);+ const __m128i ones = __lsx_vreplgr2vr_h(1);+ return lsx_madd_h(ones, dot);+}++// horizontally add 8 floats+static inline float hsum_float_8(const __m256 x) {+ __m128 res = lasx_extractf128(x, 1);+ ft_union tmp;+ res = __lsx_vfadd_s(res, lasx_extractf128(x, 0));+ res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res));+ res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0));+ tmp.i = __lsx_vpickve2gr_w(res, 0);+ return tmp.f;+}++// horizontally add 8 int32_t+static inline int hsum_i32_8(const __m256i a) {++ __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11);+ __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00);++ __m128i tmp1_128 = lasx_extracti128_lo(tmp1);+ __m128i tmp2_128 = lasx_extracti128_lo(tmp2);++ __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128);++ __m128i ev = __lsx_vpickev_w(sum128, sum128);+ __m128i od = __lsx_vpickod_w(sum128, sum128);+ __m128i sum64 = __lsx_vadd_w(ev, od);++ int sum64_1, sum64_2;+ sum64_1 = __lsx_vpickve2gr_w(sum64, 0);+ sum64_2 = __lsx_vpickve2gr_w(sum64, 1);++ return sum64_1 + sum64_2;+}++// horizontally add 4 int32_t+static inline int hsum_i32_4(const __m128i a) {+ __m128i ev = __lsx_vpickev_w(a, a);+ __m128i od = __lsx_vpickod_w(a, a);+ __m128i sum64 = __lsx_vadd_w(ev, od);++ int sum64_1, sum64_2;+ sum64_1 = __lsx_vpickve2gr_w(sum64, 0);+ sum64_2 = __lsx_vpickve2gr_w(sum64, 1);++ return sum64_1 + sum64_2;+}++// spread 32 bits to 32 bytes { 0x00, 0xFF }+static inline __m256i bytes_from_bits_32(const uint8_t * x) {++ uint32_t x32;+ memcpy(&x32, x, sizeof(uint32_t));+ const __m256i shuf_mask = lasx_set_d(+ 0x0303030303030303, 0x0202020202020202,+ 0x0101010101010101, 0x0000000000000000);++ __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask);+ const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe);+ bytes = __lasx_xvor_v(bytes, bit_mask);+ return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1));+}++// Unpack 32 4-bit fields into 32 bytes+// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval+static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {+ const __m128i lo = __lsx_vld((const __m128i *)rsi, 0);+ __m128i hi = __lsx_vsrli_h(lo, 4);+ return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf);+}++// add int16_t pairwise and return as float vector+static inline __m256 sum_i16_pairs_float(const __m256i x) {+ __m256i v = __lasx_xvpackod_h(x, x);+ __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v);+ return __lasx_xvffint_s_w(summed_pairs);+}++static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {+ // Perform multiplication and create 16-bit values+ const __m256i dot = lasx_maddubs_h(ax, sy);+ return sum_i16_pairs_float(dot);+}++// multiply int8_t, add results pairwise twice and return as float vector+static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {++ // Get absolute values of x vectors+ const __m256i ax = __lasx_xvsigncov_b(x, x);+ // Sign the values of the y vectors+ const __m256i sy = __lasx_xvsigncov_b(x, y);++ return mul_sum_us8_pairs_float(ax, sy);+}++static inline __m128i packNibbles( __m256i bytes ) {+ // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh+ const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF);+ __m256i high = __lasx_xvandn_v(lowByte, bytes);+ __m256i low = __lasx_xvand_v(lowByte, bytes);+ high = __lasx_xvsrli_h(high, 4);+ bytes = __lasx_xvor_v(low, high);+ // Compress uint16_t lanes into bytes+ __m128i *r0 = (__m128i *)&bytes;+ __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11);+ __m128i *r1 = (__m128i *)&tmp_h128;++ __m128i zero = __lsx_vldi(0);+ __m128i tmp, tmp2, tmp3;++ tmp = __lsx_vmax_h(zero, *r0);+ tmp2 = __lsx_vsat_hu(tmp, 7);++ tmp = __lsx_vmax_h(zero, *r1);+ tmp3 = __lsx_vsat_hu(tmp, 7);+ return __lsx_vpickev_b(tmp3, tmp2);+}+#endif //__loongarch_asx+// reference implementation for deterministic creation of model filesvoid quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int64_t k) {static const int qk = QK4_0;@@ -637,6 +1046,102 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k)// store result__riscv_vse8_v_i8m1(y[i].qs , vs, vl);}++#elif defined(__POWER9_VECTOR__)+ for (int i = 0; i < nb; i++) {+ vector float srcv [8];+ vector float asrcv[8];+ vector float amaxv[8];+ vector signed int vi[8];++ for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);+ for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);++ for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);+ for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);+ for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);++ const float amax = MAX(MAX(vec_extract(amaxv[0], 0),+ vec_extract(amaxv[0], 1)),+ MAX(vec_extract(amaxv[0], 2),+ vec_extract(amaxv[0], 3)));++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;+ const vector float vid = vec_splats(id);++ y[i].d = GGML_FP32_TO_FP16(d);++ for (int j = 0; j < 8; j++) {+ const vector float v = vec_round(vec_mul(srcv[j], vid));+ vi[j] = vec_cts(v, 0);+ }+ vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);+ vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);++#elif defined(__loongarch_asx)+ for (int i = 0; i < nb; i++) {+ ft_union fi;+ __m256 v0 = (__m256)__lasx_xvld( x , 0);+ __m256 v1 = (__m256)__lasx_xvld( x , 32);+ __m256 v2 = (__m256)__lasx_xvld( x , 64);+ __m256 v3 = (__m256)__lasx_xvld( x , 96);+ x += 32;++ // Compute max(abs(e)) for the block+ const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );+ __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );++ __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) );+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );+ __m128 tmp = max4;+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 ));+ fi.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 );+ const float max_scalar = fi.f;++ // Quantize these floats+ const float d = max_scalar / 127.f;+ y[i].d = GGML_FP32_TO_FP16(d);+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;+ const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id );++ // Apply the multiplier+ v0 = __lasx_xvfmul_s( v0, mul );+ v1 = __lasx_xvfmul_s( v1, mul );+ v2 = __lasx_xvfmul_s( v2, mul );+ v3 = __lasx_xvfmul_s( v3, mul );++ // Round to nearest integer+ __m256i i0 = __lasx_xvftintrne_w_s( v0 );+ __m256i i1 = __lasx_xvftintrne_w_s( v1 );+ __m256i i2 = __lasx_xvftintrne_w_s( v2 );+ __m256i i3 = __lasx_xvftintrne_w_s( v3 );++ __m128i ni0 = lasx_extracti128( i0, 0 );+ __m128i ni1 = lasx_extracti128( i0, 1);+ __m128i ni2 = lasx_extracti128( i1, 0);+ __m128i ni3 = lasx_extracti128( i1, 1);+ __m128i ni4 = lasx_extracti128( i2, 0);+ __m128i ni5 = lasx_extracti128( i2, 1);+ __m128i ni6 = lasx_extracti128( i3, 0);+ __m128i ni7 = lasx_extracti128( i3, 1);++ // Convert int32 to int16+ ni0 = lsx_packs_w( ni0, ni1 );+ ni2 = lsx_packs_w( ni2, ni3 );+ ni4 = lsx_packs_w( ni4, ni5 );+ ni6 = lsx_packs_w( ni6, ni7 );+ // Convert int16 to int8+ ni0 = lsx_packs_h( ni0, ni2 );+ ni4 = lsx_packs_h( ni4, ni6 );++ __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);+ __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);++ }#elseGGML_UNUSED(nb);// scalar@@ -784,12 +1289,12 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k)__m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );- const float maxScalar = _mm_cvtss_f32( max4 );+ const float max_scalar = _mm_cvtss_f32( max4 );// Quantize these floats- const float d = maxScalar / 127.f;+ const float d = max_scalar / 127.f;y[i].d = GGML_FP32_TO_FP16(d);- const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;const __m256 mul = _mm256_set1_ps( id );// Apply the multiplier@@ -892,6 +1397,114 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k)int sum = __riscv_vmv_x_s_i16m1_i16(vwrs);y[i].s = GGML_FP32_TO_FP16(sum*d);}++#elif defined(__POWER9_VECTOR__)+ for (int i = 0; i < nb; i++) {+ vector float srcv [8];+ vector float asrcv[8];+ vector float amaxv[8];+ vector signed int vi[8];++ for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);+ for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);++ for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);+ for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);+ for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);++ const float amax = MAX(MAX(vec_extract(amaxv[0], 0),+ vec_extract(amaxv[0], 1)),+ MAX(vec_extract(amaxv[0], 2),+ vec_extract(amaxv[0], 3)));++ const float d = amax / ((1 << 7) - 1);+ const float id = d ? 1.0f/d : 0.0f;+ const vector float vid = vec_splats(id);++ y[i].d = GGML_FP32_TO_FP16(d);++ vector int accv = vec_splats(0);++ for (int j = 0; j < 8; j++) {+ const vector float v = vec_round(vec_mul(srcv[j], vid));+ vi[j] = vec_cts(v, 0);++ accv = vec_add(accv, vi[j]);+ }+ vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]);+ vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);++ accv = vec_add(accv, vec_sld(accv, accv, 4));+ accv = vec_add(accv, vec_sld(accv, accv, 8));+ y[i].s = GGML_FP32_TO_FP16(d * vec_extract(accv, 0));++#elif defined(__loongarch_asx)+ for (int i = 0; i < nb; i++) {+ ft_union ft;+ __m256 v0 = (__m256)__lasx_xvld( x , 0 );+ __m256 v1 = (__m256)__lasx_xvld( x , 32 );+ __m256 v2 = (__m256)__lasx_xvld( x , 64 );+ __m256 v3 = (__m256)__lasx_xvld( x , 96 );+ x += 32;++ // Compute max(abs(e)) for the block+ const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );+ __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );+ max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );++ __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) );+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );+ __m128 tmp = max4;+ max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x10 ));+ ft.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 );+ const float max_scalar = ft.f;++ // Quantize these floats+ const float d = max_scalar / 127.f;+ y[i].d = GGML_FP32_TO_FP16(d);+ const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;+ const __m256 mul = __lasx_xvreplfr2vr_s( id );++ // Apply the multiplier+ v0 = __lasx_xvfmul_s( v0, mul );+ v1 = __lasx_xvfmul_s( v1, mul );+ v2 = __lasx_xvfmul_s( v2, mul );+ v3 = __lasx_xvfmul_s( v3, mul );++ // Round to nearest integer+ __m256i i0 = __lasx_xvftintrne_w_s( v0 );+ __m256i i1 = __lasx_xvftintrne_w_s( v1 );+ __m256i i2 = __lasx_xvftintrne_w_s( v2 );+ __m256i i3 = __lasx_xvftintrne_w_s( v3 );++ __m128i ni0 = lasx_extracti128(i0, 0);+ __m128i ni1 = lasx_extracti128( i0, 1);+ __m128i ni2 = lasx_extracti128( i1, 0);+ __m128i ni3 = lasx_extracti128( i1, 1);+ __m128i ni4 = lasx_extracti128( i2, 0 );+ __m128i ni5 = lasx_extracti128( i2, 1);+ __m128i ni6 = lasx_extracti128( i3, 0);+ __m128i ni7 = lasx_extracti128( i3, 1);++ // Compute the sum of the quants and set y[i].s+ const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3));+ const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7));+ y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1)));++ // Convert int32 to int16+ ni0 = lsx_packs_w( ni0, ni1 );+ ni2 = lsx_packs_w( ni2, ni3 );+ ni4 = lsx_packs_w( ni4, ni5 );+ ni6 = lsx_packs_w( ni6, ni7 );+ // Convert int16 to int8+ ni0 = lsx_packs_h( ni0, ni2 );+ ni4 = lsx_packs_h( ni4, ni6 );++ __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0);+ __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);+ }#elseGGML_UNUSED(nb);// scalar@@ -1031,7 +1644,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t *float ax = fabsf(x[i]);if (ax > amax) { amax = ax; max = x[i]; }}- if (amax < 1e-30f) { // all zero+ if (amax < GROUP_MAX_EPS) { // all zerofor (int i = 0; i < n; ++i) {L[i] = 0;}@@ -1065,7 +1678,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t *sumlx += w*x[i]*l;suml2 += w*l*l;}- float scale = sumlx/suml2;+ float scale = suml2 ? sumlx/suml2 : 0.0f;if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale;float best = scale * sumlx;for (int is = -9; is <= 9; ++is) {@@ -1099,7 +1712,7 @@ static float make_q3_quants(int n, int nmax, const float * restrict x, int8_t *float ax = fabsf(x[i]);if (ax > amax) { amax = ax; max = x[i]; }}- if (!amax) { // all zero+ if (amax < GROUP_MAX_EPS) { // all zerofor (int i = 0; i < n; ++i) { L[i] = 0; }return 0.f;}@@ -1568,7 +2181,7 @@ static float make_qp_quants(int n, int nmax, const float * restrict x, uint8_t *break;}}- return sumlx / suml2;+ return sumlx/suml2;}static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restrict y, int k, const float * restrict quant_weights) {@@ -1908,7 +2521,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrifor (int j = 0; j < QK_K/16; ++j) {if (quant_weights) {- const float * qw = quant_weights ? quant_weights + QK_K * i + 16*j : NULL;+ const float * qw = quant_weights + QK_K * i + 16*j;for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]);} else {for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l];@@ -2575,7 +3188,7 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict}- if (!max_abs_scale) {+ if (max_abs_scale < GROUP_MAX_EPS) {memset(&y[i], 0, sizeof(block_q6_K));y[i].d = GGML_FP32_TO_FP16(0.f);x += QK_K;@@ -2727,7 +3340,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri}- if (!max_abs_scale) {+ if (max_abs_scale < GROUP_MAX_EPS) {memset(&y[i], 0, sizeof(block_q6_K));y[i].d = GGML_FP32_TO_FP16(0.f);x += QK_K;@@ -3339,57 +3952,8 @@ void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int6}}-void quantize_row_q8_K(const float * restrict x, void * restrict vy, int64_t k) {-#ifdef __AVX2__- assert(k % QK_K == 0);- const int nb = k / QK_K;- block_q8_K * y = vy;- const __m256 signBit = _mm256_set1_ps( -0.0f );- const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );- for (int i = 0; i < nb; i++) {- const float * xb = x + i*QK_K;- __m256 maxAbs = _mm256_setzero_ps();- const float * xx = xb;- for (int ib = 0; ib < QK_K/8; ++ib) {- const __m256 v = _mm256_loadu_ps(xx); xx += 8;- maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps(signBit, v));- }- __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );- max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );- max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );- const float maxScalar = _mm_cvtss_f32( max4 );- const float d = maxScalar / 127.f;- y[i].d = d;- const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;- const __m256 mul = _mm256_set1_ps( id );- xx = xb;- int8_t * q8 = y[i].qs;- for (int ib = 0; ib < QK_K/32; ++ib) {- __m256 v0 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8;- __m256 v1 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8;- __m256 v2 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8;- __m256 v3 = _mm256_mul_ps(mul, _mm256_loadu_ps(xx)); xx += 8;- v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );- v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );- v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );- v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );- __m256i i0 = _mm256_cvtps_epi32( v0 );- __m256i i1 = _mm256_cvtps_epi32( v1 );- __m256i i2 = _mm256_cvtps_epi32( v2 );- __m256i i3 = _mm256_cvtps_epi32( v3 );- y[i].bsums[2*ib+0] = hsum_i32_8(_mm256_add_epi32(i0, i1));- y[i].bsums[2*ib+1] = hsum_i32_8(_mm256_add_epi32(i2, i3));- i0 = _mm256_packs_epi32( i0, i1 );- i2 = _mm256_packs_epi32( i2, i3 );- i0 = _mm256_packs_epi16( i0, i2 );- i0 = _mm256_permutevar8x32_epi32( i0, perm );- _mm256_storeu_si256((__m256i *)q8, i0);- q8 += 32;- }- }-#else- quantize_row_q8_K_reference(x, vy, k);-#endif+void quantize_row_q8_K(const float * restrict x, void * restrict y, int64_t k) {+ quantize_row_q8_K_reference(x, y, k);}//===================================== Dot ptoducts =================================@@ -3435,6 +3999,43 @@ static inline __m128i get_scale_shuffle(int i) {};return _mm_loadu_si128((const __m128i*)k_shuffle + i);}+#elif defined(__loongarch_asx)+// shuffles to pick the required scales in dot products+static inline __m256i get_scale_shuffle_q3k(int i) {+ static const uint8_t k_shuffle[128] = {+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,+ };+ return __lasx_xvld((const __m256i*)k_shuffle + i, 0);+}+static inline __m256i get_scale_shuffle_k4(int i) {+ static const uint8_t k_shuffle[256] = {+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,+ 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,+ 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,+ 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,+ 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,+ 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,+ 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,+ 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15+ };+ return __lasx_xvld((const __m256i*)k_shuffle + i, 0);+}+static inline __m128i get_scale_shuffle(int i) {+ static const uint8_t k_shuffle[128] = {+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,+ 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,+ 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,+ 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,+ 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,+ 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15+ };+ return __lsx_vld((const __m128i*)k_shuffle + i, 0);+}#endifvoid ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {@@ -3458,10 +4059,9 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r#if defined(__ARM_FEATURE_MATMUL_INT8)if (nrc == 2) {const block_q4_0 * restrict vx0 = vx;- const block_q4_0 * restrict vx1 = vx + bx;-+ const block_q4_0 * restrict vx1 = (const block_q4_0 *) ((const uint8_t*)vx + bx);const block_q8_0 * restrict vy0 = vy;- const block_q8_0 * restrict vy1 = vy + by;+ const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);float32x4_t sumv0 = vdupq_n_f32(0.0f);@@ -3495,10 +4095,12 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * rconst int8x16_t y1_l = vld1q_s8(b_y1->qs);const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};+ float32_t _scale[4] = { GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};++ float32x4_t scale = vld1q_f32(_scale);int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));@@ -3783,12 +4385,196 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r}*s = sumf;-#else- // scalar- float sumf = 0.0;+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector signed char v8 = vec_splats((signed char)0x8);++ vector float vsumf0 = vec_splats(0.0f);++#pragma GCC unroll 4for (int i = 0; i < nb; i++) {- int sumi = 0;+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));+ vector float vd = vec_mul(vxd, vyd);++ vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char q8y0 = vec_xl( 0, y[i].qs);+ vector signed char q8y1 = vec_xl(16, y[i].qs);++ vector signed char q4x0 = vec_and(qxs, lowMask);+ vector signed char q4x1 = vec_sr(qxs, v4);++ q4x0 = vec_sub(q4x0, v8);+ q4x1 = vec_sub(q4x1, v8);++ vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));++ qv0 = vec_add(qv0, qv1);++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)+ // Initialize accumulator with zeros+ __m256 acc = (__m256)__lasx_xvldi(0);++ // Main loop+ for (int i = 0; i < nb; ++i) {+ /* Compute combined scale for the block */+ const __m256 d = __lasx_xvreplfr2vr_s( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ __m256i qx = bytes_from_nibbles_32(x[i].qs);++ // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.+ const __m256i off = __lasx_xvreplgr2vr_b( 8 );+ qx = __lasx_xvsub_b( qx, off );++ __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ /* Multiply q with scale and accumulate */+ acc = __lasx_xvfmadd_s( d, q, acc );+ }++ *s = hsum_float_8(acc);+#elif defined(__loongarch_sx)+ // set constants+ const __m128i low_mask = __lsx_vreplgr2vr_b(0xF);+ const __m128i off = __lsx_vreplgr2vr_b(8);++ // Initialize accumulator with zeros+ __m128 acc_0 = __lsx_vldi(0);+ __m128 acc_1 = __lsx_vldi(0);+ __m128 acc_2 = __lsx_vldi(0);+ __m128 acc_3 = __lsx_vldi(0);++ // First round without accumulation+ {+ _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0);+ _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) );++ const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[0].qs, 0);++ __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);+ __m128i by_0 = __lsx_vld((const __m128i *)y[0].qs, 0);+ bx_0 = __lsx_vsub_b(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));+ __m128i by_1 = __lsx_vld((const __m128i *)(y[0].qs + 16), 0);+ bx_1 = __lsx_vsub_b(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) );++ const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[1].qs, 0);++ __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);+ __m128i by_2 = __lsx_vld((const __m128i *)y[1].qs, 0);+ bx_2 = __lsx_vsub_b(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));+ __m128i by_3 = __lsx_vld((const __m128i *)(y[1].qs + 16), 0);+ bx_3 = __lsx_vsub_b(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = __lsx_vffint_s_w(i32_0);+ __m128 p1 = __lsx_vffint_s_w(i32_1);+ __m128 p2 = __lsx_vffint_s_w(i32_2);+ __m128 p3 = __lsx_vffint_s_w(i32_3);++ // Apply the scale+ acc_0 = __lsx_vfmul_s( d_0_1, p0 );+ acc_1 = __lsx_vfmul_s( d_0_1, p1 );+ acc_2 = __lsx_vfmul_s( d_2_3, p2 );+ acc_3 = __lsx_vfmul_s( d_2_3, p3 );+ }++ assert(nb % 2 == 0); // TODO: handle odd nb++ // Main loop+ for (int i = 2; i < nb; i+=2) {++ // Compute combined scale for the block 0 and 1+ const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) );++ const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[i].qs, 0);++ __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);+ __m128i by_0 = __lsx_vld((const __m128i *)y[i].qs, 0);+ bx_0 = __lsx_vsub_b(bx_0, off);+ const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);++ __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));+ __m128i by_1 = __lsx_vld((const __m128i *)(y[i].qs + 16), 0);+ bx_1 = __lsx_vsub_b(bx_1, off);+ const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);++ //_mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0);+ //_mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0);++ // Compute combined scale for the block 2 and 3+ const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) );++ const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[i + 1].qs, 0);++ __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);+ __m128i by_2 = __lsx_vld((const __m128i *)y[i + 1].qs, 0);+ bx_2 = __lsx_vsub_b(bx_2, off);+ const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);++ __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));+ __m128i by_3 = __lsx_vld((const __m128i *)(y[i + 1].qs + 16), 0);+ bx_3 = __lsx_vsub_b(bx_3, off);+ const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);++ // Convert int32_t to float+ __m128 p0 = __lsx_vffint_s_w(i32_0);+ __m128 p1 = __lsx_vffint_s_w(i32_1);+ __m128 p2 = __lsx_vffint_s_w(i32_2);+ __m128 p3 = __lsx_vffint_s_w(i32_3);++ // Apply the scale+ __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 );+ __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 );+ __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 );+ __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 );++ // Acummulate+ acc_0 = __lsx_vfadd_s(p0_d, acc_0);+ acc_1 = __lsx_vfadd_s(p1_d, acc_1);+ acc_2 = __lsx_vfadd_s(p2_d, acc_2);+ acc_3 = __lsx_vfadd_s(p3_d, acc_3);+ }++ *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);++#else+ // scalar+ float sumf = 0.0;++ for (int i = 0; i < nb; i++) {+ int sumi = 0;for (int j = 0; j < qk/2; ++j) {const int v0 = (x[i].qs[j] & 0x0F) - 8;@@ -3825,9 +4611,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r#if defined(__ARM_FEATURE_MATMUL_INT8)if (nrc == 2) {const block_q4_1 * restrict vx0 = vx;- const block_q4_1 * restrict vx1 = vx + bx;+ const block_q4_1 * restrict vx1 = (const block_q4_1 *) ((const uint8_t*)vx + bx);const block_q8_1 * restrict vy0 = vy;- const block_q8_1 * restrict vy1 = vy + by;+ const block_q8_1 * restrict vy1 = (const block_q8_1 *) ((const uint8_t*)vy + by);float32x4_t sumv0 = vdupq_n_f32(0.0f);float32x4_t summs0 = vdupq_n_f32(0.0f);@@ -3838,11 +4624,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst block_q8_1 * restrict b_y0 = &vy0[i];const block_q8_1 * restrict b_y1 = &vy1[i];- float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),- GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),- GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};- summs0 += summs_t;+ float32_t summs_t[4] = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s),+ GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s),+ GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)};+ summs0 = vaddq_f32(summs0, vld1q_f32(summs_t));const uint8x16_t m4b = vdupq_n_u8(0x0F);@@ -3862,10 +4648,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rconst int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);// mmla into int32x4_t- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,- GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,- GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,- GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};+ float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x0->d)*b_y1->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y0->d,+ GGML_FP16_TO_FP32(b_x1->d)*b_y1->d};+ float32x4_t scale = vld1q_f32(_scale);int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));@@ -3884,7 +4671,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * rfloat32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2);float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);- sumv2 = sumv2 + summs0;+ sumv2 = vaddq_f32(sumv2, summs0);vst1_f32(s, vget_low_f32(sumv2));vst1_f32(s + bs, vget_high_f32(sumv2));@@ -4001,6 +4788,79 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r}*s = sumf;++#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);++#pragma GCC unroll 4+ for (int i = 0; i < nb; i++) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));+ vector float vd = vec_mul(vxd, vyd);++ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].m));+ vector float vys = {GGML_FP16_TO_FP32(y[i].s), 0.0f, 0.0f, 0.0f};+ vsumf0 = vec_madd(vxmin, vys, vsumf0);++ vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char q8y0 = vec_xl( 0, y[i].qs);+ vector signed char q8y1 = vec_xl(16, y[i].qs);++ vector signed char q4x0 = vec_and(qxs, lowMask);+ vector signed char q4x1 = vec_sr(qxs, v4);++ vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));++ qv0 = vec_add(qv0, qv1);++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)+ // Initialize accumulator with zeros+ __m256 acc = (__m256)__lasx_xvldi(0);++ float summs = 0;++ // Main loop+ for (int i = 0; i < nb; ++i) {+ const float d0 = GGML_FP16_TO_FP32(x[i].d);+ const float d1 = GGML_FP16_TO_FP32(y[i].d);++ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);++ const __m256 d0v = __lasx_xvreplfr2vr_s( d0 );+ const __m256 d1v = __lasx_xvreplfr2vr_s( d1 );++ // Compute combined scales+ const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v );++ // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes+ const __m256i qx = bytes_from_nibbles_32(x[i].qs);+ const __m256i qy = __lasx_xvld( (const __m256i *)y[i].qs, 0);++ const __m256 xy = mul_sum_us8_pairs_float(qx, qy);++ // Accumulate d0*d1*x*y+ acc = __lasx_xvfmadd_s( d0d1, xy, acc );+ }++ *s = hsum_float_8(acc) + summs;+#else// scalarfloat sumf = 0.0;@@ -4286,6 +5146,75 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r}*s = sumf;++#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)4);++ vector float vsumf0 = vec_splats(0.0f);++#pragma GCC unroll 4+ for (int i = 0; i < nb; ++i) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));+ vector float vd = vec_mul(vxd, vyd);++ vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[i].qh[0]]), (uint64_t)(table_b2b_1[x[i].qh[1]])};+ vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[i].qh[2]]), (uint64_t)(table_b2b_1[x[i].qh[3]])};++ vector signed char qh0 = (vector signed char)aux64x2_0;+ vector signed char qh1 = (vector signed char)aux64x2_1;++ vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs);++ vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0);+ vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1);++ vector signed char q8y0 = vec_xl( 0, y[i].qs);+ vector signed char q8y1 = vec_xl( 16, y[i].qs);++ vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));++ qv0 = vec_add(qv0, qv1);++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)+ // Initialize accumulator with zeros+ __m256 acc = (__m256)__lasx_xvldi(0);++ // Main loop+ for (int i = 0; i < nb; i++) {+ /* Compute combined scale for the block */+ const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); //FIXME++ __m256i qx = bytes_from_nibbles_32(x[i].qs);+ __m256i bxhi = bytes_from_bits_32(x[i].qh);+ bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));+ qx = __lasx_xvor_v(qx, bxhi);++ __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ /* Multiply q with scale and accumulate */+ acc = __lasx_xvfmadd_s(d, q, acc);+ }++ *s = hsum_float_8(acc);+#else// scalarfloat sumf = 0.0;@@ -4590,6 +5519,82 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r}*s = sumf;++#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);++#pragma GCC unroll 4+ for (int i = 0; i < nb; ++i) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));+ vector float vd = vec_mul(vxd, vyd);++ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].m));+ vector float vys = {GGML_FP16_TO_FP32(y[i].s), 0.f, 0.f, 0.f};+ vsumf0 = vec_madd(vxmin, vys, vsumf0);++ vector unsigned long long aux64x2_0 = {(uint64_t)(table_b2b_0[x[i].qh[0]]), (uint64_t)(table_b2b_0[x[i].qh[1]])};+ vector unsigned long long aux64x2_1 = {(uint64_t)(table_b2b_0[x[i].qh[2]]), (uint64_t)(table_b2b_0[x[i].qh[3]])};++ vector signed char qh0 = (vector signed char)aux64x2_0;+ vector signed char qh1 = (vector signed char)aux64x2_1;++ vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs);++ vector signed char q5x0 = vec_or(vec_and(qxs, lowMask), qh0);+ vector signed char q5x1 = vec_or(vec_sr(qxs, v4), qh1);++ vector signed char q8y0 = vec_xl( 0, y[i].qs);+ vector signed char q8y1 = vec_xl( 16, y[i].qs);++ vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));++ qv0 = vec_add(qv0, qv1);++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)+ // Initialize accumulator with zeros+ __m256 acc = (__m256)__lasx_xvldi(0);++ float summs = 0.0f;++ // Main loop+ for (int i = 0; i < nb; i++) {+ const __m256 dx = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d));++ summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s);++ __m256i qx = bytes_from_nibbles_32(x[i].qs);+ __m256i bxhi = bytes_from_bits_32(x[i].qh);+ bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10));+ qx = __lasx_xvor_v(qx, bxhi);++ const __m256 dy = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[i].d));+ const __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0);++ const __m256 q = mul_sum_us8_pairs_float(qx, qy);++ acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc);+ }++ *s = hsum_float_8(acc) + summs;+#else// scalarfloat sumf = 0.0;@@ -4638,9 +5643,9 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r#if defined(__ARM_FEATURE_MATMUL_INT8)if (nrc == 2) {const block_q8_0 * restrict vx0 = vx;- const block_q8_0 * restrict vx1 = vx + bx;+ const block_q8_0 * restrict vx1 = (const block_q8_0 *) ((const uint8_t*)vx + bx);const block_q8_0 * restrict vy0 = vy;- const block_q8_0 * restrict vy1 = vy + by;+ const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);float32x4_t sumv0 = vdupq_n_f32(0.0f);@@ -4662,10 +5667,11 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * rconst int8x16_t y1_l = vld1q_s8(b_y1->qs);const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);- float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),- GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};+ float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d),+ GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)};+ float32x4_t scale = vld1q_f32(_scale);int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));@@ -4765,6 +5771,66 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r}*s = sumf;++#elif defined(__POWER9_VECTOR__)+ vector float vsumf0 = vec_splats(0.0f);++#pragma GCC unroll 4+ for (int i = 0; i < nb; i++) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));+ vector float vd = vec_mul(vxd, vyd);++ vector signed char q8x0 = vec_xl( 0, x[i].qs);+ vector signed char q8x1 = vec_xl(16, x[i].qs);+ vector signed char q8y0 = vec_xl( 0, y[i].qs);+ vector signed char q8y1 = vec_xl(16, y[i].qs);++ vector signed short qv0 = vec_mule(q8x0, q8y0);+ vector signed short qv1 = vec_mulo(q8x0, q8y0);+ vector signed short qv2 = vec_mule(q8x1, q8y1);+ vector signed short qv3 = vec_mulo(q8x1, q8y1);++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackh(qv1));+ vector signed int vsumi1 = vec_add(vec_unpackl(qv0), vec_unpackl(qv1));+ vector signed int vsumi2 = vec_add(vec_unpackh(qv2), vec_unpackh(qv3));+ vector signed int vsumi3 = vec_add(vec_unpackl(qv2), vec_unpackl(qv3));++ vsumi0 = vec_add(vsumi0, vsumi2);+ vsumi1 = vec_add(vsumi1, vsumi3);++ vsumi0 = vec_add(vsumi0, vsumi1);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)+ // Initialize accumulator with zeros+ __m256 acc = (__m256)__lasx_xvldi(0);++ // Main loop+ for (int i = 0; i < nb; ++i) {+ // Compute combined scale for the block+ const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));+ __m256i qx = __lasx_xvld((const __m256i *)x[i].qs, 0);+ __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0);++ const __m256 q = mul_sum_i8_pairs_float(qx, qy);++ // Multiply q with scale and accumulate+ acc = __lasx_xvfmadd_s( d, q, acc );+ }++ *s = hsum_float_8(acc);+#else// scalarfloat sumf = 0.0;@@ -5120,37 +6186,241 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;-#else+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0x3);+ const vector signed char lowScaleMask = vec_splats((signed char)0xF);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);- float sumf = 0;+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);- const uint8_t * q2 = x[i].qs;- const int8_t * q8 = y[i].qs;- const uint8_t * sc = x[i].scales;+ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));+ vector float vdmin = vec_mul(vxmin, vyd);- int summs = 0;- for (int j = 0; j < 16; ++j) {- summs += y[i].bsums[j] * (sc[j] >> 4);- }+ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);- const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);- const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);+ vector signed char q2xmins = (vector signed char)vec_xl( 0, x[i].scales);+ vector signed char vscales = vec_and(q2xmins, lowScaleMask);- int isum = 0;- int is = 0;- int d;- for (int k = 0; k < QK_K/128; ++k) {- int shift = 0;- for (int j = 0; j < 4; ++j) {- d = sc[is++] & 0xF;- int isuml = 0;- for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);- isum += d * isuml;- d = sc[is++] & 0xF;- isuml = 0;- for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ q2xmins = vec_sr(q2xmins, v4);+ vector signed short q2xmins0 = vec_unpackh(q2xmins);+ vector signed short q2xmins1 = vec_unpackl(q2xmins);++ vector signed int prod0 = vec_mule(q2xmins0, q8ysums0);+ vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0);+ vector signed int prod2 = vec_mule(q2xmins1, q8ysums1);+ vector signed int prod3 = vec_mulo(q2xmins1, q8ysums1);++ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);+++ for (int j = 0; j < QK_K/128; ++j) {+ __builtin_prefetch(q2, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q2);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q2);+ q2 += 32;++ vector signed char q2x00 = vec_and(qxs0, lowMask);+ vector signed char q2x01 = vec_and(vec_sr(qxs0, v2), lowMask);+ vector signed char q2x02 = vec_and(vec_sr(qxs0, v4), lowMask);+ vector signed char q2x03 = vec_and(vec_sr(qxs0, v6), lowMask);+ vector signed char q2x10 = vec_and(qxs1, lowMask);+ vector signed char q2x11 = vec_and(vec_sr(qxs1, v2), lowMask);+ vector signed char q2x12 = vec_and(vec_sr(qxs1, v4), lowMask);+ vector signed char q2x13 = vec_and(vec_sr(qxs1, v6), lowMask);++ vector signed char q8y00 = vec_xl( 0, q8);+ vector signed char q8y10 = vec_xl( 16, q8);+ vector signed char q8y01 = vec_xl( 32, q8);+ vector signed char q8y11 = vec_xl( 48, q8);+ vector signed char q8y02 = vec_xl( 64, q8);+ vector signed char q8y12 = vec_xl( 80, q8);+ vector signed char q8y03 = vec_xl( 96, q8);+ vector signed char q8y13 = vec_xl(112, q8);+ q8 += 128;++ vector signed short qv0 = vec_add(vec_mule(q2x00, q8y00), vec_mulo(q2x00, q8y00));+ vector signed short qv1 = vec_add(vec_mule(q2x01, q8y01), vec_mulo(q2x01, q8y01));+ vector signed short qv2 = vec_add(vec_mule(q2x02, q8y02), vec_mulo(q2x02, q8y02));+ vector signed short qv3 = vec_add(vec_mule(q2x03, q8y03), vec_mulo(q2x03, q8y03));+ vector signed short qv4 = vec_add(vec_mule(q2x10, q8y10), vec_mulo(q2x10, q8y10));+ vector signed short qv5 = vec_add(vec_mule(q2x11, q8y11), vec_mulo(q2x11, q8y11));+ vector signed short qv6 = vec_add(vec_mule(q2x12, q8y12), vec_mulo(q2x12, q8y12));+ vector signed short qv7 = vec_add(vec_mule(q2x13, q8y13), vec_mulo(q2x13, q8y13));++ vector signed short vscales_h = vec_unpackh(vscales);+ vector signed short vs0 = vec_splat(vscales_h, 0);+ vector signed short vs1 = vec_splat(vscales_h, 1);+ vector signed short vs2 = vec_splat(vscales_h, 2);+ vector signed short vs3 = vec_splat(vscales_h, 3);+ vector signed short vs4 = vec_splat(vscales_h, 4);+ vector signed short vs5 = vec_splat(vscales_h, 5);+ vector signed short vs6 = vec_splat(vscales_h, 6);+ vector signed short vs7 = vec_splat(vscales_h, 7);+ vscales = vec_sld(vscales, vscales, 8);++ qv0 = vec_mul(qv0, vs0);+ qv1 = vec_mul(qv1, vs2);+ qv2 = vec_mul(qv2, vs4);+ qv3 = vec_mul(qv3, vs6);++ qv0 = vec_madd(qv4, vs1, qv0);+ qv1 = vec_madd(qv5, vs3, qv1);+ qv2 = vec_madd(qv6, vs5, qv2);+ qv3 = vec_madd(qv7, vs7, qv3);++ vsumi0 = vec_add(vec_unpackh(qv0), vsumi0);+ vsumi1 = vec_add(vec_unpackh(qv1), vsumi1);+ vsumi2 = vec_add(vec_unpackh(qv2), vsumi2);+ vsumi3 = vec_add(vec_unpackh(qv3), vsumi3);++ vsumi4 = vec_add(vec_unpackl(qv0), vsumi4);+ vsumi5 = vec_add(vec_unpackl(qv1), vsumi5);+ vsumi6 = vec_add(vec_unpackl(qv2), vsumi6);+ vsumi7 = vec_add(vec_unpackl(qv3), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m3 = __lasx_xvreplgr2vr_b(3);+ const __m128i m4 = __lsx_vreplgr2vr_b(0xF);++ __m256 acc = (__m256)__lasx_xvldi(0);++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const __m128i mins_and_scales = __lsx_vld((const __m128i*)x[i].scales, 0);+ const __m128i scales8 = __lsx_vand_v(mins_and_scales, m4);+ const __m128i mins8 = __lsx_vand_v(__lsx_vsrli_h(mins_and_scales, 4), m4);+ const __m256i mins = lasx_ext8_16(mins8);+ const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0));++ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc);++ const __m256i all_scales = lasx_ext8_16(scales8);+ const __m128i l_scales = lasx_extracti128(all_scales, 0);+ const __m128i h_scales = lasx_extracti128(all_scales, 1);+ const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)};++ __m256i sumi = __lasx_xvldi(0);++ for (int j = 0; j < QK_K/128; ++j) {++ const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32;++ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;++ const __m256i q2_0 = __lasx_xvand_v(q2bits, m3);+ const __m256i q2_1 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 2), m3);+ const __m256i q2_2 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 4), m3);+ const __m256i q2_3 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 6), m3);++ __m256i p0 = lasx_maddubs_h(q2_0, q8_0);+ __m256i p1 = lasx_maddubs_h(q2_1, q8_1);+ __m256i p2 = lasx_maddubs_h(q2_2, q8_2);+ __m256i p3 = lasx_maddubs_h(q2_3, q8_3);++ p0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(0)), p0);+ p1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(1)), p1);+ p2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(2)), p2);+ p3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(3)), p3);++ p0 = __lasx_xvadd_w(p0, p1);+ p2 = __lasx_xvadd_w(p2, p3);++ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2));+ }++ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);++ }++ *s = hsum_float_8(acc);++#else++ float sumf = 0;++ for (int i = 0; i < nb; ++i) {++ const uint8_t * q2 = x[i].qs;+ const int8_t * q8 = y[i].qs;+ const uint8_t * sc = x[i].scales;++ int summs = 0;+ for (int j = 0; j < 16; ++j) {+ summs += y[i].bsums[j] * (sc[j] >> 4);+ }++ const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ int isum = 0;+ int is = 0;+ int d;+ for (int k = 0; k < QK_K/128; ++k) {+ int shift = 0;+ for (int j = 0; j < 4; ++j) {+ d = sc[is++] & 0xF;+ int isuml = 0;+ for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);+ isum += d * isuml;+ d = sc[is++] & 0xF;+ isuml = 0;+ for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);isum += d * isuml;shift += 2;q8 += 32;@@ -5390,6 +6660,145 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;++#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0x3);+ const vector signed char lowScaleMask = vec_splats((signed char)0xF);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++#pragma GCC unroll 2+ for (int i = 0; i < nb; ++i) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));+ vector float vdmin = vec_mul(vxmin, vyd);++ vector signed short q8ysums0 = vec_xl_len(y[i].bsums, 8);++ vector signed char q2xmins = (vector signed char)vec_xl_len(x[i].scales, 4);+ vector signed char vscales = vec_and(q2xmins, lowScaleMask);++ q2xmins = vec_sr(q2xmins, v4);+ vector signed short q2xmins0 = vec_unpackh((vector signed char)q2xmins);++ vector signed int prod0 = vec_mule(q2xmins0, q8ysums0);+ vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0);++ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char q2x00 = vec_and(qxs0, lowMask);+ vector signed char q2x01 = vec_and(vec_sr(qxs0, v2), lowMask);+ vector signed char q2x02 = vec_and(vec_sr(qxs0, v4), lowMask);+ vector signed char q2x03 = vec_and(vec_sr(qxs0, v6), lowMask);++ vector signed char q8y00 = vec_xl( 0, y[i].qs);+ vector signed char q8y01 = vec_xl( 16, y[i].qs);+ vector signed char q8y02 = vec_xl( 32, y[i].qs);+ vector signed char q8y03 = vec_xl( 48, y[i].qs);++ vector signed short qv0 = vec_add(vec_mule(q2x00, q8y00), vec_mulo(q2x00, q8y00));+ vector signed short qv1 = vec_add(vec_mule(q2x01, q8y01), vec_mulo(q2x01, q8y01));+ vector signed short qv2 = vec_add(vec_mule(q2x02, q8y02), vec_mulo(q2x02, q8y02));+ vector signed short qv3 = vec_add(vec_mule(q2x03, q8y03), vec_mulo(q2x03, q8y03));++ vector signed short vscales_h = vec_unpackh(vscales);+ vector signed short vs0 = vec_splat(vscales_h, 0);+ vector signed short vs1 = vec_splat(vscales_h, 1);+ vector signed short vs2 = vec_splat(vscales_h, 2);+ vector signed short vs3 = vec_splat(vscales_h, 3);++ vector signed int vsumi0 = vec_add(vec_mule(qv0, vs0), vec_mulo(qv0, vs0));+ vector signed int vsumi1 = vec_add(vec_mule(qv1, vs1), vec_mulo(qv1, vs1));+ vector signed int vsumi2 = vec_add(vec_mule(qv2, vs2), vec_mulo(qv2, vs2));+ vector signed int vsumi3 = vec_add(vec_mule(qv3, vs3), vec_mulo(qv3, vs3));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m3 = __lasx_xvreplgr2vr_b(3);++ __m256 acc = (__m256)__lasx_xvldi(0);++ uint32_t ud, um;+ const uint8_t * restrict db = (const uint8_t *)&ud;+ const uint8_t * restrict mb = (const uint8_t *)&um;++ float summs = 0;++ // TODO: optimize this++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ const uint8_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const uint32_t * restrict sc = (const uint32_t *)x[i].scales;+ ud = (sc[0] >> 0) & 0x0f0f0f0f;+ um = (sc[0] >> 4) & 0x0f0f0f0f;++ int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3];+ summs += dmin * smin;++ const __m128i q2bits = __lsx_vld((const __m128i*)q2, 0);+ const __m256i q2_0 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 2), q2bits), m3);+ const __m256i q2_1 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 6), __lsx_vsrli_h(q2bits, 4)), m3);++ const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0);+ const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0);++ const __m256i p0 = lasx_maddubs_h(q2_0, q8_0);+ const __m256i p1 = lasx_maddubs_h(q2_1, q8_1);++ const __m256i p_0 = lasx_ext16_32(lasx_extracti128(p0, 0));+ const __m256i p_1 = lasx_ext16_32(lasx_extracti128(p0, 1));+ const __m256i p_2 = lasx_ext16_32(lasx_extracti128(p1, 0));+ const __m256i p_3 = lasx_ext16_32(lasx_extracti128(p1, 1));++ ft_union t0, t1, t2, t3;+ t0.f = d * db[0];+ t1.f = d * db[1];+ t2.f = d * db[2];+ t3.f = d * db[3];+ acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t0.i), __lasx_xvffint_s_w(p_0), acc);+ acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t1.i), __lasx_xvffint_s_w(p_1), acc);+ acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t2.i), __lasx_xvffint_s_w(p_2), acc);+ acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t3.i), __lasx_xvffint_s_w(p_3), acc);+ }++ *s = hsum_float_8(acc) + summs;+#elsefloat sumf = 0;@@ -5884,6 +7293,262 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0x3);+ const vector signed char v1 = vec_splats((signed char)0x1);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);+ const vector signed char off = vec_splats((signed char)0x20);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ uint32_t aux[3];+ uint32_t utmp[4];++ memcpy(aux, x[i].scales, 12);+ utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);+ utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);+ utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);+ utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);++ vector signed char vscales = (vector signed char)vec_xl( 0, utmp);+ vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].hmask);+ vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].hmask);++ vscales = vec_sub(vscales, off);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);+++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/128; ++j) {+ __builtin_prefetch(q3, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q3);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q3);+ q3 += 32;++ //the low 2 bits+ vector signed char qxs00 = vec_and(qxs0, lowMask);+ vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask);+ vector signed char qxs02 = vec_and(vec_sr(qxs0, v4), lowMask);+ vector signed char qxs03 = vec_and(vec_sr(qxs0, v6), lowMask);+ vector signed char qxs10 = vec_and(qxs1, lowMask);+ vector signed char qxs11 = vec_and(vec_sr(qxs1, v2), lowMask);+ vector signed char qxs12 = vec_and(vec_sr(qxs1, v4), lowMask);+ vector signed char qxs13 = vec_and(vec_sr(qxs1, v6), lowMask);++ //the 3rd bit+ vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2);+ vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, (vector unsigned char)v1)), v2);+ vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2);+ vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v3)), v2);+ vector signed char qxh10 = vec_sl(vec_andc(v1, qxhs1), v2);+ vector signed char qxh11 = vec_sl(vec_andc(v1, vec_sr(qxhs1, (vector unsigned char)v1)), v2);+ vector signed char qxh12 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v2)), v2);+ vector signed char qxh13 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v3)), v2);+ qxhs0 = vec_sr(qxhs0, v4);+ qxhs1 = vec_sr(qxhs1, v4);++ vector signed char q3x00 = vec_sub(qxs00, qxh00);+ vector signed char q3x01 = vec_sub(qxs01, qxh01);+ vector signed char q3x02 = vec_sub(qxs02, qxh02);+ vector signed char q3x03 = vec_sub(qxs03, qxh03);+ vector signed char q3x10 = vec_sub(qxs10, qxh10);+ vector signed char q3x11 = vec_sub(qxs11, qxh11);+ vector signed char q3x12 = vec_sub(qxs12, qxh12);+ vector signed char q3x13 = vec_sub(qxs13, qxh13);++ vector signed char q8y00 = vec_xl( 0, q8);+ vector signed char q8y10 = vec_xl( 16, q8);+ vector signed char q8y01 = vec_xl( 32, q8);+ vector signed char q8y11 = vec_xl( 48, q8);+ vector signed char q8y02 = vec_xl( 64, q8);+ vector signed char q8y12 = vec_xl( 80, q8);+ vector signed char q8y03 = vec_xl( 96, q8);+ vector signed char q8y13 = vec_xl(112, q8);+ q8 += 128;++ vector signed short vscales_h = vec_unpackh(vscales);+ vector signed short vs0 = vec_splat(vscales_h, 0);+ vector signed short vs1 = vec_splat(vscales_h, 1);+ vector signed short vs2 = vec_splat(vscales_h, 2);+ vector signed short vs3 = vec_splat(vscales_h, 3);+ vector signed short vs4 = vec_splat(vscales_h, 4);+ vector signed short vs5 = vec_splat(vscales_h, 5);+ vector signed short vs6 = vec_splat(vscales_h, 6);+ vector signed short vs7 = vec_splat(vscales_h, 7);+ vscales = vec_sld(vscales, vscales, 8);++ vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00));+ vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01));+ vector signed short qv02 = vec_add(vec_mule(q3x02, q8y02), vec_mulo(q3x02, q8y02));+ vector signed short qv03 = vec_add(vec_mule(q3x03, q8y03), vec_mulo(q3x03, q8y03));+ vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10));+ vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11));+ vector signed short qv12 = vec_add(vec_mule(q3x12, q8y12), vec_mulo(q3x12, q8y12));+ vector signed short qv13 = vec_add(vec_mule(q3x13, q8y13), vec_mulo(q3x13, q8y13));++ vector signed int vsum0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0));+ vector signed int vsum1 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2));+ vector signed int vsum2 = vec_add(vec_mule(qv02, vs4), vec_mulo(qv02, vs4));+ vector signed int vsum3 = vec_add(vec_mule(qv03, vs6), vec_mulo(qv03, vs6));+ vector signed int vsum4 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1));+ vector signed int vsum5 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3));+ vector signed int vsum6 = vec_add(vec_mule(qv12, vs5), vec_mulo(qv12, vs5));+ vector signed int vsum7 = vec_add(vec_mule(qv13, vs7), vec_mulo(qv13, vs7));++ vsumi0 = vec_add(vsum0, vsumi0);+ vsumi1 = vec_add(vsum1, vsumi1);+ vsumi2 = vec_add(vsum2, vsumi2);+ vsumi3 = vec_add(vsum3, vsumi3);+ vsumi4 = vec_add(vsum4, vsumi4);+ vsumi5 = vec_add(vsum5, vsumi5);+ vsumi6 = vec_add(vsum6, vsumi6);+ vsumi7 = vec_add(vsum7, vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m3 = __lasx_xvreplgr2vr_b(3);+ const __m256i mone = __lasx_xvreplgr2vr_b(1);+ const __m128i m32 = __lsx_vreplgr2vr_b(32);++ __m256 acc = (__m256)__lasx_xvldi(0);++ uint32_t aux[3];++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ // Set up scales+ memcpy(aux, x[i].scales, 12);+ __m128i scales128 = lsx_set_w(+ ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),+ ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),+ (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),+ (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));+ scales128 = __lsx_vsub_b(scales128, m32);+ const __m256i all_scales = lasx_ext8_16(scales128);+ const __m128i l_scales = lasx_extracti128(all_scales, 0);+ const __m128i h_scales = lasx_extracti128(all_scales, 1);+ const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)};++ // high bit+ const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0);++ // integer accumulator+ __m256i sumi = __lasx_xvldi(0);++ int bit = 0;+ int is = 0;++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/128; ++j) {+ // load low 2 bits+ const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32;++ // prepare low and high bits+ const __m256i q3l_0 = __lasx_xvand_v(q3bits, m3);+ const __m256i q3h_0 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 2), m3);+ const __m256i q3h_1 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_2 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 4), m3);+ const __m256i q3h_2 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2);+ ++bit;++ const __m256i q3l_3 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 6), m3);+ const __m256i q3h_3 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2);+ ++bit;++ // load Q8 quants+ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0);+ __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1);+ __m256i q8s_2 = lasx_maddubs_h(q3h_2, q8_2);+ __m256i q8s_3 = lasx_maddubs_h(q3h_3, q8_3);++ __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0);+ __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1);+ __m256i p16_2 = lasx_maddubs_h(q3l_2, q8_2);+ __m256i p16_3 = lasx_maddubs_h(q3l_3, q8_3);++ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);+ p16_2 = __lasx_xvsub_h(p16_2, q8s_2);+ p16_3 = __lasx_xvsub_h(p16_3, q8s_3);++ // multiply with scales+ p16_0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);+ p16_1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);+ p16_2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);+ p16_3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);++ // accumulate+ p16_0 = __lasx_xvadd_w(p16_0, p16_1);+ p16_2 = __lasx_xvadd_w(p16_2, p16_3);+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2));+ }+ // multiply with block scale and accumulate+ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);//FIXME+ }++ *s = hsum_float_8(acc);+#else// scalar version// This function is written like this so the compiler can manage to vectorize most of it@@ -6250,73 +7915,229 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;-#else+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0x3);+ const vector signed char v1 = vec_splats((signed char)0x1);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);+ const vector signed char off = vec_splats((signed char)0x8);- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- int32_t scales[4];- memset(sums, 0, 8*sizeof(float));+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);- float sumf = 0;+#pragma GCC unroll 2for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q3 = x[i].qs;- const uint8_t * restrict hm = x[i].hmask;- const int8_t * restrict q8 = y[i].qs;- int8_t * restrict a = aux8;- for (int l = 0; l < 8; ++l) {- a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);- a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);- a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);- a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);- a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);- a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);- a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);- a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);- }+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);- scales[0] = (x[i].scales[0] & 0xF) - 8;- scales[1] = (x[i].scales[0] >> 4) - 8;- scales[2] = (x[i].scales[1] & 0xF) - 8;- scales[3] = (x[i].scales[1] >> 4) - 8;+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);- memset(aux32, 0, 8*sizeof(int32_t));- for (int j = 0; j < QK_K/16; ++j) {- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;+ uint16_t aux16[2];+ int8_t * scales = (int8_t *)aux16;-#endif+ const uint16_t a = *(const uint16_t *)x[i].scales;+ aux16[0] = a & 0x0f0f;+ aux16[1] = (a >> 4) & 0x0f0f;-}-#endif+ vector signed char vscales = (vector signed char)vec_xl_len(scales, 8);+ vector signed char qxhs0 = (vector signed char)vec_xl_len(x[i].hmask, 8);+ qxhs0 = vec_or(qxhs0, vec_sr(vec_sld(qxhs0, qxhs0, 8), (vector unsigned char)v1));-#if QK_K == 256-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {- assert(n % QK_K == 0);- assert(nrc == 1);- UNUSED(nrc);- UNUSED(bx);- UNUSED(by);- UNUSED(bs);+ vscales = vec_sub(vscales, off);- const block_q4_K * restrict x = vx;- const block_q8_K * restrict y = vy;+ vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char qxs00 = vec_and(qxs0, lowMask);+ vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask);+ vector signed char qxs10 = vec_and(vec_sr(qxs0, v4), lowMask);+ vector signed char qxs11 = vec_and(vec_sr(qxs0, v6), lowMask);- const int nb = n / QK_K;+ //the 3rd bit+ vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2);+ vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2);+ vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v4)), v2);+ vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v6)), v2);+ qxhs0 = vec_sr(qxhs0, v4);- static const uint32_t kmask1 = 0x3f3f3f3f;- static const uint32_t kmask2 = 0x0f0f0f0f;- static const uint32_t kmask3 = 0x03030303;+ vector signed char q3x00 = vec_sub(qxs00, qxh00);+ vector signed char q3x01 = vec_sub(qxs01, qxh01);+ vector signed char q3x10 = vec_sub(qxs10, qxh02);+ vector signed char q3x11 = vec_sub(qxs11, qxh03);++ vector signed char q8y00 = vec_xl( 0, y[i].qs);+ vector signed char q8y01 = vec_xl( 16, y[i].qs);+ vector signed char q8y10 = vec_xl( 32, y[i].qs);+ vector signed char q8y11 = vec_xl( 48, y[i].qs);++ vector signed short vscales_h = vec_unpackh(vscales);+ vector signed short vs0 = vec_splat(vscales_h, 0);+ vector signed short vs1 = vec_splat(vscales_h, 1);+ vector signed short vs2 = vec_splat(vscales_h, 2);+ vector signed short vs3 = vec_splat(vscales_h, 3);++ vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00));+ vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10));+ vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01));+ vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11));++ vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0));+ vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1));+ vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2));+ vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m3 = __lasx_xvreplgr2vr_b(3);+ const __m256i m1 = __lasx_xvreplgr2vr_b(1);++ __m256 acc = (__m256)__lasx_xvldi(0);++ uint64_t aux64;++ uint16_t aux16[2];+ const int8_t * aux8 = (const int8_t *)aux16;++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q3 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const __m256i scale_0 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[2] - 8), __lasx_xvreplgr2vr_h(aux8[0] - 8));+ const __m256i scale_1 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[3] - 8), __lasx_xvreplgr2vr_h(aux8[1] - 8));++ memcpy(&aux64, x[i].hmask, 8);++ __m128i haux = __lsx_vinsgr2vr_d(haux, aux64, 0);+ haux = __lsx_vinsgr2vr_d(haux, aux64 >> 1, 1);+ __m256i q3h_0 = lasx_insertf128(__lsx_vsrli_h(haux, 2), haux);+ __m256i q3h_1 = __lasx_xvsrli_h(q3h_0, 4);+ q3h_0 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_0, m1), 2);+ q3h_1 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_1, m1), 2);++ // load low 2 bits+ const __m128i q3bits = __lsx_vld((const __m128i*)q3, 0);++ // prepare low and high bits+ const __m256i q3aux = lasx_insertf128(__lsx_vsrli_h(q3bits, 2), q3bits);+ const __m256i q3l_0 = __lasx_xvand_v(q3aux, m3);+ const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3aux, 4), m3);++ // load Q8 quants+ const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0);+ const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0);++ // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h,+ // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,+ // and 2 if the high bit was set)+ const __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0);+ const __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1);++ __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0);+ __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1);++ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);++ // multiply with scales+ p16_0 = lasx_madd_h(scale_0, p16_0);+ p16_1 = lasx_madd_h(scale_1, p16_1);++ p16_0 = __lasx_xvadd_w(p16_0, p16_1);++ // multiply with block scale and accumulate+ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(p16_0), acc);+ }++ *s = hsum_float_8(acc);++#else++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ int32_t scales[4];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict hm = x[i].hmask;+ const int8_t * restrict q8 = y[i].qs;+ int8_t * restrict a = aux8;+ for (int l = 0; l < 8; ++l) {+ a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4);+ a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4);+ a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4);+ a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4);+ a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4);+ a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4);+ a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4);+ a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4);+ }++ scales[0] = (x[i].scales[0] & 0xF) - 8;+ scales[1] = (x[i].scales[0] >> 4) - 8;+ scales[2] = (x[i].scales[1] & 0xF) - 8;+ scales[3] = (x[i].scales[1] >> 4) - 8;++ memset(aux32, 0, 8*sizeof(int32_t));+ for (int j = 0; j < QK_K/16; ++j) {+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l];+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;++#endif++}+#endif++#if QK_K == 256+void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {+ assert(n % QK_K == 0);+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);++ const block_q4_K * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++ static const uint32_t kmask1 = 0x3f3f3f3f;+ static const uint32_t kmask2 = 0x0f0f0f0f;+ static const uint32_t kmask3 = 0x03030303;uint32_t utmp[4];@@ -6602,8 +8423,205 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;-#else+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));+ vector float vdmin = vec_mul(vxmin, vyd);++ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);++ memcpy(utmp, x[i].scales, 12);++ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ vector signed char utmps = (vector signed char)vec_xl( 0, utmp);+ vector signed short vscales = vec_unpackh(utmps);+ vector signed short q4xmins = vec_unpackl(utmps);+ vector signed short q4xmins0 = vec_mergeh(q4xmins, q4xmins);+ vector signed short q4xmins1 = vec_mergel(q4xmins, q4xmins);++ vector signed int prod0 = vec_mule(q4xmins0, q8ysums0);+ vector signed int prod1 = vec_mule(q4xmins1, q8ysums1);+ vector signed int prod2 = vec_mulo(q4xmins0, q8ysums0);+ vector signed int prod3 = vec_mulo(q4xmins1, q8ysums1);++ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/64; j+=2) {+ __builtin_prefetch(q4, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q4);+ vector signed char qxs2 = (vector signed char)vec_xl(32, q4);+ vector signed char qxs3 = (vector signed char)vec_xl(48, q4);+ q4 += 64;++ vector signed char q4x00 = vec_and(qxs0, lowMask);+ vector signed char q4x01 = vec_sr(qxs0, v4);+ vector signed char q4x10 = vec_and(qxs1, lowMask);+ vector signed char q4x11 = vec_sr(qxs1, v4);+ vector signed char q4x20 = vec_and(qxs2, lowMask);+ vector signed char q4x21 = vec_sr(qxs2, v4);+ vector signed char q4x30 = vec_and(qxs3, lowMask);+ vector signed char q4x31 = vec_sr(qxs3, v4);++ vector signed char q8y00 = vec_xl( 0, q8);+ vector signed char q8y10 = vec_xl( 16, q8);+ vector signed char q8y01 = vec_xl( 32, q8);+ vector signed char q8y11 = vec_xl( 48, q8);+ vector signed char q8y20 = vec_xl( 64, q8);+ vector signed char q8y30 = vec_xl( 80, q8);+ vector signed char q8y21 = vec_xl( 96, q8);+ vector signed char q8y31 = vec_xl(112, q8);+ q8 += 128;++ vector signed short qv00 = vec_add(vec_mule(q4x00, q8y00), vec_mulo(q4x00, q8y00));+ vector signed short qv01 = vec_add(vec_mule(q4x01, q8y01), vec_mulo(q4x01, q8y01));+ vector signed short qv10 = vec_add(vec_mule(q4x10, q8y10), vec_mulo(q4x10, q8y10));+ vector signed short qv11 = vec_add(vec_mule(q4x11, q8y11), vec_mulo(q4x11, q8y11));+ vector signed short qv20 = vec_add(vec_mule(q4x20, q8y20), vec_mulo(q4x20, q8y20));+ vector signed short qv21 = vec_add(vec_mule(q4x21, q8y21), vec_mulo(q4x21, q8y21));+ vector signed short qv30 = vec_add(vec_mule(q4x30, q8y30), vec_mulo(q4x30, q8y30));+ vector signed short qv31 = vec_add(vec_mule(q4x31, q8y31), vec_mulo(q4x31, q8y31));++ vector signed short vs0 = vec_splat(vscales, 0);+ vector signed short vs1 = vec_splat(vscales, 1);+ vector signed short vs2 = vec_splat(vscales, 2);+ vector signed short vs3 = vec_splat(vscales, 3);+ vscales = vec_sld(vscales, vscales, 8);++ qv00 = vec_add(qv00, qv10);+ qv10 = vec_add(qv01, qv11);+ qv20 = vec_add(qv20, qv30);+ qv30 = vec_add(qv21, qv31);++ vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0);+ vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1);+ vsumi2 = vec_add(vec_mule(qv10, vs1), vsumi2);+ vsumi3 = vec_add(vec_mulo(qv10, vs1), vsumi3);+ vsumi4 = vec_add(vec_mule(qv20, vs2), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv20, vs2), vsumi5);+ vsumi6 = vec_add(vec_mule(qv30, vs3), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv30, vs3), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);++ __m256 acc = (__m256)__lasx_xvldi(0);+ __m128 acc_m = (__m128)__lsx_vldi(0);++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ memcpy(utmp, x[i].scales, 12);++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]));++ const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);+ const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));+ const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s);+ acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);++ const __m128i sc128 = lasx_extracti128(mins_and_scales, 0);+ const __m256i scales = lasx_insertf128(sc128, sc128);++ __m256i sumi = __lasx_xvldi(0);++ for (int j = 0; j < QK_K/64; ++j) {++ const __m256i scale_l = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_h = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1));++ const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;+ const __m256i q4l = __lasx_xvand_v(q4bits, m4);+ const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4);++ const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ __m256i p16l = lasx_maddubs_h(q4l, q8l);+ p16l = lasx_madd_h(scale_l, p16l);++ const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ __m256i p16h = lasx_maddubs_h(q4h, q8h);+ p16h = lasx_madd_h(scale_h, p16h);+ const __m256i sumj = __lasx_xvadd_w(p16l, p16h);++ sumi = __lasx_xvadd_w(sumi, sumj);+ }++ __m256 vd = __lasx_xvreplfr2vr_s(d);+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);+ }++ acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee));+ __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0);+ acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1);+ ft_union fi;+ fi.i = __lsx_vpickve2gr_w(acc_m, 0);+ *s = hsum_float_8(acc) + fi.f ;++#elseconst uint8_t * scales = (const uint8_t*)&utmp[0];const uint8_t * mins = (const uint8_t*)&utmp[2];@@ -6868,6 +8886,132 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++#pragma GCC unroll 2+ for (int i = 0; i < nb; ++i) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d[1]));+ vector float vyd = vec_splats(y[i].d);+ vector float vd= vec_mul(vxd, vyd);++ uint16_t s16[2];+ const uint8_t * scales = (const uint8_t *)s16;++ const uint16_t * restrict b = (const uint16_t *)x[i].scales;+ s16[0] = b[0] & 0x0f0f;+ s16[1] = (b[0] >> 4) & 0x0f0f;++ vector signed char utmps = (vector signed char)vec_xl_len(scales, 4);+ vector signed short vscales = (vector signed short)vec_unpackh(utmps);+ vector signed short q4xmins0 = vec_mergeh(vscales, vscales);+ q4xmins0 = vec_sld(q4xmins0, q4xmins0, 8);++ vector signed short q8ysums0 = vec_xl_len((const int16_t *)(y[i].bsums), 8);++ vector signed int prod0 = vec_mule(q4xmins0, q8ysums0);+ vector signed int prod1 = vec_mulo(q4xmins0, q8ysums0);++ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vd, vsumf0);+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vd, vsumf1);++ vd = vec_mul(vyd, vec_splats(GGML_FP16_TO_FP32(x[i].d[0])));++ vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs);+ vector signed char q4x00 = vec_and(qxs0, lowMask);+ vector signed char q4x01 = vec_sr(qxs0, v4);+ vector signed char q4x10 = vec_and(qxs1, lowMask);+ vector signed char q4x11 = vec_sr(qxs1, v4);++ vector signed char q8y00 = vec_xl( 0, y[i].qs);+ vector signed char q8y10 = vec_xl(16, y[i].qs);+ vector signed char q8y01 = vec_xl(32, y[i].qs);+ vector signed char q8y11 = vec_xl(48, y[i].qs);++ vector signed short qv00 = vec_add(vec_mule(q4x00, q8y00), vec_mulo(q4x00, q8y00));+ vector signed short qv01 = vec_add(vec_mule(q4x01, q8y01), vec_mulo(q4x01, q8y01));+ vector signed short qv10 = vec_add(vec_mule(q4x10, q8y10), vec_mulo(q4x10, q8y10));+ vector signed short qv11 = vec_add(vec_mule(q4x11, q8y11), vec_mulo(q4x11, q8y11));++ vector signed short vs0 = vec_splat(vscales, 0);+ vector signed short vs1 = vec_splat(vscales, 1);++ vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0));+ vector signed int vsumi1 = vec_add(vec_mule(qv10, vs0), vec_mulo(qv10, vs0));+ vector signed int vsumi2 = vec_add(vec_mule(qv01, vs1), vec_mulo(qv01, vs1));+ vector signed int vsumi3 = vec_add(vec_mule(qv11, vs1), vec_mulo(qv11, vs1));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);++ __m256 acc = (__m256)__lasx_xvldi(0);++ float summs = 0;++ uint16_t aux16[2];+ const uint8_t * scales = (const uint8_t *)aux16;++ for (int i = 0; i < nb; ++i) {++ const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d;+ const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d;+ const __m256 vd = __lasx_xvreplfr2vr_s(d);++ const uint16_t * a = (const uint16_t *)x[i].scales;+ aux16[0] = a[0] & 0x0f0f;+ aux16[1] = (a[0] >> 4) & 0x0f0f;++ summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]));++ const uint8_t * restrict q4 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0);+ const __m256i q4l = __lasx_xvand_v(q4bits, m4);+ const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4);++ const __m256i q8l = __lasx_xvld((const __m256i*)(q8+ 0), 0);+ const __m256i q8h = __lasx_xvld((const __m256i*)(q8+32), 0);++ const __m256i p16l = lasx_maddubs_h(q4l, q8l);+ const __m256i p16h = lasx_maddubs_h(q4h, q8h);++ const __m256i p32l = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[0]), p16l);+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32l), acc);++ const __m256i p32h = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[1]), p16h);+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32h), acc);+ }++ *s = hsum_float_8(acc) - summs;+#elseuint8_t aux8[QK_K];@@ -7269,49 +9413,251 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf+sums;-#else-- const uint8_t * scales = (const uint8_t*)&utmp[0];- const uint8_t * mins = (const uint8_t*)&utmp[2];+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v1 = vec_splats((unsigned char)0x1);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);- float sumf = 0;for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].qs;- const uint8_t * restrict hm = x[i].qh;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- uint8_t m = 1;- for (int j = 0; j < QK_K/64; ++j) {- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);- a += 32; m <<= 1;- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);- a += 32; m <<= 1;- q4 += 32;- }+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin));+ vector float vdmin = vec_mul(vxmin, vyd);+memcpy(utmp, x[i].scales, 12);+utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);const uint32_t uaux = utmp[1] & kmask1;utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);utmp[2] = uaux;utmp[0] &= kmask1;- int sumi = 0;- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];- a = aux8;- int is = 0;- for (int j = 0; j < QK_K/32; ++j) {- int32_t scale = scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);+ vector signed short q8ysums1 = vec_xl(16, y[i].bsums);++ vector signed char utmps = (vector signed char)vec_xl( 0, utmp);+ vector signed short vscales = vec_unpackh(utmps);++ vector signed short q5xmins = vec_unpackl(utmps);+ vector signed short q5xmins0 = vec_mergeh(q5xmins, q5xmins);+ vector signed short q5xmins1 = vec_mergel(q5xmins, q5xmins);++ vector signed int prod0 = vec_mule(q5xmins0, q8ysums0);+ vector signed int prod1 = vec_mule(q5xmins1, q8ysums1);+ vector signed int prod2 = vec_mulo(q5xmins0, q8ysums0);+ vector signed int prod3 = vec_mulo(q5xmins1, q8ysums1);++ vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);+ vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);+ vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);+ vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);++ vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh);+ vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].qh);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);++ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/64; ++j) {+ __builtin_prefetch(q5, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q5);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q5);+ q5 += 32;++ vector signed char qxs00 = vec_and(qxs0, lowMask);+ vector signed char qxs01 = vec_sr(qxs0, v4);+ vector signed char qxs10 = vec_and(qxs1, lowMask);+ vector signed char qxs11 = vec_sr(qxs1, v4);++ vector signed char q5h00 = vec_sl(vec_and((vector signed char)v1, qxhs0), v4);+ vector signed char q5h01 = vec_sl(vec_and((vector signed char)v2, qxhs0), v3);+ vector signed char q5h10 = vec_sl(vec_and((vector signed char)v1, qxhs1), v4);+ vector signed char q5h11 = vec_sl(vec_and((vector signed char)v2, qxhs1), v3);+ qxhs0 = vec_sr(qxhs0, v2);+ qxhs1 = vec_sr(qxhs1, v2);++ vector signed char q5x00 = vec_or(q5h00, qxs00);+ vector signed char q5x01 = vec_or(q5h01, qxs01);+ vector signed char q5x10 = vec_or(q5h10, qxs10);+ vector signed char q5x11 = vec_or(q5h11, qxs11);++ vector signed char q8y00 = vec_xl( 0, q8);+ vector signed char q8y10 = vec_xl(16, q8);+ vector signed char q8y01 = vec_xl(32, q8);+ vector signed char q8y11 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv00 = vec_add(vec_mule(q5x00, q8y00), vec_mulo(q5x00, q8y00));+ vector signed short qv01 = vec_add(vec_mule(q5x01, q8y01), vec_mulo(q5x01, q8y01));+ vector signed short qv10 = vec_add(vec_mule(q5x10, q8y10), vec_mulo(q5x10, q8y10));+ vector signed short qv11 = vec_add(vec_mule(q5x11, q8y11), vec_mulo(q5x11, q8y11));++ vector signed short vs0 = vec_splat(vscales, 0);+ vector signed short vs1 = vec_splat(vscales, 1);+ vscales = vec_sld(vscales, vscales, 12);++ qv00 = vec_add(qv00, qv10);+ qv01 = vec_add(qv01, qv11);++ vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0);+ vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1);+ vsumi2 = vec_add(vec_mule(qv01, vs1), vsumi2);+ vsumi3 = vec_add(vec_mulo(qv01, vs1), vsumi3);+ }++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);+ const __m128i mzero = __lsx_vldi(0);+ const __m256i mone = __lasx_xvreplgr2vr_b(1);++ __m256 acc = (__m256)__lasx_xvldi(0);++ float summs = 0.f;++ for (int i = 0; i < nb; ++i) {++ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++#if QK_K == 256+ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin);++ memcpy(utmp, x[i].scales, 12);+#else+ // TODO+ const float d = 0, dmin = 0;+#endif++ const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]));++ const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);+ const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));+ const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s);+ const __m128i hsum = lsx_hadd_w(lsx_hadd_w(prod, mzero), mzero);+ summs += dmin * __lsx_vpickve2gr_w(hsum, 0); //TODO check++ const __m128i sc128 = lasx_extracti128(mins_and_scales, 0);+ const __m256i scales = lasx_insertf128(sc128, sc128);++ const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0);+ __m256i hmask = mone;++ __m256i sumi = __lasx_xvldi(0);++ int bit = 0;++ for (int j = 0; j < QK_K/64; ++j) {++ const __m256i scale_0 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0));+ const __m256i scale_1 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1));++ const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32;++ const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4);+ const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvand_v(hbits, hmask), bit++), 4);+ const __m256i q5_0 = __lasx_xvadd_b(q5l_0, q5h_0);+ hmask = __lasx_xvslli_h(hmask, 1);++ const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4);+ const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvand_v(hbits, hmask), bit++), 4);+ const __m256i q5_1 = __lasx_xvadd_b(q5l_1, q5h_1);+ hmask = __lasx_xvslli_h(hmask, 1);++ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;++ __m256i p16_0 = lasx_maddubs_h(q5_0, q8_0);+ __m256i p16_1 = lasx_maddubs_h(q5_1, q8_1);++ p16_0 = lasx_madd_h(scale_0, p16_0);+ p16_1 = lasx_madd_h(scale_1, p16_1);++ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));+ }++ __m256 vd = __lasx_xvreplfr2vr_s(d);+ acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);+ }++ *s = hsum_float_8(acc) + summs;++#else++ const uint8_t * scales = (const uint8_t*)&utmp[0];+ const uint8_t * mins = (const uint8_t*)&utmp[2];++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].qs;+ const uint8_t * restrict hm = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ uint8_t m = 1;+ for (int j = 0; j < QK_K/64; ++j) {+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);+ for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);+ a += 32; m <<= 1;+ q4 += 32;+ }+ memcpy(utmp, x[i].scales, 12);+ utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);+ const uint32_t uaux = utmp[1] & kmask1;+ utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);+ utmp[2] = uaux;+ utmp[0] &= kmask1;++ int sumi = 0;+ for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];+ a = aux8;+ int is = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ int32_t scale = scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];q8 += 8; a += 8;for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];@@ -7566,6 +9912,129 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v1 = vec_splats((unsigned char)0x1);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++#pragma GCC unroll 2+ for (int i = 0; i < nb; ++i) {+ __builtin_prefetch(x[i].qs, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd= vec_mul(vxd, vyd);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs);+ vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs);+ vector signed char qxs00 = (vector signed char)vec_and(qxs0, lowMask);+ vector signed char qxs01 = (vector signed char)vec_sr(qxs0, v4);+ vector signed char qxs10 = (vector signed char)vec_and(qxs1, lowMask);+ vector signed char qxs11 = (vector signed char)vec_sr(qxs1, v4);++ vector signed char qxhs = (vector signed char)vec_xl_len(x[i].qh, 8);+ vector signed char qxhs0 = vec_or(qxhs, vec_sr(vec_sld(qxhs, qxhs, 8), v1));+ vector signed char qxhs1 = vec_sr(qxhs0, v2);+ vector signed char qxh00 = vec_sl(vec_andc((vector signed char)v1, qxhs0), v4);+ vector signed char qxh10 = vec_sl(vec_andc((vector signed char)v1, qxhs1), v4);+ vector signed char qxh01 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs0, v4)), v4);+ vector signed char qxh11 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs1, v4)), v4);++ vector signed char q5x00 = vec_sub(qxs00, qxh00);+ vector signed char q5x10 = vec_sub(qxs10, qxh10);+ vector signed char q5x01 = vec_sub(qxs01, qxh01);+ vector signed char q5x11 = vec_sub(qxs11, qxh11);++ vector signed char q8y00 = vec_xl( 0, y[i].qs);+ vector signed char q8y10 = vec_xl(16, y[i].qs);+ vector signed char q8y01 = vec_xl(32, y[i].qs);+ vector signed char q8y11 = vec_xl(48, y[i].qs);++ vector signed short qv00 = vec_add(vec_mule(q5x00, q8y00), vec_mulo(q5x00, q8y00));+ vector signed short qv01 = vec_add(vec_mule(q5x01, q8y01), vec_mulo(q5x01, q8y01));+ vector signed short qv10 = vec_add(vec_mule(q5x10, q8y10), vec_mulo(q5x10, q8y10));+ vector signed short qv11 = vec_add(vec_mule(q5x11, q8y11), vec_mulo(q5x11, q8y11));++ vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4));+ vector signed short vs0 = vec_splat(vs, 0);+ vector signed short vs1 = vec_splat(vs, 1);+ vector signed short vs2 = vec_splat(vs, 2);+ vector signed short vs3 = vec_splat(vs, 3);++ vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0));+ vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1));+ vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2));+ vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);+ const __m256i mone = __lasx_xvreplgr2vr_b(1);++ __m256 acc = (__m256)__lasx_xvldi(0);++ for (int i = 0; i < nb; ++i) {++ const uint8_t * restrict q5 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0);++ const __m256i scale_l = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[1]), __lsx_vreplgr2vr_h(x[i].scales[0]));+ const __m256i scale_h = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[3]), __lsx_vreplgr2vr_h(x[i].scales[2]));++ int64_t aux64;+ memcpy(&aux64, x[i].qh, 8);+ __m128i haux128 = __lsx_vinsgr2vr_d(haux128, aux64, 0);+ haux128 = __lsx_vinsgr2vr_d(haux128, aux64 >> 1, 1);+ const __m256i haux256 = lasx_insertf128(__lsx_vsrli_h(haux128, 2), haux128);++ const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvandn_v(haux256, mone), 4);+ const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvandn_v(__lasx_xvsrli_h(haux256, 4), mone), 4);++ const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4);+ const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4);++ const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0);+ const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0);++ const __m256i p16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5l_0, q8_0));+ const __m256i p16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5l_1, q8_1));+ const __m256i s16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5h_0, q8_0));+ const __m256i s16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5h_1, q8_1));++ const __m256i dot = __lasx_xvsub_w(__lasx_xvadd_w(p16_0, p16_1), __lasx_xvadd_w(s16_0, s16_1));++ acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(dot), acc);+ }++ *s = hsum_float_8(acc);+#elseint8_t aux8[QK_K];@@ -7996,6 +10465,229 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);+ const vector signed char off = vec_splats((signed char)0x20);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint8_t * restrict q6 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict qs = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/128; ++j) {+ __builtin_prefetch(q6, 0, 0);+ __builtin_prefetch(qh, 0, 0);+ __builtin_prefetch(q8, 0, 0);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q6);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q6);+ vector signed char qxs2 = (vector signed char)vec_xl(32, q6);+ vector signed char qxs3 = (vector signed char)vec_xl(48, q6);+ q6 += 64;++ vector signed char qxs00 = vec_and(qxs0, lowMask);+ vector signed char qxs01 = vec_sr(qxs0, v4);+ vector signed char qxs10 = vec_and(qxs1, lowMask);+ vector signed char qxs11 = vec_sr(qxs1, v4);+ vector signed char qxs20 = vec_and(qxs2, lowMask);+ vector signed char qxs21 = vec_sr(qxs2, v4);+ vector signed char qxs30 = vec_and(qxs3, lowMask);+ vector signed char qxs31 = vec_sr(qxs3, v4);++ vector signed char qxhs0 = (vector signed char)vec_xl( 0, qh);+ vector signed char qxhs1 = (vector signed char)vec_xl(16, qh);+ qh += 32;++ vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4);+ vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4);+ vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, qxhs1), v4);+ vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v4)), v4);+ vector signed char qxh20 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4);+ vector signed char qxh21 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4);+ vector signed char qxh30 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v2)), v4);+ vector signed char qxh31 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v6)), v4);++ vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off);+ vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off);+ vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off);+ vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off);+ vector signed char q6x20 = vec_sub(vec_or(qxh20, qxs20), off);+ vector signed char q6x21 = vec_sub(vec_or(qxh21, qxs21), off);+ vector signed char q6x30 = vec_sub(vec_or(qxh30, qxs30), off);+ vector signed char q6x31 = vec_sub(vec_or(qxh31, qxs31), off);++ vector signed char q8y00 = vec_xl( 0, q8);+ vector signed char q8y10 = vec_xl( 16, q8);+ vector signed char q8y20 = vec_xl( 32, q8);+ vector signed char q8y30 = vec_xl( 48, q8);+ vector signed char q8y01 = vec_xl( 64, q8);+ vector signed char q8y11 = vec_xl( 80, q8);+ vector signed char q8y21 = vec_xl( 96, q8);+ vector signed char q8y31 = vec_xl(112, q8);+ q8 += 128;++ vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00));+ vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10));+ vector signed short qv20 = vec_add(vec_mule(q6x20, q8y20), vec_mulo(q6x20, q8y20));+ vector signed short qv30 = vec_add(vec_mule(q6x30, q8y30), vec_mulo(q6x30, q8y30));+ vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01));+ vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11));+ vector signed short qv21 = vec_add(vec_mule(q6x21, q8y21), vec_mulo(q6x21, q8y21));+ vector signed short qv31 = vec_add(vec_mule(q6x31, q8y31), vec_mulo(q6x31, q8y31));++ vector signed short vscales = vec_unpackh(vec_xl_len(qs, 8));+ qs += 8;++ vector signed short vs0 = vec_splat(vscales, 0);+ vector signed short vs1 = vec_splat(vscales, 1);+ vector signed short vs2 = vec_splat(vscales, 2);+ vector signed short vs3 = vec_splat(vscales, 3);+ vector signed short vs4 = vec_splat(vscales, 4);+ vector signed short vs5 = vec_splat(vscales, 5);+ vector signed short vs6 = vec_splat(vscales, 6);+ vector signed short vs7 = vec_splat(vscales, 7);++ vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0);+ vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1);+ vsumi2 = vec_add(vec_mule(qv01, vs4), vsumi2);+ vsumi3 = vec_add(vec_mulo(qv01, vs4), vsumi3);+ vsumi4 = vec_add(vec_mule(qv10, vs1), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv10, vs1), vsumi5);+ vsumi6 = vec_add(vec_mule(qv11, vs5), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv11, vs5), vsumi7);++ vsumi0 = vec_add(vec_mule(qv20, vs2), vsumi0);+ vsumi1 = vec_add(vec_mulo(qv20, vs2), vsumi1);+ vsumi2 = vec_add(vec_mule(qv21, vs6), vsumi2);+ vsumi3 = vec_add(vec_mulo(qv21, vs6), vsumi3);+ vsumi4 = vec_add(vec_mule(qv30, vs3), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv30, vs3), vsumi5);+ vsumi6 = vec_add(vec_mule(qv31, vs7), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv31, vs7), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);+ const __m256i m2 = __lasx_xvreplgr2vr_b(3);+ const __m256i m32s = __lasx_xvreplgr2vr_b(32);++ __m256 acc = (__m256)__lasx_xvldi(0);++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const __m128i scales = __lsx_vld((const __m128i*)x[i].scales, 0);++ __m256i sumi = __lasx_xvldi(0);++ int is = 0;++ for (int j = 0; j < QK_K/128; ++j) {++ const __m128i scale_0 = lsx_shuffle_b(scales, get_scale_shuffle(is + 0));+ const __m128i scale_1 = lsx_shuffle_b(scales, get_scale_shuffle(is + 1));+ const __m128i scale_2 = lsx_shuffle_b(scales, get_scale_shuffle(is + 2));+ const __m128i scale_3 = lsx_shuffle_b(scales, get_scale_shuffle(is + 3));+ is += 4;++ const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;+ const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;+ const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32;++ const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(q4bitsH, m2), 4);+ const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 2), m2), 4);+ const __m256i q4h_2 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 4), m2), 4);+ const __m256i q4h_3 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 6), m2), 4);++ const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0);+ const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(q4bits2, m4), q4h_1);+ const __m256i q4_2 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_2);+ const __m256i q4_3 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits2, 4), m4), q4h_3);++ const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;++ __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0);+ __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1);+ __m256i q8s_2 = lasx_maddubs_h(m32s, q8_2);+ __m256i q8s_3 = lasx_maddubs_h(m32s, q8_3);++ __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0);+ __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1);+ __m256i p16_2 = lasx_maddubs_h(q4_2, q8_2);+ __m256i p16_3 = lasx_maddubs_h(q4_3, q8_3);++ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);+ p16_2 = __lasx_xvsub_h(p16_2, q8s_2);+ p16_3 = __lasx_xvsub_h(p16_3, q8s_3);++ p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0);+ p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1);+ p16_2 = lasx_madd_h(lasx_ext8_16(scale_2), p16_2);+ p16_3 = lasx_madd_h(lasx_ext8_16(scale_3), p16_3);++ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));+ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3));+ }++ acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);+ }++ *s = hsum_float_8(acc);+#elseint8_t aux8[QK_K];@@ -8302,55 +10994,193 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r*s = sumf;-#else+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v2 = vec_splats((unsigned char)0x2);+ const vector unsigned char v3 = vec_splats((unsigned char)0x3);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);+ const vector unsigned char v6 = vec_splats((unsigned char)0x6);+ const vector signed char off = vec_splats((signed char)0x20);- int8_t aux8[QK_K];- int16_t aux16[8];- float sums [8];- int32_t aux32[8];- memset(sums, 0, 8*sizeof(float));+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);- float sumf = 0;+#pragma GCC unroll 2for (int i = 0; i < nb; ++i) {- const uint8_t * restrict q4 = x[i].ql;- const uint8_t * restrict qh = x[i].qh;- const int8_t * restrict q8 = y[i].qs;- memset(aux32, 0, 8*sizeof(int32_t));- int8_t * restrict a = aux8;- for (int l = 0; l < 16; ++l) {- a[l+ 0] = (int8_t)((q4[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;- a[l+16] = (int8_t)((q4[l+16] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;- a[l+32] = (int8_t)((q4[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;- a[l+48] = (int8_t)((q4[l+16] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;- }- int is = 0;- for (int j = 0; j < QK_K/16; ++j) {- int scale = x[i].scales[is++];- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];- q8 += 8; a += 8;- }- const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];- }- for (int l = 0; l < 8; ++l) sumf += sums[l];- *s = sumf;-#endif-}+ __builtin_prefetch(x[i].ql, 0, 1);+ __builtin_prefetch(x[i].qh, 0, 1);+ __builtin_prefetch(y[i].qs, 0, 1);-#endif+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd= vec_mul(vxd, vyd);-#if defined (__AVX2__) || defined (__ARM_NEON)-static const int8_t keven_signs_q2xs[1024] = {- 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,- 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,- 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,- 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,- 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,- 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,+ vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].ql);+ vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].ql);+ vector signed char qxs00 = vec_and(qxs0, lowMask);+ vector signed char qxs01 = vec_sr(qxs0, v4);+ vector signed char qxs10 = vec_and(qxs1, lowMask);+ vector signed char qxs11 = vec_sr(qxs1, v4);++ vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh);++ vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4);+ vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4);+ vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4);+ vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4);++ vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off);+ vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off);+ vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off);+ vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off);++ vector signed char q8y00 = vec_xl( 0, y[i].qs);+ vector signed char q8y10 = vec_xl(16, y[i].qs);+ vector signed char q8y01 = vec_xl(32, y[i].qs);+ vector signed char q8y11 = vec_xl(48, y[i].qs);++ vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00));+ vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10));+ vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01));+ vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11));++ vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4));+ vector signed short vs0 = vec_splat(vs, 0);+ vector signed short vs1 = vec_splat(vs, 1);+ vector signed short vs2 = vec_splat(vs, 2);+ vector signed short vs3 = vec_splat(vs, 3);++ vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0));+ vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1));+ vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2));+ vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined __loongarch_asx++ const __m256i m4 = __lasx_xvreplgr2vr_b(0xF);+ const __m256i m2 = __lasx_xvreplgr2vr_b(3);+ const __m256i m32s = __lasx_xvreplgr2vr_b(32);++ __m256 acc = (__m256)__lasx_xvldi(0);++ for (int i = 0; i < nb; ++i) {++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);++ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;++ const __m64 scales_1 = __lasx_xvreplgr2vr_b(x[i].scales[0]);+ const __m64 scales_2 = __lasx_xvreplgr2vr_b(x[i].scales[1]);+ const __m64 scales_3 = __lasx_xvreplgr2vr_b(x[i].scales[2]);+ const __m64 scales_4 = __lasx_xvreplgr2vr_b(x[i].scales[3]);++ __m256i sumi = __lasx_xvldi(0);++ __m128i scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_1, 0);+ scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_2, 1);+ __m128i scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_3, 0);+ scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_4, 1);++ const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0);+ const __m128i q4bitsH = __lsx_vld((const __m128i*)qh, 0);++ const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 2), q4bitsH), m2), 4);+ const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 6), __lasx_xvsrli_h(q4bitsH, 4)), m2), 4);++ const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0);+ const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_1);++ const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0);+ const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0);++ __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0);+ __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1);++ __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0);+ __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1);++ p16_0 = __lasx_xvsub_h(p16_0, q8s_0);+ p16_1 = __lasx_xvsub_h(p16_1, q8s_1);++ p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0);+ p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1);++ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));++ acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);+ }++ *s = hsum_float_8(acc);++#else++ int8_t aux8[QK_K];+ int16_t aux16[8];+ float sums [8];+ int32_t aux32[8];+ memset(sums, 0, 8*sizeof(float));++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const uint8_t * restrict q4 = x[i].ql;+ const uint8_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;+ memset(aux32, 0, 8*sizeof(int32_t));+ int8_t * restrict a = aux8;+ for (int l = 0; l < 16; ++l) {+ a[l+ 0] = (int8_t)((q4[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;+ a[l+16] = (int8_t)((q4[l+16] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;+ a[l+32] = (int8_t)((q4[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;+ a[l+48] = (int8_t)((q4[l+16] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;+ }+ int is = 0;+ for (int j = 0; j < QK_K/16; ++j) {+ int scale = x[i].scales[is++];+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];+ for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];+ q8 += 8; a += 8;+ }+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];+ }+ for (int l = 0; l < 8; ++l) sumf += sums[l];+ *s = sumf;+#endif+}++#endif++#if defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) || defined(__loongarch_asx)+static const int8_t keven_signs_q2xs[1024] = {+ 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1,+ 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1,+ 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, -1,+ 1, 1, -1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, 1, 1,+ 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, -1,+ 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1,1, 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, 1,1, 1, -1, -1, -1, 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, 1, 1, -1,1, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, -1, 1, -1,@@ -8476,6 +11306,146 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void*s = 0.125f * hsum_float_8(accumf);+#elif defined(__POWER9_VECTOR__)+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/32; j += 2) {+ __builtin_prefetch(q2, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ memcpy(aux32, q2, 4*sizeof(uint32_t));+ q2 += 8;++ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1])};+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3])};+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9])};+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11])};++ vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127))};+ vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127))};+ vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127))};+ vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127))};++ vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);+ vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);+ vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);+ vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));++ const uint16_t ls0 = aux32[1] >> 28;+ const uint16_t ls1 = aux32[3] >> 28;++ vector signed short vscales01 = vec_splats((int16_t)(2*ls0+1));+ vector signed short vscales23 = vec_splats((int16_t)(2*ls1+1));++ vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = 0.125f * vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[4];+ const uint8_t * aux8 = (const uint8_t *)aux32;++ __m256 accumf = (__m256)__lasx_xvldi(0);+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;++ const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);+ const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);+ const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],+ signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]);+ const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);+ const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);+ const uint16_t ls1 = aux32[1] >> 28;+ const uint16_t ls2 = aux32[3] >> 28;+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));+ sumi1 = __lasx_xvadd_w(sumi1, p1);+ sumi2 = __lasx_xvadd_w(sumi2, p2);+ }++ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);+ }++ *s = 0.125f * hsum_float_8(accumf);+#elseuint32_t aux32[2];@@ -8750,7 +11720,280 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void **s = 0.125f * hsum_float_8(accumf);#endif+#elif defined(__loongarch_asx)++ const __m256i mone = __lasx_xvreplgr2vr_b(1);+ static const char block_sign_shuffle_mask_1[32] = {+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,+ 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,+ };+ static const char block_sign_shuffle_mask_2[32] = {+ 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,+ };+ static const uint8_t bit_selector_mask_bytes[32] = {+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0);+ const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0);+ const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0);++#if QK_K == 64+ static const uint8_t k_bit_helper[16] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m128i bit_helper = __lsx_vld((const __m128i*)k_bit_helper, 0);+ const __m128i m511 = __lsx_vreplgr2vr_h(511);+ typedef union {+ __m128i vec_index;+ uint16_t index[8];+ } index_t;++ index_t idx;+ __m256 accumf = (__m256)__lasx_xvldi(0);+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const __m128i q2_data = __lsx_vld((const __m128i*)x[i].qs, 0);+ idx.vec_index = __lsx_vand_v(q2_data, m511);++ const __m128i partial_sign_bits = __lsx_vsrli_h(q2_data, 9);+ const __m128i partial_sign_bits_upper = __lsx_vsrli_h(q2_data, 13);+ const __m128i partial_sign_bits_for_counting = __lsx_vxor_v(partial_sign_bits, partial_sign_bits_upper);++ const __m128i odd_bits = lsx_shuffle_b(bit_helper, partial_sign_bits_for_counting);+ const __m128i full_sign_bits = __lsx_vor_v(partial_sign_bits, odd_bits);+ const __m256i full_signs = lasx_insertf128(full_sign_bits, full_sign_bits);++ const __m256i q8_1 = __lasx_xvld((const __m256i *)y[i].qs, 0);+ const __m256i q8_2 = __lasx_xvld((const __m256i *)(y[i].qs+32), 0);++ const __m256i q2_1 = lasx_set_d(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]],+ iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]);+ const __m256i q2_2 = lasx_set_d(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]],+ iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]);+ __m256i signs;+ signs = lasx_shuffle_b(full_signs, block_sign_shuffle_1);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);++ signs = lasx_shuffle_b(full_signs, block_sign_shuffle_2);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);++ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);++ const __m256i sc1 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[0] & 0xf)+1));+ const __m256i sc2 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[1] & 0xf)+1));++ const __m256i sum = __lasx_xvadd_w(lasx_madd_h(sc1, dot1), lasx_madd_h(sc2, dot2));++ accumf = __lasx_vfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sum), accumf);+ }++ *s = 0.125f * hsum_float_8(accumf);+#else++ static const uint8_t k_bit_helper[32] = {+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,+ };+ const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0);+ const __m256i m511 = __lasx_xvreplgr2vr_h(511);+ const __m128i m4 = __lsx_vreplgr2vr_b(0xf);+ const __m128i m1 = __lsx_vreplgr2vr_b(1);++ uint64_t aux64;++ // somewhat hacky, but gives a significant boost in performance+ __m256i aux_gindex;+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;++ __m256 accumf = (__m256)__lasx_xvldi(0);+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ __m128i stmp = __lsx_vreplgr2vr_d(aux64);+ stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4));+ const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1);++ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {++ const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0); q2 += 16;+ aux_gindex = __lasx_xvand_v(q2_data, m511);++ const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9);+ const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13);+ const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper);++ const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting);+ const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits);++ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;++ const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],+ iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);+ const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],+ iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);+ const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],+ iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);+ const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],+ iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);++ const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0);+ const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1);+ const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l);+ const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h);++ __m256i signs;+ signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);++ signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);++ signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3);++ signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2);+ signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);+ const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4);++ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);+ const __m256i dot3 = lasx_maddubs_h(q2_3, q8s_3);+ const __m256i dot4 = lasx_maddubs_h(q2_4, q8s_4);++ const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0)));+ const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1)));+ const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2)));+ const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3)));++ sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1));+ sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2));+ sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3));+ sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4));+ }++ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);+#endif+++#elif defined(__POWER9_VECTOR__)+ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint16_t * restrict q2 = x[i].qs;+ const uint8_t * restrict sc = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/64; ++j) {+ __builtin_prefetch(q2, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xs_grid + (q2[0] & 511)), *(const int64_t *)(iq2xs_grid + (q2[1] & 511))};+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xs_grid + (q2[2] & 511)), *(const int64_t *)(iq2xs_grid + (q2[3] & 511))};+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xs_grid + (q2[4] & 511)), *(const int64_t *)(iq2xs_grid + (q2[5] & 511))};+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xs_grid + (q2[6] & 511)), *(const int64_t *)(iq2xs_grid + (q2[7] & 511))};++ vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((q2[0] >> 9))), *(const int64_t *)(signs64 + ((q2[1] >> 9)))};+ vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((q2[2] >> 9))), *(const int64_t *)(signs64 + ((q2[3] >> 9)))};+ vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((q2[4] >> 9))), *(const int64_t *)(signs64 + ((q2[5] >> 9)))};+ vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((q2[6] >> 9))), *(const int64_t *)(signs64 + ((q2[7] >> 9)))};+ q2 += 8;++ vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);+ vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);+ vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);+ vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));++ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);+ const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);+ const uint16_t ls3 = (uint16_t)(sc[1] >> 4);+ sc += 2;++ vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));+ vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));+ vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));+ vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));++ vsumi0 = vec_add(vec_mule(qv0, vscales0), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales1), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales2), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales3), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales0), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales1), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales2), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales3), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);+ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = 0.125f * vec_extract(vsumf0, 0);#elsefloat sumf = 0.f;@@ -8951,6 +12194,199 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void **s = 0.125f * hsum_float_8(accumf);+#elif defined(__POWER9_VECTOR__)+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ const vector unsigned char mask0 = vec_xl( 0, k_mask1);+ const vector unsigned char mask1 = vec_xl(16, k_mask1);+ const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint8_t * restrict q2 = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);+ const uint8_t * restrict sc = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;++ for (int j = 0; j < QK_K/32; j += 2) {+ __builtin_prefetch(q2, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed long long aux64x2_0 = {*(const int64_t *)(iq2s_grid + (q2[0] | ((qh[0] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[1] | ((qh[0] << 6) & 0x300)))};+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq2s_grid + (q2[2] | ((qh[0] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[3] | ((qh[0] << 2) & 0x300)))};+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq2s_grid + (q2[4] | ((qh[1] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[5] | ((qh[1] << 6) & 0x300)))};+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq2s_grid + (q2[6] | ((qh[1] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[7] | ((qh[1] << 2) & 0x300)))};+ q2 += 8;+ qh += 2;++ vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);+ vector signed char vsigns23 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);+ signs += 4;++ vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);+ vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);+ vector signed char vsigns2 = vec_perm(vsigns23, vsigns23, mask0);+ vector signed char vsigns3 = vec_perm(vsigns23, vsigns23, mask1);++ vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);+ vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);+ vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);+ vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);++ vector signed char q2x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux64x2_0), vsigns0);+ vector signed char q2x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux64x2_1), vsigns1);+ vector signed char q2x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux64x2_2), vsigns2);+ vector signed char q2x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux64x2_3), vsigns3);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));++ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);+ const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);+ const uint16_t ls3 = (uint16_t)(sc[1] >> 4);+ sc += 2;++ vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));+ vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));+ vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));+ vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));++ vsumi0 = vec_add(vec_mule(qv0, vscales0), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales1), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales2), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales3), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales0), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales1), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales2), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales3), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = 0.125f * vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)++ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };+++ const __m128i m4 = __lsx_vreplgr2vr_b(0xf);+ const __m128i m1 = __lsx_vreplgr2vr_b(1);++ const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);+ const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);+ uint64_t aux64;++ __m256 accumf = (__m256)__lasx_xvldi(0);+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict qs = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8);+ const int8_t * restrict q8 = y[i].qs;++ __m128i tmp1;+ memcpy(&aux64, x[i].scales, 8);+ tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0);+ tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1);+ const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1);+ const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15++ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],+ iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],+ iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],+ iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);+ const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],+ iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],+ iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],+ iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);+ qs += 8;++ __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16));+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);+ const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);+ const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);++ aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16));+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);+ const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);+ const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);++ signs += 4;++ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3++ const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0)));+ const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1)));+ sumi1 = __lasx_xvadd_w(sumi1, p1);+ sumi2 = __lasx_xvadd_w(sumi2, p2);+ }++ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);+ }++ *s = 0.125f * hsum_float_8(accumf);+#elsefloat sumf = 0;@@ -9045,52 +12481,195 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const voidsumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28));sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[1] >> 28));}- sumf += d*(sumf1 + sumf2);+ sumf += d*(sumf1 + sumf2);+ }+ *s = 0.5f * sumf;++#elif defined(__AVX2__)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint32_t aux32[2];++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict gas = x[i].qs + QK_K/4;+ const int8_t * restrict q8 = y[i].qs;+ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ q3 += 8;+ memcpy(aux32, gas, 8); gas += 8;+ const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],+ signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const uint16_t ls1 = aux32[0] >> 28;+ const uint16_t ls2 = aux32[1] >> 28;+ const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));+ const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));+ sumi1 = _mm256_add_epi32(sumi1, p1);+ sumi2 = _mm256_add_epi32(sumi2, p2);+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.25f * hsum_float_8(accumf);++#elif defined(__POWER9_VECTOR__)+ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ const uint8_t * restrict q3 = x[i].qs;+ const uint32_t * restrict signs = (const uint32_t *)(x[i].qs + QK_K/4);+ const int8_t * restrict q8 = y[i].qs;++#pragma GCC unroll 1+ for (int j = 0; j < QK_K/32; j += 2) {+ __builtin_prefetch(q3, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector unsigned int aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]};+ vector unsigned int aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]};+ vector unsigned int aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]};+ vector unsigned int aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]};+ q3 += 16;++ vector unsigned long long aux64x2_0 = {(uint64_t)(signs64[(signs[0] >> 0) & 127]), (uint64_t)(signs64[(signs[0] >> 7) & 127])};+ vector unsigned long long aux64x2_1 = {(uint64_t)(signs64[(signs[0] >> 14) & 127]), (uint64_t)(signs64[(signs[0] >> 21) & 127])};+ vector unsigned long long aux64x2_2 = {(uint64_t)(signs64[(signs[1] >> 0) & 127]), (uint64_t)(signs64[(signs[1] >> 7) & 127])};+ vector unsigned long long aux64x2_3 = {(uint64_t)(signs64[(signs[1] >> 14) & 127]), (uint64_t)(signs64[(signs[1] >> 21) & 127])};++ vector signed char q3x0 = vec_mul((vector signed char)aux64x2_0, (vector signed char)aux32x4_0);+ vector signed char q3x1 = vec_mul((vector signed char)aux64x2_1, (vector signed char)aux32x4_1);+ vector signed char q3x2 = vec_mul((vector signed char)aux64x2_2, (vector signed char)aux32x4_2);+ vector signed char q3x3 = vec_mul((vector signed char)aux64x2_3, (vector signed char)aux32x4_3);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));++ const uint16_t ls0 = (uint16_t)(signs[0] >> 28);+ const uint16_t ls1 = (uint16_t)(signs[1] >> 28);+ signs += 2;++ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));++ vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);}- *s = 0.5f * sumf;-#elif defined(__AVX2__)+ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = 0.25f * vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;uint32_t aux32[2];- __m256 accumf = _mm256_setzero_ps();+ __m256 accumf = (__m256)__lasx_xvldi(0);for (int i = 0; i < nb; ++i) {const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;const uint8_t * restrict q3 = x[i].qs;const uint8_t * restrict gas = x[i].qs + QK_K/4;const int8_t * restrict q8 = y[i].qs;- __m256i sumi1 = _mm256_setzero_si256();- __m256i sumi2 = _mm256_setzero_si256();+ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {- const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;- const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],- iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);q3 += 8;- const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],- iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);+ const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],+ iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);q3 += 8;memcpy(aux32, gas, 8); gas += 8;- const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],++ const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]);- const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],+ const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]);- const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);- const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);- const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);- const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);+ const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);+ const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);+ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);const uint16_t ls1 = aux32[0] >> 28;const uint16_t ls2 = aux32[1] >> 28;- const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));- const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));- sumi1 = _mm256_add_epi32(sumi1, p1);- sumi2 = _mm256_add_epi32(sumi2, p2);- }- accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));+ sumi1 = __lasx_xvadd_w(sumi1, p1);+ sumi2 = __lasx_xvadd_w(sumi2, p2);+ }+ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);}*s = 0.25f * hsum_float_8(accumf);@@ -9322,6 +12901,207 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void **s = hsum_float_8(accumf);+#elif defined(__POWER9_VECTOR__)+ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ const vector unsigned char mask0 = vec_xl( 0, k_mask1);+ const vector unsigned char mask1 = vec_xl(16, k_mask1);+ const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ const uint8_t * restrict q3 = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const uint16_t * restrict signs = (const uint16_t *)(x[i].signs);+ const uint8_t * restrict sc = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ for (int j = 0; j < QK_K/32; j += 2) {+ __builtin_prefetch(q3, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector unsigned int aux32x4_0 = {iq3s_grid[q3[ 0] | ((qh[0] << 8) & 256)], iq3s_grid[q3[ 1] | ((qh[0] << 7) & 256)],+ iq3s_grid[q3[ 2] | ((qh[0] << 6) & 256)], iq3s_grid[q3[ 3] | ((qh[0] << 5) & 256)]};+ vector unsigned int aux32x4_1 = {iq3s_grid[q3[ 4] | ((qh[0] << 4) & 256)], iq3s_grid[q3[ 5] | ((qh[0] << 3) & 256)],+ iq3s_grid[q3[ 6] | ((qh[0] << 2) & 256)], iq3s_grid[q3[ 7] | ((qh[0] << 1) & 256)]};+ vector unsigned int aux32x4_2 = {iq3s_grid[q3[ 8] | ((qh[1] << 8) & 256)], iq3s_grid[q3[ 9] | ((qh[1] << 7) & 256)],+ iq3s_grid[q3[10] | ((qh[1] << 6) & 256)], iq3s_grid[q3[11] | ((qh[1] << 5) & 256)]};+ vector unsigned int aux32x4_3 = {iq3s_grid[q3[12] | ((qh[1] << 4) & 256)], iq3s_grid[q3[13] | ((qh[1] << 3) & 256)],+ iq3s_grid[q3[14] | ((qh[1] << 2) & 256)], iq3s_grid[q3[15] | ((qh[1] << 1) & 256)]};+ q3 += 16;+ qh += 2;++ vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);+ vector signed char vsigns02 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);+ signs += 4;++ vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);+ vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);+ vector signed char vsigns2 = vec_perm(vsigns02, vsigns02, mask0);+ vector signed char vsigns3 = vec_perm(vsigns02, vsigns02, mask1);++ vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);+ vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);+ vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);+ vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);++ vector signed char q3x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux32x4_0), vsigns0);+ vector signed char q3x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux32x4_1), vsigns1);+ vector signed char q3x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux32x4_2), vsigns2);+ vector signed char q3x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux32x4_3), vsigns3);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));++ const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);+ const uint16_t ls1 = (uint16_t)(sc[0] >> 4);+ sc ++;++ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));++ vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)++ static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03+ };++ static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ };++ const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);+ const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);++ __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8);+ const __m256i idx_mask = __lasx_xvreplgr2vr_w(256);++ typedef union {+ __m256i vec[2];+ uint32_t index[16];+ } index_t;++ index_t idx;++ __m256 accumf = (__m256)__lasx_xvldi(0);+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint8_t * restrict qs = x[i].qs;+ const uint8_t * restrict qh = x[i].qh;+ const uint16_t * restrict signs = (const uint16_t *)x[i].signs;+ const int8_t * restrict q8 = y[i].qs;+ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16;+ idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]);+ idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]);+ idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask);+ idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask);+ idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0)));+ idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1)));++ // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.+ //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);+ //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);+ const __m256i q2_1 = lasx_set_w(+ iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],+ iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]+ );+ const __m256i q2_2 = lasx_set_w(+ iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],+ iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]+ );++ __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16));+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);+ const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);+ const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);++ aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16));+ aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);+ const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);+ const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);++ signs += 4;++ const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);+ const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2);+ const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;+ const uint16_t ls2 = x[i].scales[ib32/2] >> 4;+ const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));+ const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));+ sumi1 = __lasx_xvadd_w(sumi1, p1);+ sumi2 = __lasx_xvadd_w(sumi2, p2);+ }++ accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);+ }++ *s = hsum_float_8(accumf);+#elsefloat sumf = 0.f;@@ -9369,12 +13149,22 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void *}-#ifdef __AVX2__+#if defined(__AVX2__)static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {const __m256i ax = _mm256_sign_epi8(x, x);const __m256i sy = _mm256_sign_epi8(y, x);return _mm256_maddubs_epi16(ax, sy);}+#elif defined(__loongarch_asx)+static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {+ const __m256i ax = __lasx_xvsigncov_b(x, x);+ const __m256i sy = __lasx_xvsigncov_b(x, y);+ __m256i tmp1, tmp2, tmp3;+ tmp1 = __lasx_xvmulwev_h_bu_b(ax, sy);+ tmp2 = __lasx_xvmulwod_h_bu_b(ax, sy);+ tmp3 = __lasx_xvadd_h(tmp1, tmp2);+ return __lasx_xvsat_h(tmp3, 15);+}#endifvoid ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {@@ -9476,6 +13266,169 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void*s = hsum_float_8(accum) + IQ1S_DELTA * accum1;+#elif defined(__POWER9_VECTOR__)+ const vector unsigned char v0 = vec_splats((unsigned char)0x0);+ const vector unsigned short vsign = vec_splats((unsigned short)0x8000);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ for (int i = 0; i < nb; ++i) {+ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));+ vector float vyd = vec_splats(y[i].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);+ vector signed int vsumi8 = vec_splats((int32_t)0);++ const uint8_t * restrict q1 = x[i].qs;+ const uint16_t * restrict qh = x[i].qh;+ const int8_t * restrict q8 = y[i].qs;+ const int16_t * restrict qs = y[i].bsums;++ for (int j = 0; j < QK_K/32; j += 2) {+ __builtin_prefetch(q1, 0, 1);+ __builtin_prefetch(qh, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed long long aux64x2_0 = {*(const int64_t *)(iq1s_grid + (q1[0] | ((qh[0] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[1] | ((qh[0] << 5) & 0x700)))};+ vector signed long long aux64x2_1 = {*(const int64_t *)(iq1s_grid + (q1[2] | ((qh[0] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[3] | ((qh[0] >> 1) & 0x700)))};+ vector signed long long aux64x2_2 = {*(const int64_t *)(iq1s_grid + (q1[4] | ((qh[1] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[5] | ((qh[1] << 5) & 0x700)))};+ vector signed long long aux64x2_3 = {*(const int64_t *)(iq1s_grid + (q1[6] | ((qh[1] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[7] | ((qh[1] >> 1) & 0x700)))};+ q1 += 8;++ vector signed char q1x0 = (vector signed char)aux64x2_0;+ vector signed char q1x1 = (vector signed char)aux64x2_1;+ vector signed char q1x2 = (vector signed char)aux64x2_2;+ vector signed char q1x3 = (vector signed char)aux64x2_3;++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q1x0, q8y0), vec_mulo(q1x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q1x1, q8y1), vec_mulo(q1x1, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q1x2, q8y2), vec_mulo(q1x2, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q1x3, q8y3), vec_mulo(q1x3, q8y3));++ const uint16_t ls0 = (uint16_t)((qh[0] >> 12) & 7);+ const uint16_t ls1 = (uint16_t)((qh[1] >> 12) & 7);++ vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));+ vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));+ vector signed short vscales = vec_sld(vscales23, vscales01, 8);++ vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7);++ vector signed short q8ysums = vec_xl_len(qs, 8);+ qs += 4;+ q8ysums = vec_mergeh(q8ysums, (vector signed short)v0);++ vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8);+ qh += 2;+ vector __bool short vsel = vec_cmpge(qxh, (vector signed short)v0);++ vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel);++ vsumi8 = vec_add(vec_mule(q8ysum, vscales), vsumi8);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);++ vsumf0 = vec_madd(vec_ctf(vsumi8, 0), vec_mul(vd, vec_splats(IQ1S_DELTA)), vsumf0);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)++ __m256 accum = (__m256)__lasx_xvldi(0);+ float accum1 = 0;+ for (int i = 0; i < nb; ++i) {++ const int8_t * q8 = y[i].qs;+ const uint8_t * qs = x[i].qs;+ const uint16_t * qh = x[i].qh;++ __m256i sumi = __lasx_xvldi(0);+ int sumi1 = 0;+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0);+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1);+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2);+ q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3);++ __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0);+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1);+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2);+ q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3);++ qs += 8;+ const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;+ const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;++ const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);+ const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);+ const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;+ const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;++ __m256i tmp1, tmp5, tmp6;+ tmp1 = __lasx_xvreplgr2vr_h(ls1);+ tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1);+ tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1);+ const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6);++ tmp1 = __lasx_xvreplgr2vr_h(ls2);+ tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1);+ tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1);+ const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6);++ sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2));+ sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1+ + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;+ }++ const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d);+ accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum);+ accum1 += d * sumi1;+ }++ *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;+#elsefloat sumf = 0;@@ -9832,6 +13785,84 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void **s = hsum_float_8(_mm256_add_ps(accum1, accum2));+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);++ const vector signed char values = vec_xl( 0, kvalues_iq4nl);++#pragma GCC unroll 4+ for (int ib = 0; ib < nb; ++ib) {+ __builtin_prefetch(x[ib].qs, 0, 1);+ __builtin_prefetch(y[ib].qs, 0, 1);+++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));+ vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));+ vector float vd = vec_mul(vxd, vyd);++ vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);+ vector signed char q4x0 = vec_and(qxs, lowMask);+ vector signed char q4x1 = vec_sr(qxs, v4);++ q4x0 = vec_perm(values, values, (vector unsigned char)q4x0);+ q4x1 = vec_perm(values, values, (vector unsigned char)q4x1);++ vector signed char q8y0 = vec_xl( 0, y[ib].qs);+ vector signed char q8y1 = vec_xl(16, y[ib].qs);++ vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));++ vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));+ vector signed int vsumi1 = vec_add(vec_unpackh(qv1), vec_unpackl(qv1));++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ }++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined (__loongarch_asx)++ const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);+ const __m128i m4b = __lsx_vreplgr2vr_b(0x0f);+ const __m256i mone = __lasx_xvreplgr2vr_h(1);++ __m256 accum1 = (__m256)__lasx_xvldi(0);+ __m256 accum2 = (__m256)__lasx_xvldi(0);+ for (int ib = 0; ib < nb; ib += 2) {+ const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[0].qs, 0);+ const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[1].qs, 0);+ const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[0].qs, 0);+ const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[1].qs, 0);+ const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)),+ lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b)));+ const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)),+ lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b)));+ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const __m256i p_1 = lasx_madd_h(p16_1, mone);+ const __m256i p_2 = lasx_madd_h(p16_2, mone);+ accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[0].d)*GGML_FP16_TO_FP32(x[0].d)),+ __lasx_xvffint_s_w(p_1), accum1);+ accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[1].d)*GGML_FP16_TO_FP32(x[1].d)),+ __lasx_xvffint_s_w(p_2), accum2);++ y += 2;+ x += 2;+ }++ *s = hsum_float_8(__lasx_xvfadd_s(accum1, accum2));+#elsefloat sumf = 0;for (int ib = 0; ib < nb; ++ib) {@@ -9943,6 +13974,179 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void **s = hsum_float_8(accum);+#elif defined(__POWER9_VECTOR__)+ const vector signed char lowMask = vec_splats((signed char)0xF);+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);++ vector float vsumf0 = vec_splats(0.0f);+ vector float vsumf1 = vec_splats(0.0f);+ vector float vsumf2 = vec_splats(0.0f);+ vector float vsumf3 = vec_splats(0.0f);++ const vector signed char values = vec_xl( 0, kvalues_iq4nl);++ for (int ibl = 0; ibl < nb; ++ibl) {++ vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ibl].d));+ vector float vyd = vec_splats(y[ibl].d);+ vector float vd = vec_mul(vxd, vyd);++ vector signed int vsumi0 = vec_splats((int32_t)0);+ vector signed int vsumi1 = vec_splats((int32_t)0);+ vector signed int vsumi2 = vec_splats((int32_t)0);+ vector signed int vsumi3 = vec_splats((int32_t)0);+ vector signed int vsumi4 = vec_splats((int32_t)0);+ vector signed int vsumi5 = vec_splats((int32_t)0);+ vector signed int vsumi6 = vec_splats((int32_t)0);+ vector signed int vsumi7 = vec_splats((int32_t)0);++ uint16_t h = x[ibl].scales_h;++ const uint8_t * restrict q4 = x[ibl].qs;+ const uint8_t * restrict sc = x[ibl].scales_l;+ const int8_t * restrict q8 = y[ibl].qs;++ for (int ib = 0; ib < QK_K/64; ib ++ ) {+ __builtin_prefetch(q4, 0, 1);+ __builtin_prefetch(q8, 0, 1);++ vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);+ vector signed char qxs1 = (vector signed char)vec_xl(16, q4);+ q4 += 32;++ vector signed char q4x00 = (vector signed char)vec_and(qxs0, lowMask);+ vector signed char q4x01 = (vector signed char)vec_sr(qxs0, v4);+ vector signed char q4x10 = (vector signed char)vec_and(qxs1, lowMask);+ vector signed char q4x11 = (vector signed char)vec_sr(qxs1, v4);++ q4x00 = vec_perm(values, values, (vector unsigned char)q4x00);+ q4x01 = vec_perm(values, values, (vector unsigned char)q4x01);+ q4x10 = vec_perm(values, values, (vector unsigned char)q4x10);+ q4x11 = vec_perm(values, values, (vector unsigned char)q4x11);++ vector signed char q8y0 = vec_xl( 0, q8);+ vector signed char q8y1 = vec_xl(16, q8);+ vector signed char q8y2 = vec_xl(32, q8);+ vector signed char q8y3 = vec_xl(48, q8);+ q8 += 64;++ vector signed short qv0 = vec_add(vec_mule(q4x00, q8y0), vec_mulo(q4x00, q8y0));+ vector signed short qv1 = vec_add(vec_mule(q4x01, q8y1), vec_mulo(q4x01, q8y1));+ vector signed short qv2 = vec_add(vec_mule(q4x10, q8y2), vec_mulo(q4x10, q8y2));+ vector signed short qv3 = vec_add(vec_mule(q4x11, q8y3), vec_mulo(q4x11, q8y3));++ const uint16_t ls0 = (uint16_t)(((sc[0] & 0xf) | ((h << 4) & 0x30)) - 32);+ const uint16_t ls1 = (uint16_t)(((sc[0] >> 4) | ((h << 2) & 0x30)) - 32);+ h >>= 4;+ sc ++;++ vector signed short vscales01 = vec_splats((int16_t)ls0);+ vector signed short vscales23 = vec_splats((int16_t)ls1);++ vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0);+ vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1);+ vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2);+ vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3);+ vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4);+ vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5);+ vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6);+ vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7);+ }++ vsumi0 = vec_add(vsumi0, vsumi4);+ vsumi1 = vec_add(vsumi1, vsumi5);+ vsumi2 = vec_add(vsumi2, vsumi6);+ vsumi3 = vec_add(vsumi3, vsumi7);++ vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);+ vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);+ vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);+ vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);+ }++ vsumf0 = vec_add(vsumf0, vsumf2);+ vsumf1 = vec_add(vsumf1, vsumf3);++ vsumf0 = vec_add(vsumf0, vsumf1);++ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));+ vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));++ *s = vec_extract(vsumf0, 0);++#elif defined(__loongarch_asx)++ const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);+ const __m128i m4b = __lsx_vreplgr2vr_b(0x0f);++ __m256 accum = (__m256)__lasx_xvldi(0);+ __m256i tmp1;+ __m128i tmp0, tmp2, tmp3, tmp4, mask_8f, mask;++ mask_8f = __lsx_vreplgr2vr_b(0x8f);+ for (int ibl = 0; ibl < nb; ++ibl) {+ const uint8_t * qs = x[ibl].qs;+ const int8_t * q8 = y[ibl].qs;+ uint16_t sh = x[ibl].scales_h;+ __m256i sumi1 = __lasx_xvldi(0);+ __m256i sumi2 = __lasx_xvldi(0);+ __m128i zero = __lsx_vldi(0);+ for (int ib = 0; ib < QK_K/32; ib += 2) {+ const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16;+ const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16;+ const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;+ tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b), mask_8f);+ tmp0 = __lsx_vori_b(tmp2, 0x10);+ mask = __lsx_vsle_b(zero, tmp2);+ tmp3 = __lsx_vand_v(tmp0, mask);+ tmp3 = __lsx_vshuf_b(values128, zero, tmp3);++ tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_1, m4b), mask_8f);+ tmp0 = __lsx_vori_b(tmp2, 0x10);+ mask = __lsx_vsle_b(zero, tmp2);+ tmp4 = __lsx_vand_v(tmp0, mask);+ tmp4 = __lsx_vshuf_b(values128, zero, tmp4);++ const __m256i q4b_1 = lasx_insertf128(tmp3, tmp4);++ tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b), mask_8f);+ tmp0 = __lsx_vori_b(tmp2, 0x10);+ mask = __lsx_vsle_b(zero, tmp2);+ tmp3 = __lsx_vand_v(tmp0, mask);+ tmp3 = __lsx_vshuf_b(values128, zero, tmp3);++ tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_2, m4b), mask_8f);+ tmp0 = __lsx_vori_b(tmp2, 0x10);+ mask = __lsx_vsle_b(zero, tmp2);+ tmp4 = __lsx_vand_v(tmp0, mask);+ tmp4 = __lsx_vshuf_b(values128, zero, tmp4);++ const __m256i q4b_2 = lasx_insertf128(tmp3, tmp4);++ const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);+ const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);+ const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;+ const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32;+ sh >>= 4;+ __m256i tmp5, tmp6;+ tmp1 = __lasx_xvreplgr2vr_h(ls1);+ tmp5 = __lasx_xvmulwev_w_h(p16_1, tmp1);+ tmp6 = __lasx_xvmulwod_w_h(p16_1, tmp1);+ const __m256i p_1 = __lasx_xvadd_w(tmp5, tmp6);+ tmp1 = __lasx_xvreplgr2vr_h(ls2);+ tmp5 = __lasx_xvmulwev_w_h(p16_2, tmp1);+ tmp6 = __lasx_xvmulwod_w_h(p16_2, tmp1);+ const __m256i p_2 = __lasx_xvadd_w(tmp5, tmp6);+ sumi1 = __lasx_xvadd_w(p_1, sumi1);+ sumi2 = __lasx_xvadd_w(p_2, sumi2);+ }+ accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d),+ __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum);+ }++ *s = hsum_float_8(accum);+#elsefloat sumf = 0;for (int ibl = 0; ibl < nb; ++ibl) {@@ -10474,7 +14678,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict}float max = xval[0];for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);- if (!max) {+ if (max < GROUP_MAX_EPS) {scales[ib] = 0;memset(L, 0, 32);continue;@@ -10650,7 +14854,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v}float max = xval[0];for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);- if (!max) {+ if (max < GROUP_MAX_EPS) {scales[ib] = 0;memset(L, 0, 16);continue;@@ -11091,7 +15295,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v}float max = xval[0];for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);- if (!max) {+ if (max < GROUP_MAX_EPS_IQ3_XXS) {scales[ib] = 0;memset(L, 0, 32);continue;@@ -11631,7 +15835,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vyfor (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);float max = fabsf(xb[0]);for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));- if (!max) {+ if (max < GROUP_MAX_EPS_IQ1_S) {scales[ib] = 0;memset(L, 1, block_size);continue;@@ -11819,7 +16023,7 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy}float max = fabsf(xb[0]);for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));- if (!max) {+ if (max < GROUP_MAX_EPS_IQ1_M) {scales[ib] = 0;memset(L, 1, block_size);continue;@@ -12083,7 +16287,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int blockamax = ax; max = xb[j];}}- if (!amax) {+ if (amax < GROUP_MAX_EPS) {scales[ib] = 0;continue;}@@ -12304,7 +16508,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy}float max = xval[0];for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);- if (!max) {+ if (max < GROUP_MAX_EPS_IQ2_S) {scales[ib] = 0;continue;}diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.cindex 17d0f136..6c5a75fe 100644--- a/llama.cpp/ggml-vector-amd-avx.c+++ b/llama.cpp/ggml-vector-amd-avx.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_avx#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_avx#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.cindex bf00db96..820d0dcb 100644--- a/llama.cpp/ggml-vector-amd-avx2.c+++ b/llama.cpp/ggml-vector-amd-avx2.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx2#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx2#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx2+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_avx2#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx2-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx2#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx2+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx2+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_avx2#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.cindex c73f2bf2..45afda41 100644--- a/llama.cpp/ggml-vector-amd-avx512.c+++ b/llama.cpp/ggml-vector-amd-avx512.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_avx512#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_avx512#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.cindex 835803eb..c636e716 100644--- a/llama.cpp/ggml-vector-amd-avx512bf16.c+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_avx512bf16#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_avx512bf16#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_avx512bf16+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_avx512bf16#define ggml_vec_max_f32 ggml_vec_max_f32_amd_avx512bf16-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_avx512bf16#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_avx512bf16+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_avx512bf16+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_avx512bf16#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.cindex 2344e8e9..331ce2d3 100644--- a/llama.cpp/ggml-vector-amd-f16c.c+++ b/llama.cpp/ggml-vector-amd-f16c.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_f16c#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_f16c#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_f16c+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_f16c#define ggml_vec_max_f32 ggml_vec_max_f32_amd_f16c-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_f16c#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_f16c+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_f16c+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_f16c#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.cindex fbfd057d..db0ec4de 100644--- a/llama.cpp/ggml-vector-amd-fma.c+++ b/llama.cpp/ggml-vector-amd-fma.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_amd_fma#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_amd_fma#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_amd_fma+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_amd_fma#define ggml_vec_max_f32 ggml_vec_max_f32_amd_fma-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_amd_fma#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_amd_fma+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_amd_fma+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_amd_fma#define GGML_VECTOR#include "ggml-vector.inc"#endif // __x86_64__diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.cindex bc4595a1..03fa85f7 100644--- a/llama.cpp/ggml-vector-arm80.c+++ b/llama.cpp/ggml-vector-arm80.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm80#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm80#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm80+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_arm80#define ggml_vec_max_f32 ggml_vec_max_f32_arm80-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm80#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm80+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm80+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_arm80#define GGML_VECTOR#include "ggml-vector.inc"#endif // __aarch64__diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.cindex 20fb8d44..234d7fd3 100644--- a/llama.cpp/ggml-vector-arm82.c+++ b/llama.cpp/ggml-vector-arm82.c@@ -49,10 +49,12 @@#define ggml_vec_sum_f32 ggml_vec_sum_f32_arm82#define ggml_vec_sum_f32_ggf ggml_vec_sum_f32_ggf_arm82#define ggml_vec_sum_f16_ggf ggml_vec_sum_f16_ggf_arm82+#define ggml_vec_sum_bf16_ggf ggml_vec_sum_bf16_ggf_arm82#define ggml_vec_max_f32 ggml_vec_max_f32_arm82-#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82#define ggml_vec_argmax_f32 ggml_vec_argmax_f32_arm82#define ggml_vec_soft_max_f32 ggml_vec_soft_max_f32_arm82+#define ggml_vec_norm_inv_f32 ggml_vec_norm_inv_f32_arm82+#define ggml_vec_sigmoid_f32 ggml_vec_sigmoid_f32_arm82#define GGML_VECTOR#include "ggml-vector.inc"#endif // __aarch64__diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cppindex 3a9a5b8e..52c199c1 100644--- a/llama.cpp/ggml-vector.cpp+++ b/llama.cpp/ggml-vector.cpp@@ -453,6 +453,15 @@ extern "C" void ggml_vec_sum_f16_ggf_amd_avx(const int n, float * s, const ggml_extern "C" void ggml_vec_sum_f16_ggf_arm82(const int n, float * s, const ggml_fp16_t * x);extern "C" void ggml_vec_sum_f16_ggf_arm80(const int n, float * s, const ggml_fp16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_avx512bf16(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_avx512(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_avx2(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_f16c(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_fma(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_amd_avx(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_arm82(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_sum_bf16_ggf_arm80(const int n, float * s, const ggml_bf16_t * x);+extern "C" void ggml_vec_max_f32_amd_avx512bf16(const int n, float * s, const float * x);extern "C" void ggml_vec_max_f32_amd_avx512(const int n, float * s, const float * x);extern "C" void ggml_vec_max_f32_amd_avx2(const int n, float * s, const float * x);@@ -462,15 +471,6 @@ extern "C" void ggml_vec_max_f32_amd_avx(const int n, float * s, const float * xextern "C" void ggml_vec_max_f32_arm82(const int n, float * s, const float * x);extern "C" void ggml_vec_max_f32_arm80(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_avx512bf16(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_avx512(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_avx2(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_f16c(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_fma(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_amd_avx(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_arm82(const int n, float * s, const float * x);-extern "C" void ggml_vec_norm_inv_f32_arm80(const int n, float * s, const float * x);-extern "C" void ggml_vec_argmax_f32_amd_avx512bf16(const int n, int * s, const float * x);extern "C" void ggml_vec_argmax_f32_amd_avx512(const int n, int * s, const float * x);extern "C" void ggml_vec_argmax_f32_amd_avx2(const int n, int * s, const float * x);@@ -480,14 +480,32 @@ extern "C" void ggml_vec_argmax_f32_amd_avx(const int n, int * s, const float *extern "C" void ggml_vec_argmax_f32_arm82(const int n, int * s, const float * x);extern "C" void ggml_vec_argmax_f32_arm80(const int n, int * s, const float * x);-extern "C" float ggml_vec_soft_max_f32_amd_avx512bf16(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_amd_avx512(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_amd_avx2(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_amd_f16c(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_amd_fma(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_amd_avx(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_arm82(const int n, float * y, const float * x, float max);-extern "C" float ggml_vec_soft_max_f32_arm80(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx512bf16(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx512(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx2(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_f16c(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_fma(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_amd_avx(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_arm82(const int n, float * y, const float * x, float max);+extern "C" ggml_float ggml_vec_soft_max_f32_arm80(const int n, float * y, const float * x, float max);++extern "C" void ggml_vec_norm_inv_f32_amd_avx512bf16(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx512(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx2(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_f16c(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_fma(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_amd_avx(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_arm82(const int n, float * s, const float * x);+extern "C" void ggml_vec_norm_inv_f32_arm80(const int n, float * s, const float * x);++extern "C" void ggml_vec_sigmoid_f32_amd_avx512bf16 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_amd_avx512 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_amd_avx2 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_amd_f16c (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_amd_fma (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_amd_avx (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_arm82 (const int n, float * y, const float * x);+extern "C" void ggml_vec_sigmoid_f32_arm80 (const int n, float * y, const float * x);static const struct VectorFuncs {typeof(ggml_fp16_to_fp32_row) *ptr_ggml_fp16_to_fp32_row;@@ -540,10 +558,12 @@ static const struct VectorFuncs {typeof(ggml_vec_sum_f32) *ptr_ggml_vec_sum_f32;typeof(ggml_vec_sum_f32_ggf) *ptr_ggml_vec_sum_f32_ggf;typeof(ggml_vec_sum_f16_ggf) *ptr_ggml_vec_sum_f16_ggf;+ typeof(ggml_vec_sum_bf16_ggf) *ptr_ggml_vec_sum_bf16_ggf;typeof(ggml_vec_max_f32) *ptr_ggml_vec_max_f32;- typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;typeof(ggml_vec_argmax_f32) *ptr_ggml_vec_argmax_f32;typeof(ggml_vec_soft_max_f32) *ptr_ggml_vec_soft_max_f32;+ typeof(ggml_vec_norm_inv_f32) *ptr_ggml_vec_norm_inv_f32;+ typeof(ggml_vec_sigmoid_f32) *ptr_ggml_vec_sigmoid_f32;VectorFuncs() {#ifdef __x86_64__@@ -598,10 +618,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512bf16;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512bf16;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512bf16;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_avx512bf16;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512bf16;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512bf16;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512bf16;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512bf16;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_avx512bf16;return;}#endif@@ -657,10 +679,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx512;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx512;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx512;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_avx512;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx512;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx512;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx512;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx512;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_avx512;return;}#endif@@ -716,10 +740,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx2;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx2;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx2;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_avx2;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx2;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx2;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx2;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx2;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_avx2;return;}#endif@@ -775,10 +801,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_f16c;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_f16c;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_f16c;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_f16c;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_f16c;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_f16c;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_f16c;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_f16c;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_f16c;return;}#endif@@ -834,10 +862,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_fma;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_fma;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_fma;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_fma;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_fma;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_fma;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_fma;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_fma;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_fma;return;}#endif@@ -893,10 +923,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_amd_avx;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_amd_avx;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_amd_avx;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_amd_avx;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_amd_avx;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_amd_avx;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_amd_avx;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_amd_avx;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_amd_avx;return;}#endif@@ -952,10 +984,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm82;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm82;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm82;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_arm82;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm82;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm82;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm82;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm82;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_arm82;return;}#endif@@ -1011,10 +1045,12 @@ static const struct VectorFuncs {ptr_ggml_vec_sum_f32 = ggml_vec_sum_f32_arm80;ptr_ggml_vec_sum_f32_ggf = ggml_vec_sum_f32_ggf_arm80;ptr_ggml_vec_sum_f16_ggf = ggml_vec_sum_f16_ggf_arm80;+ ptr_ggml_vec_sum_bf16_ggf = ggml_vec_sum_bf16_ggf_arm80;ptr_ggml_vec_max_f32 = ggml_vec_max_f32_arm80;- ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;ptr_ggml_vec_argmax_f32 = ggml_vec_argmax_f32_arm80;ptr_ggml_vec_soft_max_f32 = ggml_vec_soft_max_f32_arm80;+ ptr_ggml_vec_norm_inv_f32 = ggml_vec_norm_inv_f32_arm80;+ ptr_ggml_vec_sigmoid_f32 = ggml_vec_sigmoid_f32_arm80;return;}#endif@@ -1221,19 +1257,27 @@ void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {return funcs.ptr_ggml_vec_sum_f16_ggf(n, s, x);}-void ggml_vec_max_f32(const int n, float * s, const float * x) {- return funcs.ptr_ggml_vec_max_f32(n, s, x);+void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {+ return funcs.ptr_ggml_vec_sum_bf16_ggf(n, s, x);}-void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {- return funcs.ptr_ggml_vec_norm_inv_f32(n, s, x);+void ggml_vec_max_f32(const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_max_f32(n, s, x);}void ggml_vec_argmax_f32(const int n, int * s, const float * x) {return funcs.ptr_ggml_vec_argmax_f32(n, s, x);}-float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {+ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {return funcs.ptr_ggml_vec_soft_max_f32(n, y, x, max);}+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {+ return funcs.ptr_ggml_vec_norm_inv_f32(n, s, x);+}++void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) {+ return funcs.ptr_ggml_vec_sigmoid_f32(n, y, x);+}+diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.hindex c6aa5e04..b986fa26 100644--- a/llama.cpp/ggml-vector.h+++ b/llama.cpp/ggml-vector.h@@ -58,10 +58,12 @@ void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, constvoid ggml_vec_sum_f32(const int n, float * s, const float * x);void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x);void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x);+void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x);void ggml_vec_max_f32(const int n, float * s, const float * x);-void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);void ggml_vec_argmax_f32(const int n, int * s, const float * x);-float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);+ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);+void ggml_vec_norm_inv_f32(const int n, float * s, const float * x);+void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incindex ed660e7a..de820dbe 100644--- a/llama.cpp/ggml-vector.inc+++ b/llama.cpp/ggml-vector.inc@@ -382,6 +382,8 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO#define GGML_F16_VEC_SET1 GGML_F32x4_SET1#define GGML_F16_VEC_FMA GGML_F32x4_FMA+#define GGML_F16_VEC_ADD GGML_F32x4_ADD+#define GGML_F16_VEC_MUL GGML_F32x4_MUL#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE// Use vec_xl, not vec_ld, in case the load address is not aligned.#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \@@ -604,6 +606,195 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#elif defined(__loongarch_asx)++#define GGML_SIMD++// F32 LASX+#define GGML_F32_STEP 32+#define GGML_F32_EPR 8++#define GGML_F32x8 __m256+#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))+#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)+#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0)+#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)+#define GGML_F32x8_ADD __lasx_xvfadd_s+#define GGML_F32x8_MUL __lasx_xvfmul_s+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ float *tmp_p = (float *)&x[0]; \+ res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \+} while (0)+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x8+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD+#define GGML_F32_VEC_STORE GGML_F32x8_STORE+#define GGML_F32_VEC_FMA GGML_F32x8_FMA+#define GGML_F32_VEC_ADD GGML_F32x8_ADD+#define GGML_F32_VEC_MUL GGML_F32x8_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE++// F16 LASX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 8++// F16 arithmetic is not supported by AVX, so we use F32 instead++#define GGML_F32Cx8 __m256+#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x))++static inline __m256 __lasx_f32cx8_load(ggml_fp16_t *x) {+ float tmp[8];++ for (int i = 0; i < 8; i++) {+ tmp[i] = GGML_FP16_TO_FP32(x[i]);+ }++ return (__m256)__lasx_xvld(tmp, 0);+}+static inline void __lasx_f32cx8_store(ggml_fp16_t *x, __m256 y) {+ float arr[8];++ __lasx_xvst(y, arr, 0);++ for (int i = 0; i < 8; i++)+ x[i] = GGML_FP32_TO_FP16(arr[i]);+}+#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)+#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)++#define GGML_F32Cx8_FMA GGML_F32x8_FMA+#define GGML_F32Cx8_ADD __lasx_xvfadd_s+#define GGML_F32Cx8_MUL __lasx_xvfmul_s+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE++#define GGML_F16_VEC GGML_F32Cx8+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE++#elif defined(__loongarch_sx)++#define GGML_SIMD++// F32 LSX++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 __m128+#define GGML_F32x4_ZERO __lsx_vldi(0)+#define GGML_F32x4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)+#define GGML_F32x4_LOAD(x) __lsx_vld((x), 0)+#define GGML_F32x4_STORE((x),(y)) __lsx_vst((y), (x), 0)+#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)+#define GGML_F32x4_ADD __lsx_vfadd_s+#define GGML_F32x4_MUL __lsx_vfmul_s+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ __m128i tmp = __lsx_vsrli_d((__m128i)x[0], 32); \+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, x[0]); \+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \+ const __m128 t0 = __lsx_vshuf4i_w(tmp, 0x88); \+ tmp = __lsx_vsrli_d((__m128i)t0, 32); \+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, t0); \+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \+ res = (ggml_float) __lsx_vpickve2gr_w(__lsx_vshuf4i_w(tmp, 0x88), 0); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 LSX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 4++static inline __m128 __lsx_f16x4_load(ggml_fp16_t *x) {+ float tmp[4];++ tmp[0] = GGML_FP16_TO_FP32(x[0]);+ tmp[1] = GGML_FP16_TO_FP32(x[1]);+ tmp[2] = GGML_FP16_TO_FP32(x[2]);+ tmp[3] = GGML_FP16_TO_FP32(x[3]);++ return __lsx_vld(tmp, 0);+}++static inline void __lsx_f16x4_store(ggml_fp16_t *x, __m128 y) {+ float arr[4];++ __lsx_vst(y, arr, 0);++ x[0] = GGML_FP32_TO_FP16(arr[0]);+ x[1] = GGML_FP32_TO_FP16(arr[1]);+ x[2] = GGML_FP32_TO_FP16(arr[2]);+ x[3] = GGML_FP32_TO_FP16(arr[3]);+}++#define GGML_F32Cx4 __m128+#define GGML_F32Cx4_ZERO __lsx_vldi(0)+#define GGML_F32Cx4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)+#define GGML_F32Cx4_LOAD(x) __lsx_f16x4_load(x)+#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)+#define GGML_F32Cx4_FMA GGML_F32x4_FMA+#define GGML_F32Cx4_ADD __lsx_vfadd_s+#define GGML_F32Cx4_MUL __lsx_vfmul_s+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++#define GGML_F16_VEC GGML_F32Cx4+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#endif// GGML_F32_ARR / GGML_F16_ARR@@ -670,10 +861,10 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {int i = 0;#if defined(__AVX512BF16__)for (; i + 32 <= n; i += 32) {- _mm512_storeu_ps(- (__m512 *)(y + i),- (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),- _mm512_loadu_ps(x + i)));+ _mm512_storeu_si512(+ (__m512i *)(y + i),+ m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),+ _mm512_loadu_ps(x + i))));}#endiffor (; i < n; i++) {@@ -762,10 +953,10 @@ void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restr__m512 c1 = _mm512_setzero_ps();__m512 c2 = _mm512_setzero_ps();for (; i + 64 <= n; i += 64) {- c1 = _mm512_dpbf16_ps(c1, (__m512bh)_mm512_loadu_ps((const float *)(x + i)),- (__m512bh)_mm512_loadu_ps((const float *)(y + i)));- c2 = _mm512_dpbf16_ps(c2, (__m512bh)_mm512_loadu_ps((const float *)(x + i + 32)),- (__m512bh)_mm512_loadu_ps((const float *)(y + i + 32)));+ c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),+ m512bh(_mm512_loadu_si512((y + i))));+ c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),+ m512bh(_mm512_loadu_si512((y + i + 32))));}sumf += (ggml_float)_mm512_reduce_add_ps(c1);sumf += (ggml_float)_mm512_reduce_add_ps(c2);@@ -1096,6 +1287,7 @@ void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i =void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }+void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }// TODO: optimize performancevoid ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }@@ -1137,21 +1329,11 @@ void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (i// -7 -6 -5 -4 -3 -2 -1 0 1//-inline static float ggml_silu_f32(float x) {- // SiLU is the favored by LLaMA, Mistral, Phi, Rocket, etc.- return x/(1.f + expf(-x));-}-inline static float ggml_gelu_f32(float x) {// GeLU approximation that goes slower and we seem to be stuck with.return .5f * x * (1.f + tanhf(sqrtf(M_2_PI) * (x + .044715f * x * x * x)));}-inline static float ggml_gelu_quick_f32(float x) {- // "Quick" GeLU is used by LLaVA's CLIP vision model.- return x*(1.f/(1.f+expf(-1.702f*x)));-}-static void ggml_vec_gelu_f16_init(void) {ggml_table_gelu_f16 = malloc(sizeof(float) * 65536);for (int i = 0; i < 65536; ++i) {@@ -1178,13 +1360,22 @@ void ggml_vec_gelu_f32(const int n, float * y, const float * x) {}}+inline static float ggml_gelu_quick_f32(float x) {+ return x*(1.0f/(1.0f+expf(-1.702f*x)));+}+void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {for (int i = 0; i < n; ++i) {y[i] = ggml_gelu_quick_f32(x[i]);}}-#if defined(__ARM_NEON)+// Sigmoid Linear Unit (SiLU) function+inline static float ggml_silu_f32(float x) {+ return x/(1.0f + expf(-x));+}++#if defined(__ARM_NEON) && defined(__aarch64__)// adapted from arm limited optimized routine// the maximum error is 1.45358 plus 0.5 ulps@@ -1397,7 +1588,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {for (; i + 3 < n; i += 4) {_mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));}-#elif defined(__ARM_NEON)+#elif defined(__ARM_NEON) && defined(__aarch64__)for (; i + 3 < n; i += 4) {vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));}@@ -1408,7 +1599,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {}}-float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {+ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {int i = 0;ggml_float sum = 0;if (!FLAG_trap) { // [jart] preserve this line@@ -1446,7 +1637,7 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)#endifsum += (ggml_float)_mm_cvtss_f32(val);}-#elif defined(__ARM_NEON)+#elif defined(__ARM_NEON) && defined(__aarch64__)for (; i + 3 < n; i += 4) {float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),vdupq_n_f32(max)));@@ -1502,6 +1693,14 @@ void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {*s = sum;}+void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {+ float sum = 0.0f;+ for (int i = 0; i < n; ++i) {+ sum += GGML_BF16_TO_FP32(x[i]);+ }+ *s = sum;+}+void ggml_vec_max_f32(const int n, float * s, const float * x) {#ifndef GGML_USE_ACCELERATEfloat max = -INFINITY;diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex e63f244b..b7c3557b 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -726,10 +726,6 @@ ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {// number of elements to fit in a single register//-#define GGML_SOFT_MAX_UNROLL 4-#define GGML_VEC_DOT_UNROLL 2-#define GGML_VEC_MAD_UNROLL 32-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)#define GGML_SIMD@@ -1098,6 +1094,8 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO#define GGML_F16_VEC_SET1 GGML_F32x4_SET1#define GGML_F16_VEC_FMA GGML_F32x4_FMA+#define GGML_F16_VEC_ADD GGML_F32x4_ADD+#define GGML_F16_VEC_MUL GGML_F32x4_MUL#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE// Use vec_xl, not vec_ld, in case the load address is not aligned.#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \@@ -1320,6 +1318,195 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) {#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#elif defined(__loongarch_asx)++#define GGML_SIMD++// F32 LASX+#define GGML_F32_STEP 32+#define GGML_F32_EPR 8++#define GGML_F32x8 __m256+#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))+#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)+#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0)+#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)+#define GGML_F32x8_ADD __lasx_xvfadd_s+#define GGML_F32x8_MUL __lasx_xvfmul_s+#define GGML_F32x8_REDUCE(res, x) \+do { \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \+ } \+ float *tmp_p = (float *)&x[0]; \+ res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \+} while (0)+// TODO: is this optimal ?++#define GGML_F32_VEC GGML_F32x8+#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x8_SET1+#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD+#define GGML_F32_VEC_STORE GGML_F32x8_STORE+#define GGML_F32_VEC_FMA GGML_F32x8_FMA+#define GGML_F32_VEC_ADD GGML_F32x8_ADD+#define GGML_F32_VEC_MUL GGML_F32x8_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE++// F16 LASX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 8++// F16 arithmetic is not supported by AVX, so we use F32 instead++#define GGML_F32Cx8 __m256+#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)+#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x))++static inline __m256 __lasx_f32cx8_load(ggml_fp16_t *x) {+ float tmp[8];++ for (int i = 0; i < 8; i++) {+ tmp[i] = GGML_FP16_TO_FP32(x[i]);+ }++ return (__m256)__lasx_xvld(tmp, 0);+}+static inline void __lasx_f32cx8_store(ggml_fp16_t *x, __m256 y) {+ float arr[8];++ __lasx_xvst(y, arr, 0);++ for (int i = 0; i < 8; i++)+ x[i] = GGML_FP32_TO_FP16(arr[i]);+}+#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)+#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)++#define GGML_F32Cx8_FMA GGML_F32x8_FMA+#define GGML_F32Cx8_ADD __lasx_xvfadd_s+#define GGML_F32Cx8_MUL __lasx_xvfmul_s+#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE++#define GGML_F16_VEC GGML_F32Cx8+#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE++#elif defined(__loongarch_sx)++#define GGML_SIMD++// F32 LSX++#define GGML_F32_STEP 32+#define GGML_F32_EPR 4++#define GGML_F32x4 __m128+#define GGML_F32x4_ZERO __lsx_vldi(0)+#define GGML_F32x4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)+#define GGML_F32x4_LOAD(x) __lsx_vld((x), 0)+#define GGML_F32x4_STORE((x),(y)) __lsx_vst((y), (x), 0)+#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)+#define GGML_F32x4_ADD __lsx_vfadd_s+#define GGML_F32x4_MUL __lsx_vfmul_s+#define GGML_F32x4_REDUCE(res, x) \+{ \+ int offset = GGML_F32_ARR >> 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ offset >>= 1; \+ for (int i = 0; i < offset; ++i) { \+ x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \+ } \+ __m128i tmp = __lsx_vsrli_d((__m128i)x[0], 32); \+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, x[0]); \+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \+ const __m128 t0 = __lsx_vshuf4i_w(tmp, 0x88); \+ tmp = __lsx_vsrli_d((__m128i)t0, 32); \+ tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, t0); \+ tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \+ res = (ggml_float) __lsx_vpickve2gr_w(__lsx_vshuf4i_w(tmp, 0x88), 0); \+}++#define GGML_F32_VEC GGML_F32x4+#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO+#define GGML_F32_VEC_SET1 GGML_F32x4_SET1+#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD+#define GGML_F32_VEC_STORE GGML_F32x4_STORE+#define GGML_F32_VEC_FMA GGML_F32x4_FMA+#define GGML_F32_VEC_ADD GGML_F32x4_ADD+#define GGML_F32_VEC_MUL GGML_F32x4_MUL+#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE++// F16 LSX++#define GGML_F16_STEP 32+#define GGML_F16_EPR 4++static inline __m128 __lsx_f16x4_load(ggml_fp16_t *x) {+ float tmp[4];++ tmp[0] = GGML_FP16_TO_FP32(x[0]);+ tmp[1] = GGML_FP16_TO_FP32(x[1]);+ tmp[2] = GGML_FP16_TO_FP32(x[2]);+ tmp[3] = GGML_FP16_TO_FP32(x[3]);++ return __lsx_vld(tmp, 0);+}++static inline void __lsx_f16x4_store(ggml_fp16_t *x, __m128 y) {+ float arr[4];++ __lsx_vst(y, arr, 0);++ x[0] = GGML_FP32_TO_FP16(arr[0]);+ x[1] = GGML_FP32_TO_FP16(arr[1]);+ x[2] = GGML_FP32_TO_FP16(arr[2]);+ x[3] = GGML_FP32_TO_FP16(arr[3]);+}++#define GGML_F32Cx4 __m128+#define GGML_F32Cx4_ZERO __lsx_vldi(0)+#define GGML_F32Cx4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0)+#define GGML_F32Cx4_LOAD(x) __lsx_f16x4_load(x)+#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)+#define GGML_F32Cx4_FMA GGML_F32x4_FMA+#define GGML_F32Cx4_ADD __lsx_vfadd_s+#define GGML_F32Cx4_MUL __lsx_vfmul_s+#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE++#define GGML_F16_VEC GGML_F32Cx4+#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO+#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1+#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p)+#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i])+#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA+#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD+#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL+#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE+#endif// GGML_F32_ARR / GGML_F16_ARR@@ -1408,6 +1595,61 @@ void ggml_syncthreads(struct ggml_barrier * b, int nth) {sync_wait_end();}+//+// ggml context+//++struct ggml_context {+ size_t mem_size;+ void* mem_buffer;+ bool mem_buffer_owned;+ bool no_alloc;+ bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers++ int n_objects;++ struct ggml_object* objects_begin;+ struct ggml_object* objects_end;++ struct ggml_scratch scratch;+ struct ggml_scratch scratch_save;+};++struct ggml_context_container {+ bool used;++ struct ggml_context context;+};++struct ggml_compute_state_shared {+ const struct ggml_cgraph* cgraph;+ const struct ggml_cplan* cplan;++ int64_t perf_node_start_cycles;+ int64_t perf_node_start_time_us;++ const int n_threads;++ // synchronization primitives+ atomic_int n_active; // num active threads+ atomic_int node_n; // active graph node+ atomic_int node_task; // active graph node task phase+ struct ggml_barrier barrier;++ ggml_abort_callback abort_callback; // abort ggml_graph_compute when true+ void* abort_callback_data;+};++typedef pthread_t ggml_thread_t;++struct ggml_compute_state {+ ggml_thread_t thrd;+ int ith;+ struct ggml_compute_state_shared* shared;+ enum ggml_status ec;+ bool is_main_thread; // [jart]+};+//// data types//@@ -1459,7 +1701,6 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"SOFT_MAX_BACK","ROPE","ROPE_BACK",- "ALIBI","CLAMP","CONV_TRANSPOSE_1D","IM2COL",@@ -1501,7 +1742,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"CROSS_ENTROPY_LOSS_BACK",};-static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");+static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"none",@@ -1550,7 +1791,6 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"soft_max_back(x)","rope(x)","rope_back(x)",- "alibi(x)","clamp(x)","conv_transpose_1d(x)","im2col(x)",@@ -1592,7 +1832,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"cross_entropy_loss_back(x,y)",};-static_assert(GGML_OP_COUNT == 77, "GGML_OP_COUNT != 77");+static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76");static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");@@ -1605,6 +1845,7 @@ static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {"TANH","ELU","RELU",+ "SIGMOID","GELU","GELU_QUICK","SILU",@@ -1612,7 +1853,7 @@ static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {"HARDSIGMOID",};-static_assert(GGML_UNARY_OP_COUNT == 12, "GGML_UNARY_OP_COUNT != 12");+static_assert(GGML_UNARY_OP_COUNT == 13, "GGML_UNARY_OP_COUNT != 13");static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");@@ -1653,32 +1894,6 @@ static void ggml_setup_op_has_task_pass(void) {}}-//-// ggml context-//--struct ggml_context {- size_t mem_size;- void * mem_buffer;- bool mem_buffer_owned;- bool no_alloc;- bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers-- int n_objects;-- struct ggml_object * objects_begin;- struct ggml_object * objects_end;-- struct ggml_scratch scratch;- struct ggml_scratch scratch_save;-};--struct ggml_context_container {- bool used;-- struct ggml_context context;-};-//// NUMA support//@@ -2097,6 +2312,16 @@ GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct g(t0->ne[3] == t1->ne[3] );}+bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");++ return+ (t0->nb[0] == t1->nb[0] ) &&+ (t0->nb[1] == t1->nb[1] ) &&+ (t0->nb[2] == t1->nb[2] ) &&+ (t0->nb[3] == t1->nb[3] );+}+// check if t1 can be represented as a repeatition of t0static inline bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");@@ -2149,11 +2374,12 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {{const uint64_t t_start = ggml_time_us(); UNUSED(t_start);- ggml_fp16_t ii;for (int i = 0; i < (1 << 16); ++i) {- uint16_t ui = i;- memcpy(&ii, &ui, sizeof(ii));- ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(ii);+ union {+ uint16_t u16;+ ggml_fp16_t fp16;+ } u = {i};+ float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(u.fp16);}const uint64_t t_end = ggml_time_us(); UNUSED(t_end);@@ -2239,7 +2465,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {GGML_PRINT_DEBUG("%s: context initialized\n", __func__);- llamafile_trapping_enabled(+1);ggml_critical_section_end();return ctx;@@ -2439,6 +2664,12 @@ static struct ggml_tensor * ggml_new_tensor_impl(struct ggml_tensor * const result = (struct ggml_tensor *)((char *)ctx->mem_buffer + obj_new->offs);+#ifdef __clang__+ // temporary until ggml_tensor::backend is removed+ #pragma clang diagnostic push+ #pragma clang diagnostic ignored "-Wdeprecated-declarations"+#endif+*result = (struct ggml_tensor) {/*.type =*/ type,/*.backend =*/ GGML_BACKEND_TYPE_CPU,@@ -2461,6 +2692,10 @@ static struct ggml_tensor * ggml_new_tensor_impl(/*.padding =*/ { 0 },};+#ifdef __clang__+ #pragma clang diagnostic pop+#endif+// TODO: this should not be needed as long as we don't rely on aligned SIMD loads//ggml_assert_aligned(result->data);@@ -2619,6 +2854,13 @@ struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));}} break;+ case GGML_TYPE_BF16:+ {+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));+ for (int i = 0; i < n; i++) {+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));+ }+ } break;case GGML_TYPE_F32:{assert(tensor->nb[0] == sizeof(float));@@ -2671,6 +2913,13 @@ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));}} break;+ case GGML_TYPE_BF16:+ {+ assert(tensor->nb[0] == sizeof(ggml_bf16_t));+ for (int i = 0; i < n; i++) {+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));+ }+ } break;case GGML_TYPE_F32:{assert(tensor->nb[0] == sizeof(float));@@ -2738,6 +2987,11 @@ int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);}+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);+ }case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -2780,6 +3034,11 @@ void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -2803,6 +3062,8 @@ int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int ireturn ((int32_t *) data)[0];case GGML_TYPE_F16:return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);+ case GGML_TYPE_BF16:+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);case GGML_TYPE_F32:return ((float *) data)[0];default:@@ -2831,6 +3092,10 @@ void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,{((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{((float *)(data))[0] = value;@@ -2869,6 +3134,11 @@ float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);}+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);+ }case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -2911,6 +3181,11 @@ void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -2934,6 +3209,8 @@ float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,return ((int32_t *) data)[0];case GGML_TYPE_F16:return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);+ case GGML_TYPE_BF16:+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);case GGML_TYPE_F32:return ((float *) data)[0];default:@@ -2962,6 +3239,10 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,{((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{((float *)(data))[0] = value;@@ -3156,7 +3437,11 @@ static struct ggml_tensor * ggml_add_cast_impl(// TODO: support less-strict constraint// GGML_ASSERT(ggml_can_repeat(b, a));GGML_ASSERT(ggml_can_repeat_rows(b, a));- GGML_ASSERT(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16); // currently only supported for quantized input and f16++ // currently only supported for quantized input and f16+ GGML_ASSERT(ggml_is_quantized(a->type) ||+ a->type == GGML_TYPE_F16 ||+ a->type == GGML_TYPE_BF16);bool is_node = false;@@ -3789,21 +4074,35 @@ struct ggml_tensor * ggml_leaky_relu(return result;}-// ggml_gelu+// ggml_sigmoid-struct ggml_tensor * ggml_gelu(+struct ggml_tensor * ggml_sigmoid(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_unary(ctx, a, GGML_UNARY_OP_GELU);+ return ggml_unary(ctx, a, GGML_UNARY_OP_SIGMOID);}-struct ggml_tensor * ggml_gelu_inplace(+struct ggml_tensor * ggml_sigmoid_inplace(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU);+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SIGMOID);}-// ggml_gelu_quick+// ggml_gelu++struct ggml_tensor * ggml_gelu(+ struct ggml_context * ctx,+ struct ggml_tensor * a) {+ return ggml_unary(ctx, a, GGML_UNARY_OP_GELU);+}++struct ggml_tensor * ggml_gelu_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a) {+ return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU);+}++// ggml_gelu_quickstruct ggml_tensor * ggml_gelu_quick(struct ggml_context * ctx,@@ -4872,7 +5171,6 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,- struct ggml_tensor * pos,float scale,float max_bias,bool inplace) {@@ -4886,18 +5184,8 @@ static struct ggml_tensor * ggml_soft_max_impl(GGML_ASSERT(mask->ne[1] >= a->ne[1]);}- if (pos) {- GGML_ASSERT(ggml_is_vector(pos));- GGML_ASSERT(pos->type == GGML_TYPE_F16 || pos->type == GGML_TYPE_F32);- GGML_ASSERT(pos->ne[0] == a->ne[0]);- }-- if (pos && mask) {- GGML_ASSERT(pos->type == mask->type);- }-if (max_bias > 0.0f) {- GGML_ASSERT(pos);+ GGML_ASSERT(mask);}bool is_node = false;@@ -4915,7 +5203,6 @@ static struct ggml_tensor * ggml_soft_max_impl(result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;result->src[1] = mask;- result->src[2] = pos;return result;}@@ -4923,23 +5210,22 @@ static struct ggml_tensor * ggml_soft_max_impl(struct ggml_tensor * ggml_soft_max(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, false);+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, 0.0f, false);}struct ggml_tensor * ggml_soft_max_inplace(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_soft_max_impl(ctx, a, NULL, NULL, 1.0f, 0.0f, true);+ return ggml_soft_max_impl(ctx, a, NULL, 1.0f, 0.0f, true);}struct ggml_tensor * ggml_soft_max_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,- struct ggml_tensor * pos,float scale,float max_bias) {- return ggml_soft_max_impl(ctx, a, mask, pos, scale, max_bias, false);+ return ggml_soft_max_impl(ctx, a, mask, scale, max_bias, false);}// ggml_soft_max_back@@ -4985,6 +5271,7 @@ static struct ggml_tensor * ggml_rope_impl(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -5002,6 +5289,11 @@ static struct ggml_tensor * ggml_rope_impl(GGML_ASSERT(b->type == GGML_TYPE_I32);GGML_ASSERT(a->ne[2] == b->ne[0]);+ if (c) {+ GGML_ASSERT(c->type == GGML_TYPE_F32);+ GGML_ASSERT(c->ne[0] >= n_dims / 2);+ }+bool is_node = false;if (a->grad) {@@ -5025,6 +5317,7 @@ static struct ggml_tensor * ggml_rope_impl(result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;result->src[1] = b;+ result->src[2] = c;return result;}@@ -5037,7 +5330,7 @@ struct ggml_tensor * ggml_rope(int mode,int n_ctx) {return ggml_rope_impl(- ctx, a, b, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, false+ ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, false);}@@ -5049,14 +5342,15 @@ struct ggml_tensor * ggml_rope_inplace(int mode,int n_ctx) {return ggml_rope_impl(- ctx, a, b, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, true+ ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, true);}-struct ggml_tensor * ggml_rope_custom(+struct ggml_tensor * ggml_rope_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -5068,15 +5362,16 @@ struct ggml_tensor * ggml_rope_custom(float beta_fast,float beta_slow) {return ggml_rope_impl(- ctx, a, b, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,+ ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false);}-struct ggml_tensor * ggml_rope_custom_inplace(+struct ggml_tensor * ggml_rope_ext_inplace(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -5088,19 +5383,49 @@ struct ggml_tensor * ggml_rope_custom_inplace(float beta_fast,float beta_slow) {return ggml_rope_impl(- ctx, a, b, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,+ ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true);}-struct ggml_tensor * ggml_rope_xpos_inplace(+struct ggml_tensor * ggml_rope_custom(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode,+ int n_ctx,+ int n_orig_ctx,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow) {+ return ggml_rope_impl(+ ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false+ );+}++struct ggml_tensor * ggml_rope_custom_inplace(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,int n_dims,- float base,- bool down) {- return ggml_rope_impl(ctx, a, b, n_dims, 0, 0, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, base, down, true);+ int mode,+ int n_ctx,+ int n_orig_ctx,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow) {+ return ggml_rope_impl(+ ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true+ );}// ggml_rope_back@@ -5109,6 +5434,7 @@ struct ggml_tensor * ggml_rope_back(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -5124,6 +5450,7 @@ struct ggml_tensor * ggml_rope_back(GGML_ASSERT(ggml_is_vector(b));GGML_ASSERT(b->type == GGML_TYPE_I32);GGML_ASSERT(a->ne[2] == b->ne[0]);+ GGML_ASSERT(c == NULL && "freq factors not implemented yet");GGML_ASSERT((mode & 4) == 0 && "ggml_rope_back() for ChatGLM not implemented yet");@@ -5154,37 +5481,6 @@ struct ggml_tensor * ggml_rope_back(return result;}-// ggml_alibi--struct ggml_tensor * ggml_alibi(- struct ggml_context * ctx,- struct ggml_tensor * a,- int n_past,- int n_head,- float bias_max) {- GGML_ASSERT(n_past >= 0);- bool is_node = false;-- if (a->grad) {- GGML_ASSERT(false); // TODO: implement backward- is_node = true;- }-- // TODO: when implement backward, fix this:- //struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);- struct ggml_tensor * result = ggml_view_tensor(ctx, a);-- int32_t op_params[3] = { n_past, n_head };- memcpy(op_params + 2, &bias_max, sizeof(float));- ggml_set_op_params(result, op_params, sizeof(op_params));-- result->op = GGML_OP_ALIBI;- result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;- result->src[0] = a;-- return result;-}-// ggml_clampstruct ggml_tensor * ggml_clamp(@@ -5534,7 +5830,10 @@ struct ggml_tensor * ggml_pool_2d(static struct ggml_tensor * ggml_upscale_impl(struct ggml_context * ctx,struct ggml_tensor * a,- int scale_factor) {+ int ne0,+ int ne1,+ int ne2,+ int ne3) {bool is_node = false;if (a->grad) {@@ -5542,19 +5841,45 @@ static struct ggml_tensor * ggml_upscale_impl(is_node = true;}+ GGML_ASSERT(a->ne[0] <= ne0);+ GGML_ASSERT(a->ne[1] <= ne1);+ GGML_ASSERT(a->ne[2] <= ne2);+ GGML_ASSERT(a->ne[3] <= ne3);+struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type,- a->ne[0] * scale_factor,- a->ne[1] * scale_factor,- a->ne[2], a->ne[3]);+ ne0,+ ne1,+ ne2,+ ne3+ );result->op = GGML_OP_UPSCALE;- result->op_params[0] = scale_factor;+result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;return result;}+struct ggml_tensor * ggml_upscale(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int scale_factor) {+ return ggml_upscale_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3]);+}++struct ggml_tensor * ggml_upscale_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int ne0,+ int ne1,+ int ne2,+ int ne3) {+ return ggml_upscale_impl(ctx, a, ne0, ne1, ne2, ne3);+}++// ggml_pad+struct ggml_tensor * ggml_pad(struct ggml_context * ctx,struct ggml_tensor * a,@@ -5579,12 +5904,7 @@ struct ggml_tensor * ggml_pad(return result;}-struct ggml_tensor * ggml_upscale(- struct ggml_context * ctx,- struct ggml_tensor * a,- int scale_factor) {- return ggml_upscale_impl(ctx, a, scale_factor);-}+// ggml_arangestruct ggml_tensor * ggml_arange(struct ggml_context * ctx,@@ -5606,6 +5926,8 @@ struct ggml_tensor * ggml_arange(return result;}+// ggml_timestep_embedding+struct ggml_tensor * ggml_timestep_embedding(struct ggml_context * ctx,struct ggml_tensor * timesteps,@@ -5712,9 +6034,11 @@ struct ggml_tensor * ggml_flash_attn_ext(struct ggml_tensor * k,struct ggml_tensor * v,struct ggml_tensor * mask,- float scale) {+ float scale,+ float max_bias) {GGML_ASSERT(ggml_can_mul_mat(k, q));// TODO: check if vT can be multiplied by (k*qT)+if (mask) {GGML_ASSERT(ggml_is_contiguous(mask));GGML_ASSERT(mask->ne[2] == 1);@@ -5724,6 +6048,10 @@ struct ggml_tensor * ggml_flash_attn_ext(//GGML_ASSERT(ggml_can_repeat_rows(mask, qk));}+ if (max_bias > 0.0f) {+ GGML_ASSERT(mask);+ }+bool is_node = false;if (q->grad || k->grad || v->grad) {@@ -5734,7 +6062,7 @@ struct ggml_tensor * ggml_flash_attn_ext(int64_t ne[4] = { q->ne[0], q->ne[2], q->ne[1], q->ne[3] };struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);- float params[] = { scale };+ float params[] = { scale, max_bias };ggml_set_op_params(result, params, sizeof(params));result->op = GGML_OP_FLASH_ATTN_EXT;@@ -5754,7 +6082,7 @@ void ggml_flash_attn_ext_set_prec(const int32_t prec_i32 = (int32_t) prec;- ggml_set_op_params_i32(a, 1, prec_i32); // scale is on first pos+ ggml_set_op_params_i32(a, 2, prec_i32); // scale is on first pos, max_bias on second}// ggml_flash_ff@@ -6633,8 +6961,8 @@ static void ggml_compute_forward_dup_same_cont(((char *) src0->data + ie0*nb00),(ie1 - ie0) * ggml_type_size(src0->type));}-}+static void ggml_compute_forward_dup_f16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -6908,7 +7236,7 @@ static void ggml_compute_forward_dup_f16(}}-static void ggml_compute_forward_dup_f32(+static void ggml_compute_forward_dup_bf16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -6956,10 +7284,11 @@ static void ggml_compute_forward_dup_f32(return;}+ // TODO: add more special-case implementations for tensor shapes/strides that can benefit from memcpy+if (ggml_is_contiguous(dst)) {- // TODO: simplify- if (nb00 == sizeof(float)) {- if (dst->type == GGML_TYPE_F32) {+ if (nb00 == sizeof(ggml_bf16_t)) {+ if (dst->type == GGML_TYPE_BF16) {size_t id = 0;const size_t rs = ne00 * nb00;char * dst_ptr = (char *) dst->data;@@ -6975,8 +7304,43 @@ static void ggml_compute_forward_dup_f32(id += rs * (ne01 - ir1);}}+ } else if (dst->type == GGML_TYPE_F16) {+ size_t id = 0;+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);+ for (int i00 = 0; i00 < ne00; i00++) {+ dst_ptr[id] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(src0_ptr[i00]));+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }+ } else if (dst->type == GGML_TYPE_F32) {+ size_t id = 0;+ float * dst_ptr = (float *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);+ for (int i00 = 0; i00 < ne00; i00++) {+ dst_ptr[id] = GGML_BF16_TO_FP32(src0_ptr[i00]);+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }} else if (type_traits[dst->type].from_float) {ggml_from_float_t const quantize_row_q = type_traits[dst->type].from_float;+ float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;size_t id = 0;size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));@@ -6986,8 +7350,13 @@ static void ggml_compute_forward_dup_f32(for (int i02 = 0; i02 < ne02; i02++) {id += rs * ir0;for (int i01 = ir0; i01 < ir1; i01++) {- const float * src0_ptr = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);- quantize_row_q(src0_ptr, dst_ptr + id, ne00);+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);++ for (int i00 = 0; i00 < ne00; i00++) {+ src0_f32[i00] = GGML_BF16_TO_FP32(src0_ptr[i00]);+ }++ quantize_row_q(src0_f32, dst_ptr + id, ne00);id += rs;}id += rs * (ne01 - ir1);@@ -7008,7 +7377,25 @@ static void ggml_compute_forward_dup_f32(id += ne00 * ir0;for (int i01 = ir0; i01 < ir1; i01++) {for (int i00 = 0; i00 < ne00; i00++) {- const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);++ dst_ptr[id] = GGML_BF16_TO_FP32(*src0_ptr);+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }+ } else if (dst->type == GGML_TYPE_BF16) {+ size_t id = 0;+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);dst_ptr[id] = *src0_ptr;id++;@@ -7026,9 +7413,9 @@ static void ggml_compute_forward_dup_f32(id += ne00 * ir0;for (int i01 = ir0; i01 < ir1; i01++) {for (int i00 = 0; i00 < ne00; i00++) {- const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ const ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);- dst_ptr[id] = GGML_FP32_TO_FP16(*src0_ptr);+ dst_ptr[id] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(*src0_ptr));id++;}}@@ -7039,18 +7426,16 @@ static void ggml_compute_forward_dup_f32(GGML_ASSERT(false); // TODO: implement}}-return;}// dst counters-int64_t i10 = 0;int64_t i11 = 0;int64_t i12 = 0;int64_t i13 = 0;- if (dst->type == GGML_TYPE_F32) {+ if (dst->type == GGML_TYPE_BF16) {for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {i10 += ne00 * ir0;@@ -7071,7 +7456,59 @@ static void ggml_compute_forward_dup_f32(const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);- memcpy(dst_ptr, src0_ptr, sizeof(float));+ memcpy(dst_ptr, src0_ptr, sizeof(ggml_bf16_t));++ if (++i10 == ne00) {+ i10 = 0;+ if (++i11 == ne01) {+ i11 = 0;+ if (++i12 == ne02) {+ i12 = 0;+ if (++i13 == ne03) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ } else if (dst->type == GGML_TYPE_F16) {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ *(ggml_fp16_t *) dst_ptr = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(*(const ggml_bf16_t *) src0_ptr));if (++i10 == ne0) {i10 = 0;@@ -7102,7 +7539,7 @@ static void ggml_compute_forward_dup_f32(}}}- } else if (dst->type == GGML_TYPE_F16) {+ } else if (dst->type == GGML_TYPE_F32) {for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {i10 += ne00 * ir0;@@ -7123,7 +7560,7 @@ static void ggml_compute_forward_dup_f32(const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);- *(ggml_fp16_t *) dst_ptr = GGML_FP32_TO_FP16(*(const float *) src0_ptr);+ *(float *) dst_ptr = GGML_BF16_TO_FP32(*(const ggml_bf16_t *) src0_ptr);if (++i10 == ne0) {i10 = 0;@@ -7159,31 +7596,27 @@ static void ggml_compute_forward_dup_f32(}}-// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.-static void ggml_compute_forward_dup_bytes(+static void ggml_compute_forward_dup_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));- GGML_ASSERT(src0->type == dst->type);if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {return;}- if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {- ggml_compute_forward_dup_same_cont(params, dst);- return;- }-- GGML_TENSOR_UNARY_OP_LOCALS;+ GGML_TENSOR_UNARY_OP_LOCALS- const size_t type_size = ggml_type_size(src0->type);const int ith = params->ith; // thread indexconst int nth = params->nth; // number of threads+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {+ ggml_compute_forward_dup_same_cont(params, dst);+ return;+ }// parallelize by rowsconst int nr = ne01;@@ -7195,9 +7628,9 @@ static void ggml_compute_forward_dup_bytes(if (src0->type == dst->type &&ne00 == ne0 &&- nb00 == type_size && nb0 == type_size) {+ nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {// copy by rows- const size_t rs = ne00 * type_size;+ const size_t rs = ne00*nb00;for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {for (int64_t i01 = ir0; i01 < ir1; i01++) {@@ -7212,13 +7645,338 @@ static void ggml_compute_forward_dup_bytes(}if (ggml_is_contiguous(dst)) {- size_t id = 0;- char * dst_ptr = (char *) dst->data;- const size_t rs = ne00 * type_size;-- if (nb00 == type_size) {- // src0 is contigous on first dimension, copy by rows- for (int64_t i03 = 0; i03 < ne03; i03++) {+ // TODO: simplify+ if (nb00 == sizeof(float)) {+ if (dst->type == GGML_TYPE_F32) {+ size_t id = 0;+ const size_t rs = ne00 * nb00;+ char * dst_ptr = (char *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += rs * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;+ memcpy(dst_ptr + id, src0_ptr, rs);+ id += rs;+ }+ id += rs * (ne01 - ir1);+ }+ }+ } else if (type_traits[dst->type].from_float) {+ ggml_from_float_t const quantize_row_q = type_traits[dst->type].from_float;++ size_t id = 0;+ size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));+ char * dst_ptr = (char *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += rs * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ const float * src0_ptr = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);+ quantize_row_q(src0_ptr, dst_ptr + id, ne00);+ id += rs;+ }+ id += rs * (ne01 - ir1);+ }+ }+ } else {+ GGML_ASSERT(false); // TODO: implement+ }+ } else {+ //printf("%s: this is not optimal - fix me\n", __func__);++ if (dst->type == GGML_TYPE_F32) {+ size_t id = 0;+ float * dst_ptr = (float *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);++ dst_ptr[id] = *src0_ptr;+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }+ } else if (dst->type == GGML_TYPE_F16) {+ size_t id = 0;+ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);++ dst_ptr[id] = GGML_FP32_TO_FP16(*src0_ptr);+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }+ } else if (dst->type == GGML_TYPE_BF16) {+ size_t id = 0;+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);++ dst_ptr[id] = GGML_FP32_TO_BF16(*src0_ptr);+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }+ } else {+ GGML_ASSERT(false); // TODO: implement+ }+ }++ return;+ }++ // dst counters++ int64_t i10 = 0;+ int64_t i11 = 0;+ int64_t i12 = 0;+ int64_t i13 = 0;++ if (dst->type == GGML_TYPE_F32) {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ memcpy(dst_ptr, src0_ptr, sizeof(float));++ if (++i10 == ne0) {+ i10 = 0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ } else if (dst->type == GGML_TYPE_F16) {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ *(ggml_fp16_t *) dst_ptr = GGML_FP32_TO_FP16(*(const float *) src0_ptr);++ if (++i10 == ne0) {+ i10 = 0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ } else if (dst->type == GGML_TYPE_BF16) {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ *(ggml_bf16_t *) dst_ptr = GGML_FP32_TO_BF16(*(const float *) src0_ptr);++ if (++i10 == ne0) {+ i10 = 0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ } else {+ GGML_ASSERT(false); // TODO: implement+ }+}++// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.+static void ggml_compute_forward_dup_bytes(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));+ GGML_ASSERT(src0->type == dst->type);++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {+ ggml_compute_forward_dup_same_cont(params, dst);+ return;+ }++ GGML_TENSOR_UNARY_OP_LOCALS;++ const size_t type_size = ggml_type_size(src0->type);+ const int ith = params->ith; // thread index+ const int nth = params->nth; // number of threads+++ // parallelize by rows+ const int nr = ne01;+ // number of rows per thread+ const int dr = (nr + nth - 1) / nth;+ // row range for this thread+ const int ir0 = dr * ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (src0->type == dst->type &&+ ne00 == ne0 &&+ nb00 == type_size && nb0 == type_size) {+ // copy by rows+ const size_t rs = ne00 * type_size;+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ memcpy(+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),+ rs);+ }+ }+ }+ return;+ }++ if (ggml_is_contiguous(dst)) {+ size_t id = 0;+ char * dst_ptr = (char *) dst->data;+ const size_t rs = ne00 * type_size;++ if (nb00 == type_size) {+ // src0 is contigous on first dimension, copy by rows+ for (int64_t i03 = 0; i03 < ne03; i03++) {for (int64_t i02 = 0; i02 < ne02; i02++) {id += rs * ir0;for (int64_t i01 = ir0; i01 < ir1; i01++) {@@ -7327,6 +8085,10 @@ static void ggml_compute_forward_dup({ggml_compute_forward_dup_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_dup_bf16(params, dst);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_dup_f32(params, dst);@@ -7420,17 +8182,96 @@ static void ggml_compute_forward_add_f32(float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 );float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);- for (int64_t i0 = 0; i0 < ne0; ++i0) {- const int64_t i10 = i0 % ne10;- float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);+ for (int64_t i0 = 0; i0 < ne0; ++i0) {+ const int64_t i10 = i0 % ne10;+ float * src1_ptr = (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10);++ dst_ptr[i0] = src0_ptr[i0] + *src1_ptr;+ }+ }+ }+}++static void ggml_compute_forward_add_f16_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_BINARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ if (dst->type == GGML_TYPE_F32) {+ GGML_ASSERT( nb0 == sizeof(float));+ }+ else {+ GGML_ASSERT(dst->type == GGML_TYPE_F16);+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));+ }++ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (nb10 == sizeof(float)) {+ if (dst->type == GGML_TYPE_F16) {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);++ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);+ }+ }+ } else {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);- dst_ptr[i0] = src0_ptr[i0] + *src1_ptr;+ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i];+ }}}}+ else {+ // src1 is not contiguous+ GGML_ASSERT(false);+ }}-static void ggml_compute_forward_add_f16_f32(+static void ggml_compute_forward_add_bf16_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -7450,18 +8291,18 @@ static void ggml_compute_forward_add_f16_f32(GGML_TENSOR_BINARY_OP_LOCALS- GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);GGML_ASSERT(src1->type == GGML_TYPE_F32);if (dst->type == GGML_TYPE_F32) {GGML_ASSERT( nb0 == sizeof(float));}else {- GGML_ASSERT(dst->type == GGML_TYPE_F16);- GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));}- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));// rows per threadconst int dr = (nr + nth - 1)/nth;@@ -7471,19 +8312,19 @@ static void ggml_compute_forward_add_f16_f32(const int ir1 = MIN(ir0 + dr, nr);if (nb10 == sizeof(float)) {- if (dst->type == GGML_TYPE_F16) {+ if (dst->type == GGML_TYPE_BF16) {for (int ir = ir0; ir < ir1; ++ir) {// src0, src1 and dst are same shape => same indicesconst int i3 = ir/(ne2*ne1);const int i2 = (ir - i3*ne2*ne1)/ne1;const int i1 = (ir - i3*ne2*ne1 - i2*ne1);- ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);- ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);for (int i = 0; i < ne0; i++) {- dst_ptr[i] = GGML_FP32_TO_FP16(GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);}}} else {@@ -7494,11 +8335,11 @@ static void ggml_compute_forward_add_f16_f32(const int i1 = (ir - i3*ne2*ne1 - i2*ne1);float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);- ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);for (int i = 0; i < ne0; i++) {- dst_ptr[i] = GGML_FP16_TO_FP32(src0_ptr[i]) + src1_ptr[i];+ dst_ptr[i] = GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i];}}}@@ -7565,6 +8406,62 @@ static void ggml_compute_forward_add_f16_f16(}}+static void ggml_compute_forward_add_bf16_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_BINARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);++ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (nb10 == sizeof(ggml_bf16_t)) {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ ggml_bf16_t * src1_ptr = (ggml_bf16_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);++ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + GGML_BF16_TO_FP32(src1_ptr[i]));+ }+ }+ }+ else {+ // src1 is not contiguous+ GGML_ASSERT(false);+ }+}+static void ggml_compute_forward_add_q_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -7674,6 +8571,18 @@ static void ggml_compute_forward_add(GGML_ASSERT(false);}} break;+ case GGML_TYPE_BF16:+ {+ if (src1->type == GGML_TYPE_BF16) {+ ggml_compute_forward_add_bf16_bf16(params, dst);+ }+ else if (src1->type == GGML_TYPE_F32) {+ ggml_compute_forward_add_bf16_f32(params, dst);+ }+ else {+ GGML_ASSERT(false);+ }+ } break;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -7932,6 +8841,110 @@ static void ggml_compute_forward_add1_q_f32(}}+static void ggml_compute_forward_add1_bf16_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_is_scalar(src1));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // scalar to add+ const float v = *(float *) src1->data;++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_UNARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);++ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int ir = ir0; ir < ir1; ++ir) {+ // src0 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);+ }+ }+}++static void ggml_compute_forward_add1_bf16_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_is_scalar(src1));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // scalar to add+ const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_UNARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);++ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int ir = ir0; ir < ir1; ++ir) {+ // src0 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);+ }+ }+}+static void ggml_compute_forward_add1(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -7956,6 +8969,18 @@ static void ggml_compute_forward_add1(GGML_ASSERT(false);}} break;+ case GGML_TYPE_BF16:+ {+ if (src1->type == GGML_TYPE_BF16) {+ ggml_compute_forward_add1_bf16_bf16(params, dst);+ }+ else if (src1->type == GGML_TYPE_F32) {+ ggml_compute_forward_add1_bf16_f32(params, dst);+ }+ else {+ GGML_ASSERT(false);+ }+ } break;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -8084,6 +9109,7 @@ static void ggml_compute_forward_acc(ggml_compute_forward_acc_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -8605,6 +9631,40 @@ static void ggml_compute_forward_sum_f16(((ggml_fp16_t *) dst->data)[0] = GGML_FP32_TO_FP16(sum);}+static void ggml_compute_forward_sum_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ assert(params->ith == 0);+ assert(ggml_is_scalar(dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ assert(src0->nb[0] == sizeof(ggml_bf16_t));++ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)++ float sum = 0;+ float row_sum = 0;++ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ ggml_vec_sum_bf16_ggf(ne00,+ &row_sum,+ (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));+ sum += row_sum;+ }+ }+ }+ ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);+}+static void ggml_compute_forward_sum(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -8620,6 +9680,10 @@ static void ggml_compute_forward_sum({ggml_compute_forward_sum_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_sum_bf16(params, dst);+ } break;default:{GGML_ASSERT(false);@@ -8894,6 +9958,7 @@ static void ggml_compute_forward_repeat(switch (src0->type) {case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_I16:{ggml_compute_forward_repeat_f16(params, dst);@@ -9337,7 +10402,53 @@ static void ggml_compute_forward_elu(// ggml_compute_forward_relu-static void ggml_compute_forward_relu_f32(+static void ggml_compute_forward_relu_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ assert(params->ith == 0);+ assert(ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int n = ggml_nrows(src0);+ const int nc = src0->ne[0];++ assert(dst->nb[0] == sizeof(float));+ assert(src0->nb[0] == sizeof(float));++ for (int i = 0; i < n; i++) {+ ggml_vec_relu_f32(nc,+ (float *) ((char *) dst->data + i*( dst->nb[1])),+ (float *) ((char *) src0->data + i*(src0->nb[1])));+ }+}++static void ggml_compute_forward_relu(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_relu_f32(params, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++// ggml_compute_forward_sigmoid++static void ggml_compute_forward_sigmoid_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -9357,13 +10468,13 @@ static void ggml_compute_forward_relu_f32(assert(src0->nb[0] == sizeof(float));for (int i = 0; i < n; i++) {- ggml_vec_relu_f32(nc,+ ggml_vec_sigmoid_f32(nc,(float *) ((char *) dst->data + i*( dst->nb[1])),(float *) ((char *) src0->data + i*(src0->nb[1])));}}-static void ggml_compute_forward_relu(+static void ggml_compute_forward_sigmoid(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -9372,7 +10483,7 @@ static void ggml_compute_forward_relu(switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_relu_f32(params, dst);+ ggml_compute_forward_sigmoid_f32(params, dst);} break;default:{@@ -10277,8 +11388,6 @@ static void ggml_compute_forward_mul_mat(// nb01 >= nb00 - src0 is not transposed// compute by src0 rows- GGML_ASSERT(params->type == GGML_TASK_TYPE_COMPUTE);-#if defined(GGML_USE_CLBLAST)if (ggml_cl_can_mul_mat(src0, src1, dst)) {if (params->ith == 0 && params->type == GGML_TASK_TYPE_COMPUTE) {@@ -10728,6 +11837,10 @@ IQK_MulMat_Not_Available:;#undef MMID_MATRIX_ROW}+#define GGML_SOFT_MAX_UNROLL 4+#define GGML_VEC_DOT_UNROLL 2+#define GGML_VEC_MAD_UNROLL 32+// ggml_compute_forward_out_prodstatic void ggml_compute_forward_out_prod_f32(@@ -11235,6 +12348,7 @@ static void ggml_compute_forward_set(ggml_compute_forward_set_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -11845,7 +12959,6 @@ static void ggml_compute_forward_soft_max_f32(const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- const struct ggml_tensor * src2 = dst->src[2];assert(ggml_is_contiguous(dst));assert(ggml_are_same_shape(src0, dst));@@ -11871,8 +12984,8 @@ static void ggml_compute_forward_soft_max_f32(// TODO: is this supposed to be ceil instead of floor?// https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370- const uint32_t n_head_kv = ne02;- const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head_kv));+ const uint32_t n_head = ne02;+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);@@ -11889,13 +13002,13 @@ static void ggml_compute_forward_soft_max_f32(float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;- // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching- ggml_fp16_t * pos_f16 = src2 ? (ggml_fp16_t *) src2->data : src0->data;- float * pos_f32 = src2 ? (float *) src2->data : src0->data;-- const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16) || (src2 && src2->type == GGML_TYPE_F16);+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);for (int i1 = ir0; i1 < ir1; i1++) {+ // ALiBi+ const uint32_t h = (i1/ne01)%ne02; // head+ const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;+float * sp = (float *)((char *) src0->data + i1*src0->nb[1]);float * dp = (float *)((char *) dst->data + i1*dst->nb[1]);@@ -11908,27 +13021,11 @@ static void ggml_compute_forward_soft_max_f32(if (mp_f32) {if (use_f16) {for (int i = 0; i < nc; ++i) {- wp[i] += GGML_FP16_TO_FP32(mp_f16[i]);- }- } else {- for (int i = 0; i < nc; ++i) {- wp[i] += mp_f32[i];- }- }- }-- // ALiBi bias- if (max_bias > 0.0f) {- const uint32_t h = (i1/ne01)%ne02; // head- const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1);-- if (use_f16) {- for (int i = 0; i < nc; ++i) {- wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[i]);+ wp[i] += slope*GGML_FP16_TO_FP32(mp_f16[i]);}} else {for (int i = 0; i < nc; ++i) {- wp[i] += slope*pos_f32[i];+ wp[i] += slope*mp_f32[i];}}}@@ -11943,7 +13040,7 @@ static void ggml_compute_forward_soft_max_f32(float max = -INFINITY;ggml_vec_max_f32(nc, &max, wp);- float sum = ggml_vec_soft_max_f32(nc, dp, wp, max);+ ggml_float sum = ggml_vec_soft_max_f32(nc, dp, wp, max);assert(sum > 0.0);sum = 1.0/sum;@@ -12075,177 +13172,6 @@ static void ggml_compute_forward_soft_max_back(}}-// ggml_compute_forward_alibi--static void ggml_compute_forward_alibi_f32(- const struct ggml_compute_params * params,- struct ggml_tensor * dst) {-- const struct ggml_tensor * src0 = dst->src[0];-- assert(params->ith == 0);-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));-- const int64_t ne0 = src0->ne[0]; // all_seq_len = n_past + ne1- const int64_t ne1 = src0->ne[1]; // seq_len_without_past- const int64_t ne2 = src0->ne[2]; // n_head -> this is k- //const int64_t ne3 = src0->ne[3]; // 1 -> bsz-- const int64_t n = ggml_nrows(src0);- const int64_t ne2_ne3 = n/ne1; // ne2*ne3-- const size_t nb0 = src0->nb[0];- const size_t nb1 = src0->nb[1];- const size_t nb2 = src0->nb[2];- //const int nb3 = src0->nb[3];-- GGML_ASSERT(nb0 == sizeof(float));- GGML_ASSERT(n_head == ne2);-- // add alibi to src0 (KQ_scaled)- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));-- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);-- for (int64_t k = 0; k < ne2_ne3; k++) {- // TODO: k*nb2 or k*nb3- float m_k;-- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);- }-- for (int64_t i = 0; i < ne0; i++) {- for (int64_t j = 0; j < ne1; j++) {- float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);- float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);- pdst[0] = i * m_k + src[0];- }- }- }-}--static void ggml_compute_forward_alibi_f16(- const struct ggml_compute_params * params,- struct ggml_tensor * dst) {-- const struct ggml_tensor * src0 = dst->src[0];-- assert(params->ith == 0);-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));-- const int ne0 = src0->ne[0]; // all_seq_len = n_past + ne1- const int ne1 = src0->ne[1]; // seq_len_without_past- const int ne2 = src0->ne[2]; // n_head -> this is k- //const int ne3 = src0->ne[3]; // 1 -> bsz-- const int n = ggml_nrows(src0);- const int ne2_ne3 = n/ne1; // ne2*ne3-- const int nb0 = src0->nb[0];- const int nb1 = src0->nb[1];- const int nb2 = src0->nb[2];- //const int nb3 = src0->nb[3];-- GGML_ASSERT(nb0 == sizeof(ggml_fp16_t));- //GGML_ASSERT(ne1 + n_past == ne0); (void) n_past;- GGML_ASSERT(n_head == ne2);-- // add alibi to src0 (KQ_scaled)- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));-- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);-- for (int k = 0; k < ne2_ne3; k++) {- // TODO: k*nb2 or k*nb3- float m_k;-- if (k < n_heads_log2_floor) {- m_k = powf(m0, k + 1);- } else {- m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);- }-- for (int i = 0; i < ne0; i++) {- for (int j = 0; j < ne1; j++) {- ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);- float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);-- // we return F32- pdst[0] = i * m_k + GGML_FP16_TO_FP32(src[0]);- }- }- }-}--static void ggml_compute_forward_alibi(- const struct ggml_compute_params * params,- struct ggml_tensor * dst) {-- const struct ggml_tensor * src0 = dst->src[0];-- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_alibi_f16(params, dst);- } break;- case GGML_TYPE_F32:- {- ggml_compute_forward_alibi_f32(params, dst);- } break;- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_Q8_1:- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_IQ2_XXS:- case GGML_TYPE_IQ2_XS:- case GGML_TYPE_IQ3_XXS:- case GGML_TYPE_IQ1_S:- case GGML_TYPE_IQ1_M:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_IQ4_XS:- case GGML_TYPE_IQ3_S:- case GGML_TYPE_IQ2_S:- case GGML_TYPE_Q8_K:- case GGML_TYPE_I8:- case GGML_TYPE_I16:- case GGML_TYPE_I32:- case GGML_TYPE_I64:- case GGML_TYPE_F64:- case GGML_TYPE_COUNT:- {- GGML_ASSERT(false);- } break;- }-}-// ggml_compute_forward_clampstatic void ggml_compute_forward_clamp_f32(@@ -12302,6 +13228,7 @@ static void ggml_compute_forward_clamp(ggml_compute_forward_clamp_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -12400,6 +13327,7 @@ static void ggml_compute_forward_rope_f32(const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];+ const struct ggml_tensor * src2 = dst->src[2];if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {return;@@ -12459,6 +13387,17 @@ static void ggml_compute_forward_rope_f32(const bool is_neox = mode & 2;const bool is_glm = mode & 4;+ const float * freq_factors = NULL;+ if (is_neox) {+ if (src2 != NULL) {+ GGML_ASSERT(src2->type == GGML_TYPE_F32);+ GGML_ASSERT(src2->ne[0] >= n_dims / 2);+ freq_factors = (const float *) src2->data;+ }+ } else {+ GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for mode 1");+ }+// backward process uses inverse rotation by cos and sin.// cos and sin build a rotation matrix, where the inverse is the transpose.// this essentially just switches the sign of sin.@@ -12535,10 +13474,11 @@ static void ggml_compute_forward_rope_f32(// simplified from `(ib * n_dims + ic) * inv_ndims`float cur_rot = inv_ndims * ic - ib;+ float freq_factor = freq_factors ? freq_factors[ic/2] : 1.0f;float cos_theta, sin_theta;rope_yarn(- theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,+ theta_base/freq_factor, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,&cos_theta, &sin_theta);sin_theta *= sin_sign;@@ -13467,25 +14407,28 @@ static void ggml_compute_forward_upscale_f32(return;}- GGML_ASSERT(src0->nb[0] == sizeof(float));+ GGML_ASSERT(src0->type == GGML_TYPE_F32);const int ith = params->ith;const int nth = params->nth;GGML_TENSOR_UNARY_OP_LOCALS- const int scale_factor = dst->op_params[0];+ const float sf0 = (float)ne0/src0->ne[0];+ const float sf1 = (float)ne1/src0->ne[1];+ const float sf2 = (float)ne2/src0->ne[2];+ const float sf3 = (float)ne3/src0->ne[3];// TODO: optimizefor (int64_t i3 = 0; i3 < ne3; i3++) {- const int64_t i03 = i3;+ const int64_t i03 = i3 / sf3;for (int64_t i2 = ith; i2 < ne2; i2 += nth) {- const int64_t i02 = i2;+ const int64_t i02 = i2 / sf2;for (int64_t i1 = 0; i1 < ne1; i1++) {- const int64_t i01 = i1 / scale_factor;+ const int64_t i01 = i1 / sf1;for (int64_t i0 = 0; i0 < ne0; i0++) {- const int64_t i00 = i0 / scale_factor;+ const int64_t i00 = i0 / sf0;const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);float * y = (float *)((char *) dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);@@ -13515,6 +14458,7 @@ static void ggml_compute_forward_upscale(}}+// ggml_compute_forward_padstatic void ggml_compute_forward_pad_f32(@@ -13868,29 +14812,7 @@ static void ggml_compute_forward_flash_attn_f32(vvexpf(S, S, &Mup);ggml_vec_sum_f32(Mup, &sum, S);#else- uint16_t scvt[GGML_SOFT_MAX_UNROLL]; UNUSED(scvt);- ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };-- for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) {- if (i >= masked_begin) {- break;- }- float * SS = S + i;-- for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) {- if (i + j >= masked_begin) {- break;- } else {- const float val = expf(SS[j] - max);- sump[j] += (ggml_float)val;- SS[j] = val;- }- }- }-- for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {- sum += sump[i];- }+ sum = ggml_vec_soft_max_f32(Mup, S, S, max);#endif}@@ -14072,21 +14994,7 @@ static void ggml_compute_forward_flash_attn_f16(vvexpf(S, S, &Mup);ggml_vec_sum_f32(Mup, &sum, S);#else- ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };-- for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) {- float * SS = S + i;-- for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) {- const float val = expf(SS[j] - max);- sump[j] += (ggml_float)val;- SS[j] = val;- }- }-- for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {- sum += sump[i];- }+ sum = ggml_vec_soft_max_f32(Mup, S, S, max);#endif}@@ -14199,9 +15107,10 @@ static void ggml_compute_forward_flash_attn_ext_f16(GGML_ASSERT(ne0 == D);GGML_ASSERT(ne2 == N);- GGML_ASSERT(nbq0 == sizeof(float));- GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));- GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));+ // input tensor rows must be contiguous+ GGML_ASSERT(nbq0 == ggml_type_size(q->type));+ GGML_ASSERT(nbk0 == ggml_type_size(k->type));+ GGML_ASSERT(nbv0 == ggml_type_size(v->type));GGML_ASSERT(neq0 == D);GGML_ASSERT(nek0 == D);@@ -14243,8 +15152,22 @@ static void ggml_compute_forward_flash_attn_ext_f16(const int ir0 = dr*ith;const int ir1 = MIN(ir0 + dr, nr);- float scale = 1.0f;- memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ float scale = 1.0f;+ float max_bias = 0.0f;++ memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));+ memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));++ const uint32_t n_head = neq2;+ const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));++ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);++ enum ggml_type const k_vec_dot_type = type_traits[k->type].vec_dot_type;+ ggml_from_float_t const q_to_vec_dot = type_traits[k_vec_dot_type].from_float;+ ggml_vec_dot_t const kq_vec_dot = type_traits[k->type].vec_dot;+ ggml_to_float_t const v_to_float = type_traits[v->type].to_float;// loop over n_batch and n_headfor (int ir = ir0; ir < ir1; ++ir) {@@ -14253,14 +15176,22 @@ static void ggml_compute_forward_flash_attn_ext_f16(const int iq2 = (ir - iq3*neq2*neq1)/neq1;const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);- float S = 0.0f;- float M = -INFINITY;+ const uint32_t h = iq2; // head index+ const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;++ float S = 0.0f; // sum+ float M = -INFINITY; // maximum KQ value- float * V32 = (float *) params->wdata + ith*(2*D + CACHE_LINE_SIZE_F32);- ggml_fp16_t * Q16 = (ggml_fp16_t *) (V32); // reuse memory- ggml_fp16_t * V16 = (ggml_fp16_t *) (V32 + D);+ float * VKQ32 = (float *) params->wdata + ith*(3*D + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator+ float * V32 = (VKQ32 + 1*D); // (temporary) FP32 V buffer+ ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*D); // (temporary) FP16 VKQ accumulator+ ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*D); // (temporary) buffer for Q converted to quantized/FP16- memset(V16, 0, D*sizeof(ggml_fp16_t));+ if (v->type == GGML_TYPE_F16) {+ memset(VKQ16, 0, D*sizeof(ggml_fp16_t));+ } else {+ memset(VKQ32, 0, D*sizeof(float));+ }const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1]) : NULL;@@ -14272,61 +15203,79 @@ static void ggml_compute_forward_flash_attn_ext_f16(const int iv3 = iq3 / rv3;const int iv2 = iq2 / rv2;+ const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));+ q_to_vec_dot(pq, Q_q, D);+// online softmax / attention// loop over n_kv and n_head_kv// ref: https://arxiv.org/pdf/2112.05682.pdffor (int64_t ic = 0; ic < nek1; ++ic) {- const float mv = mp ? GGML_FP16_TO_FP32(mp[ic]) : 0.0f;+ const float mv = mp ? slope*GGML_FP16_TO_FP32(mp[ic]) : 0.0f;if (mv == -INFINITY) {continue;}- float s;+ float s; // KQ value- // convert Q to F16 in V32- {- const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));+ const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);+ kq_vec_dot(D, &s, 0, k_data, 0, Q_q, 0, 1);- for (int64_t d = 0; d < D; ++d) {- Q16[d] = GGML_FP32_TO_FP16(pq[d]);- }- }+ s = s*scale + mv; // scale KQ value and apply mask- ggml_vec_dot_f16(D,- &s, 0,- (ggml_fp16_t *) ((char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,- Q16, 0, 1);+ const float Mold = M;- s = s*scale + mv;+ float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value+ float vs = 1.0f; // post-softmax KQ value, expf(s - M)- const float Mold = M;+ const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));- float ms = 1.0f;- float vs = 1.0f;+ if (v->type== GGML_TYPE_F16) {+ if (s > M) {+ // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f+ M = s;+ ms = expf(Mold - M);- if (s > M) {- M = s;- ms = expf(Mold - M);+ // V = V*expf(Mold - M)+ ggml_vec_scale_f16(D, VKQ16, ms);+ } else {+ // no new maximum, ms == 1.0f, vs != 1.0f+ vs = expf(s - M);+ }- // V = V*expf(Mold - M)- ggml_vec_scale_f16(D, V16, ms);+ // V += v*expf(s - M)+ ggml_vec_mad_f16(D, VKQ16, (const ggml_fp16_t *) v_data, vs);} else {- vs = expf(s - M);- }+ if (s > M) {+ // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f+ M = s;+ ms = expf(Mold - M);++ // V = V*expf(Mold - M)+ ggml_vec_scale_f32(D, VKQ32, ms);+ } else {+ // no new maximum, ms == 1.0f, vs != 1.0f+ vs = expf(s - M);+ }- const ggml_fp16_t * v16 = (const ggml_fp16_t *) ((char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));+ v_to_float(v_data, V32, D);- // V += v*expf(s - M)- ggml_vec_mad_f16(D, V16, v16, vs);+ // V += v*expf(s - M)+ ggml_vec_mad_f32(D, VKQ32, V32, vs);+ }- S = S*ms + vs;+ S = S*ms + vs; // scale and increment sum with partial sum}- // V /= S- for (int64_t d = 0; d < D; ++d) {- V32[d] = GGML_FP16_TO_FP32(V16[d])/S;+ if (v->type == GGML_TYPE_F16) {+ for (int64_t d = 0; d < D; ++d) {+ VKQ32[d] = GGML_FP16_TO_FP32(VKQ16[d]);+ }}+ // V /= S+ const float S_inv = 1.0f/S;+ ggml_vec_scale_f32(D, VKQ32, S_inv);+// dst indicesconst int i1 = iq1;const int i2 = iq2;@@ -14336,7 +15285,7 @@ static void ggml_compute_forward_flash_attn_ext_f16(//memcpy((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3), V, nev0*sizeof(float));// permute(0, 2, 1, 3)- memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, V32, nb1);+ memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1);}}@@ -14347,7 +15296,7 @@ static void ggml_compute_forward_flash_attn_ext(const struct ggml_tensor * v,const struct ggml_tensor * mask,struct ggml_tensor * dst) {- switch (dst->op_params[1]) {+ switch (dst->op_params[2]) {case GGML_PREC_DEFAULT:case GGML_PREC_F32:{@@ -14701,30 +15650,7 @@ static void ggml_compute_forward_flash_attn_back_f32(vvexpf(SM, SM, &Mup);ggml_vec_sum_f32(Mup, &sum, SM);#else- uint16_t scvt[GGML_SOFT_MAX_UNROLL]; UNUSED(scvt);- ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };-- for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) {- if (i >= masked_begin) {- break;- }- float * SR = S + i;- float * SW = SM + i;-- for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) {- if (i + j >= masked_begin) {- break;- } else {- const float val = expf(SR[j] - max);- sump[j] += (ggml_float)val;- SW[j] = val;- }- }- }-- for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {- sum += sump[i];- }+ sum = ggml_vec_soft_max_f32(Mup, SM, S, max);#endif}@@ -15306,6 +16232,10 @@ static void ggml_compute_forward_unary({ggml_compute_forward_relu(params, dst);} break;+ case GGML_UNARY_OP_SIGMOID:+ {+ ggml_compute_forward_sigmoid(params, dst);+ } break;case GGML_UNARY_OP_GELU:{ggml_compute_forward_gelu(params, dst);@@ -15372,6 +16302,7 @@ static void ggml_compute_forward_get_rel_pos(switch (src0->type) {case GGML_TYPE_F16:+ case GGML_TYPE_BF16:{ggml_compute_forward_get_rel_pos_f16(params, dst);} break;@@ -15745,25 +16676,15 @@ static void ggml_compute_forward_cross_entropy_loss_f32(assert(!isnan(s1[i]));}#endif+// soft_max- ggml_float sum = 0.0;- {- float max = -INFINITY;- ggml_vec_max_f32(nc, &max, s0);-- uint16_t scvt; UNUSED(scvt);- for (int i = 0; i < nc; i++) {- const float s = s0[i] - max;- const float val = expf(s);- sum += (ggml_float)val;- st[i] = val;- }+ float max = -INFINITY;+ ggml_vec_max_f32(nc, &max, s0);+ ggml_float sum = ggml_vec_soft_max_f32(nc, st, s0, max);+ assert(sum > 0.0);+ sum = (1.0 - eps) / sum;- assert(sum > 0.0);- // sum = 1.0/sum;- }// avoid log(0) by rescaling from [0..1] to [eps..1]- sum = (1.0 - eps) / sum;ggml_vec_scale_f32(nc, st, sum);ggml_vec_add1_f32(nc, st, st, eps);ggml_vec_log_f32(nc, st, st);@@ -15853,22 +16774,11 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(#endif// soft_max- ggml_float sum = 0.0;- {- float max = -INFINITY;- ggml_vec_max_f32(nc, &max, s0);-- uint16_t scvt; UNUSED(scvt);- for (int i = 0; i < nc; i++) {- const float s = s0[i] - max;- const float val = expf(s);- sum += (ggml_float)val;- ds0[i] = val;- }-- assert(sum > 0.0);- sum = (1.0 - eps)/sum;- }+ float max = -INFINITY;+ ggml_vec_max_f32(nc, &max, s0);+ ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);+ assert(sum > 0.0);+ sum = (1.0 - eps) / sum;// grad(src0) = (softmax(src0) - src1) * grad(cross_entropy_loss(src0, src1)) / nrggml_vec_scale_f32(nc, ds0, sum);@@ -16090,10 +17000,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm{ggml_compute_forward_rope_back(params, tensor);} break;- case GGML_OP_ALIBI:- {- ggml_compute_forward_alibi(params, tensor);- } break;case GGML_OP_CLAMP:{ggml_compute_forward_clamp(params, tensor);@@ -16535,6 +17441,7 @@ static struct ggml_tensor * ggml_sub_or_set(struct ggml_context * ctx, struct ggstatic void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, struct ggml_hash_set zero_table) {struct ggml_tensor * src0 = tensor->src[0];struct ggml_tensor * src1 = tensor->src[1];+ struct ggml_tensor * src2 = tensor->src[2];switch (tensor->op) {case GGML_OP_DUP:@@ -17066,6 +17973,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorggml_rope_back(ctx,tensor->grad,src1,+ src2,n_dims,mode,n_ctx,@@ -17105,6 +18013,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorggml_rope_impl(ctx,tensor->grad,src1,+ src2,n_dims,mode,n_ctx,@@ -17121,10 +18030,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorzero_table);}} break;- case GGML_OP_ALIBI:- {- GGML_ASSERT(false); // TODO: not implemented- } break;case GGML_OP_CLAMP:{GGML_ASSERT(false); // TODO: not implemented@@ -17190,7 +18095,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensormasked);}- struct ggml_tensor * src2 = tensor->src[2];const int64_t elem_q = ggml_nelements(src0);const int64_t elem_k = ggml_nelements(src1);const int64_t elem_v = ggml_nelements(src2);@@ -17295,6 +18199,10 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorzero_table);}} break;+ case GGML_UNARY_OP_SIGMOID:+ {+ GGML_ASSERT(false); // TODO: not implemented+ } break;case GGML_UNARY_OP_GELU:{GGML_ASSERT(false); // TODO: not implemented@@ -17746,33 +18654,6 @@ static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }static void clear_numa_thread_affinity(void) {}#endif-struct ggml_compute_state_shared {- const struct ggml_cgraph * cgraph;- const struct ggml_cplan * cplan;-- int64_t perf_node_start_cycles;- int64_t perf_node_start_time_us;-- const int n_threads;-- // synchronization primitives- atomic_int n_active; // num active threads- atomic_int node_n; // active graph node- atomic_int node_task; // active graph node task phase- struct ggml_barrier barrier;-- ggml_abort_callback abort_callback; // abort ggml_graph_compute when true- void * abort_callback_data;-};--struct ggml_compute_state {- ggml_thread_t thrd;- int ith;- struct ggml_compute_state_shared * shared;- enum ggml_status ec;- bool is_main_thread; // [jart]-};-static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) {int64_t cycles_cur = ggml_perf_cycles() - st->perf_node_start_cycles;int64_t time_us_cur = ggml_perf_time_us() - st->perf_node_start_time_us;@@ -17823,6 +18704,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_case GGML_UNARY_OP_TANH:case GGML_UNARY_OP_ELU:case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_SIGMOID:case GGML_UNARY_OP_HARDSWISH: // to opt for multiple threadscase GGML_UNARY_OP_HARDSIGMOID: // to opt for multiple threads{@@ -17896,10 +18778,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_{n_tasks = n_threads;} break;- case GGML_OP_ALIBI:- {- n_tasks = 1; //TODO- } break;case GGML_OP_CLAMP:{n_tasks = 1; //TODO@@ -18269,7 +19147,10 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threacase GGML_OP_CPY:case GGML_OP_DUP:{- if (ggml_is_quantized(node->type)) {+ if (ggml_is_quantized(node->type) ||+ // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32+ (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||+ (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16)) {cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;}} break;@@ -18397,7 +19278,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa{const int64_t ne00 = node->src[0]->ne[0]; // D- cur = 2*sizeof(float)*ne00*n_tasks; // 2x head size+ cur = 3*sizeof(float)*ne00*n_tasks; // 3x head size/thread} break;case GGML_OP_FLASH_FF:{@@ -20719,7 +21600,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i}// read the tensor infos- {+ if (ctx->header.n_tensors > 0) {ctx->infos = GGML_CALLOC(ctx->header.n_tensors, sizeof(struct gguf_tensor_info));for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {@@ -21577,6 +22458,10 @@ int ggml_cpu_has_avx512_vnni(void) {return X86_HAVE(AVX512_VNNI);}+int ggml_cpu_has_avx512_bf16(void) {+ return X86_HAVE(AVX512_BF16);+}+int ggml_cpu_has_fma(void) {return X86_HAVE(FMA);}diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex bebc894d..8acee96b 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -470,7 +470,6 @@ extern "C" {GGML_OP_SOFT_MAX_BACK,GGML_OP_ROPE,GGML_OP_ROPE_BACK,- GGML_OP_ALIBI,GGML_OP_CLAMP,GGML_OP_CONV_TRANSPOSE_1D,GGML_OP_IM2COL,@@ -522,6 +521,7 @@ extern "C" {GGML_UNARY_OP_TANH,GGML_UNARY_OP_ELU,GGML_UNARY_OP_RELU,+ GGML_UNARY_OP_SIGMOID,GGML_UNARY_OP_GELU,GGML_UNARY_OP_GELU_QUICK,GGML_UNARY_OP_SILU,@@ -567,7 +567,8 @@ extern "C" {// n-dimensional tensorstruct ggml_tensor {enum ggml_type type;- enum ggml_backend_type backend;++ GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");struct ggml_backend_buffer * buffer;@@ -772,7 +773,8 @@ extern "C" {GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars- GGML_API GGML_CALL bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);+ GGML_API GGML_CALL bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);+ GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);// use this to compute the memory overhead of a tensorGGML_API size_t ggml_tensor_overhead(void);@@ -1080,6 +1082,14 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);+ GGML_API struct ggml_tensor * ggml_sigmoid(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sigmoid_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);+GGML_API struct ggml_tensor * ggml_gelu(struct ggml_context * ctx,struct ggml_tensor * a);@@ -1434,15 +1444,13 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a);- // fused soft_max(a*scale + mask + pos[i]*(ALiBi slope))+ // fused soft_max(a*scale + mask*(ALiBi slope))// mask is optional- // pos is required when max_bias > 0.0f// max_bias = 0.0f for no ALiBiGGML_API struct ggml_tensor * ggml_soft_max_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * mask,- struct ggml_tensor * pos,float scale,float max_bias);@@ -1463,6 +1471,7 @@ extern "C" {// if mode & 4 == 1, ChatGLM style//// b is an int32 vector with size a->ne[2], it contains the positions+ // c is freq factors (e.g. phi3-128k), (optional)GGML_API struct ggml_tensor * ggml_rope(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1481,10 +1490,11 @@ extern "C" {int n_ctx);// custom RoPE- GGML_API struct ggml_tensor * ggml_rope_custom(+ GGML_API struct ggml_tensor * ggml_rope_ext(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -1497,10 +1507,11 @@ extern "C" {float beta_slow);// in-place, returns view(a)- GGML_API struct ggml_tensor * ggml_rope_custom_inplace(+ GGML_API struct ggml_tensor * ggml_rope_ext_inplace(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -1512,18 +1523,41 @@ extern "C" {float beta_fast,float beta_slow);- // compute correction dims for YaRN RoPE scaling- GGML_CALL void ggml_rope_yarn_corr_dims(- int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]);+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode,+ int n_ctx,+ int n_orig_ctx,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow),+ "use ggml_rope_ext instead");- // xPos RoPE, in-place, returns view(a)- GGML_API struct ggml_tensor * ggml_rope_xpos_inplace(+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,int n_dims,- float base,- bool down);+ int mode,+ int n_ctx,+ int n_orig_ctx,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow),+ "use ggml_rope_ext_inplace instead");++ // compute correction dims for YaRN RoPE scaling+ GGML_CALL void ggml_rope_yarn_corr_dims(+ int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]);// rotary position embedding backward, i.e compute dx from dy// a - dy@@ -1531,6 +1565,7 @@ extern "C" {struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b,+ struct ggml_tensor * c,int n_dims,int mode,int n_ctx,@@ -1544,16 +1579,6 @@ extern "C" {float xpos_base,bool xpos_down);- // alibi position embedding- // in-place, returns view(a)- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_alibi(- struct ggml_context * ctx,- struct ggml_tensor * a,- int n_past,- int n_head,- float bias_max),- "use ggml_soft_max_ext instead (will be removed in Mar 2024)");-// clamp// in-place, returns view(a)GGML_API struct ggml_tensor * ggml_clamp(@@ -1683,12 +1708,24 @@ extern "C" {float p1);// nearest interpolate+ // multiplies ne0 and ne1 by scale factor// used in stable-diffusionGGML_API struct ggml_tensor * ggml_upscale(struct ggml_context * ctx,struct ggml_tensor * a,int scale_factor);+ // nearest interpolate+ // nearest interpolate to specified dimensions+ // used in tortoise.cpp+ GGML_API struct ggml_tensor * ggml_upscale_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int ne0,+ int ne1,+ int ne2,+ int ne3);+// pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]GGML_API struct ggml_tensor * ggml_pad(struct ggml_context * ctx,@@ -1750,7 +1787,8 @@ extern "C" {struct ggml_tensor * k,struct ggml_tensor * v,struct ggml_tensor * mask,- float scale);+ float scale,+ float max_bias);GGML_API void ggml_flash_attn_ext_set_prec(struct ggml_tensor * a,@@ -2385,6 +2423,7 @@ extern "C" {GGML_API int ggml_cpu_has_avx512 (void);GGML_API int ggml_cpu_has_avx512_vbmi(void);GGML_API int ggml_cpu_has_avx512_vnni(void);+ GGML_API int ggml_cpu_has_avx512_bf16(void);GGML_API int ggml_cpu_has_fma (void);GGML_API int ggml_cpu_has_neon (void);GGML_API int ggml_cpu_has_arm_fma (void);diff --git a/llama.cpp/grammar-parser.cpp b/llama.cpp/grammar-parser.cppindex f90b1523..b5bc7d49 100644--- a/llama.cpp/grammar-parser.cpp+++ b/llama.cpp/grammar-parser.cpp@@ -1,6 +1,3 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*--// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi-#include "grammar-parser.h"#include <cstdint>#include <cwchar>@@ -29,7 +26,7 @@ namespace grammar_parser {static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());- auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id));+ auto result = state.symbol_ids.emplace(std::string(src, len), next_id);return result.first->second;}@@ -145,7 +142,7 @@ namespace grammar_parser {pos++;last_sym_start = out_elements.size();while (*pos != '"') {- if (!*pos) { // [jart] don't sync until upstream fixes bug+ if (!*pos) {throw std::runtime_error("unexpected end of input");}auto char_pair = parse_char(pos);@@ -162,7 +159,7 @@ namespace grammar_parser {}last_sym_start = out_elements.size();while (*pos != ']') {- if (!*pos) { // [jart] don't sync until upstream fixes bug+ if (!*pos) {throw std::runtime_error("unexpected end of input");}auto char_pair = parse_char(pos);@@ -173,7 +170,7 @@ namespace grammar_parser {out_elements.push_back({type, char_pair.first});if (pos[0] == '-' && pos[1] != ']') {- if (!pos[1]) { // [jart] don't sync until upstream fixes bug+ if (!pos[1]) {throw std::runtime_error("unexpected end of input");}auto endchar_pair = parse_char(pos + 1);diff --git a/llama.cpp/json-schema-to-grammar.cpp b/llama.cpp/json-schema-to-grammar.cppindex 0f8f1b1d..9a71f5d8 100644--- a/llama.cpp/json-schema-to-grammar.cpp+++ b/llama.cpp/json-schema-to-grammar.cpp@@ -272,7 +272,7 @@ private:if (literal.empty()) {return false;}- ret.push_back(std::make_pair(literal, true));+ ret.emplace_back(literal, true);literal.clear();return true;};@@ -298,7 +298,7 @@ private:while (i < length) {char c = sub_pattern[i];if (c == '.') {- seq.push_back(std::make_pair(get_dot(), false));+ seq.emplace_back(get_dot(), false);i++;} else if (c == '(') {i++;@@ -307,7 +307,7 @@ private:_warnings.push_back("Unsupported pattern syntax");}}- seq.push_back(std::make_pair("(" + to_rule(transform()) + ")", false));+ seq.emplace_back("(" + to_rule(transform()) + ")", false);} else if (c == ')') {i++;if (start > 0 && sub_pattern[start - 1] != '(') {@@ -331,9 +331,9 @@ private:}square_brackets += ']';i++;- seq.push_back(std::make_pair(square_brackets, false));+ seq.emplace_back(square_brackets, false);} else if (c == '|') {- seq.push_back(std::make_pair("|", false));+ seq.emplace_back("|", false);i++;} else if (c == '*' || c == '+' || c == '?') {seq.back() = std::make_pair(to_rule(seq.back()) + c, false);@@ -417,7 +417,7 @@ private:}}if (!literal.empty()) {- seq.push_back(std::make_pair(literal, true));+ seq.emplace_back(literal, true);}}}diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex ee3758bc..4a450ba9 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -155,10 +155,10 @@ enum llm_arch {LLM_ARCH_GPTNEOX,LLM_ARCH_MPT,LLM_ARCH_STARCODER,- LLM_ARCH_PERSIMMON,LLM_ARCH_REFACT,LLM_ARCH_BERT,LLM_ARCH_NOMIC_BERT,+ LLM_ARCH_JINA_BERT_V2,LLM_ARCH_BLOOM,LLM_ARCH_STABLELM,LLM_ARCH_QWEN,@@ -182,39 +182,39 @@ enum llm_arch {};static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {- { LLM_ARCH_LLAMA, "llama" },- { LLM_ARCH_FALCON, "falcon" },- { LLM_ARCH_GROK, "grok" },- { LLM_ARCH_GPT2, "gpt2" },- { LLM_ARCH_GPTJ, "gptj" },- { LLM_ARCH_GPTNEOX, "gptneox" },- { LLM_ARCH_MPT, "mpt" },- { LLM_ARCH_BAICHUAN, "baichuan" },- { LLM_ARCH_STARCODER, "starcoder" },- { LLM_ARCH_PERSIMMON, "persimmon" },- { LLM_ARCH_REFACT, "refact" },- { LLM_ARCH_BERT, "bert" },- { LLM_ARCH_NOMIC_BERT, "nomic-bert" },- { LLM_ARCH_BLOOM, "bloom" },- { LLM_ARCH_STABLELM, "stablelm" },- { LLM_ARCH_QWEN, "qwen" },- { LLM_ARCH_QWEN2, "qwen2" },- { LLM_ARCH_QWEN2MOE, "qwen2moe" },- { LLM_ARCH_PHI2, "phi2" },- { LLM_ARCH_PHI3, "phi3" },- { LLM_ARCH_PLAMO, "plamo" },- { LLM_ARCH_CODESHELL, "codeshell" },- { LLM_ARCH_ORION, "orion" },- { LLM_ARCH_INTERNLM2, "internlm2" },- { LLM_ARCH_MINICPM, "minicpm" },- { LLM_ARCH_GEMMA, "gemma" },- { LLM_ARCH_STARCODER2, "starcoder2" },- { LLM_ARCH_MAMBA, "mamba" },- { LLM_ARCH_XVERSE, "xverse" },- { LLM_ARCH_COMMAND_R, "command-r" },- { LLM_ARCH_DBRX, "dbrx" },- { LLM_ARCH_OLMO, "olmo" },- { LLM_ARCH_UNKNOWN, "(unknown)" },+ { LLM_ARCH_LLAMA, "llama" },+ { LLM_ARCH_FALCON, "falcon" },+ { LLM_ARCH_GROK, "grok" },+ { LLM_ARCH_GPT2, "gpt2" },+ { LLM_ARCH_GPTJ, "gptj" },+ { LLM_ARCH_GPTNEOX, "gptneox" },+ { LLM_ARCH_MPT, "mpt" },+ { LLM_ARCH_BAICHUAN, "baichuan" },+ { LLM_ARCH_STARCODER, "starcoder" },+ { LLM_ARCH_REFACT, "refact" },+ { LLM_ARCH_BERT, "bert" },+ { LLM_ARCH_NOMIC_BERT, "nomic-bert" },+ { LLM_ARCH_JINA_BERT_V2, "jina-bert-v2" },+ { LLM_ARCH_BLOOM, "bloom" },+ { LLM_ARCH_STABLELM, "stablelm" },+ { LLM_ARCH_QWEN, "qwen" },+ { LLM_ARCH_QWEN2, "qwen2" },+ { LLM_ARCH_QWEN2MOE, "qwen2moe" },+ { LLM_ARCH_PHI2, "phi2" },+ { LLM_ARCH_PHI3, "phi3" },+ { LLM_ARCH_PLAMO, "plamo" },+ { LLM_ARCH_CODESHELL, "codeshell" },+ { LLM_ARCH_ORION, "orion" },+ { LLM_ARCH_INTERNLM2, "internlm2" },+ { LLM_ARCH_MINICPM, "minicpm" },+ { LLM_ARCH_GEMMA, "gemma" },+ { LLM_ARCH_STARCODER2, "starcoder2" },+ { LLM_ARCH_MAMBA, "mamba" },+ { LLM_ARCH_XVERSE, "xverse" },+ { LLM_ARCH_COMMAND_R, "command-r" },+ { LLM_ARCH_DBRX, "dbrx" },+ { LLM_ARCH_OLMO, "olmo" },+ { LLM_ARCH_UNKNOWN, "(unknown)" },};enum llm_kv {@@ -257,6 +257,7 @@ enum llm_kv {LLM_KV_ROPE_SCALE_LINEAR,LLM_KV_ROPE_SCALING_TYPE,LLM_KV_ROPE_SCALING_FACTOR,+ LLM_KV_ROPE_SCALING_ATTN_FACTOR,LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,LLM_KV_ROPE_SCALING_FINETUNED,@@ -334,6 +335,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" },{ LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" },{ LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" },+ { LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" },{ LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" },{ LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" },@@ -389,6 +391,8 @@ enum llm_tensor {LLM_TENSOR_OUTPUT,LLM_TENSOR_OUTPUT_NORM,LLM_TENSOR_ROPE_FREQS,+ LLM_TENSOR_ROPE_FACTORS_LONG,+ LLM_TENSOR_ROPE_FACTORS_SHORT,LLM_TENSOR_ATTN_Q,LLM_TENSOR_ATTN_K,LLM_TENSOR_ATTN_V,@@ -546,23 +550,6 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},- {- LLM_ARCH_PERSIMMON,- {- { LLM_TENSOR_TOKEN_EMBD, "token_embd"},- { LLM_TENSOR_OUTPUT_NORM, "output_norm"},- { LLM_TENSOR_OUTPUT, "output"},- { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm"},- { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv"},- { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output"},- { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm"},- { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm"},- { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm"},- { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down"},- { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up"},- { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd"},- },- },{LLM_ARCH_MPT,{@@ -645,6 +632,25 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_JINA_BERT_V2,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },+ { LLM_TENSOR_TOKEN_TYPES, "token_types" },+ { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_BLOOM,{@@ -754,18 +760,20 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{LLM_ARCH_PHI3,{- { LLM_TENSOR_TOKEN_EMBD, "token_embd" },- { LLM_TENSOR_OUTPUT_NORM, "output_norm" },- { LLM_TENSOR_OUTPUT, "output" },- { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },- { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },- { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },- { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },- { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },- { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },- { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },- { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },- { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FACTORS_LONG, "rope_factors_long" },+ { LLM_TENSOR_ROPE_FACTORS_SHORT, "rope_factors_short" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},{@@ -1417,7 +1425,7 @@ struct llama_mlock {}LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",- size, this->size, errmsg, suggest ? get_mlock_suggestion() : "");+ size, this->size, errmsg, suggest ? get_mlock_suggestion() : "");return false;}@@ -1472,86 +1480,6 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_bufferGGML_UNUSED(host_buffer);}-static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) {- ggml_backend_buffer_type_t buft = nullptr;-- if (!buft) buft = ggml_backend_metal_buffer_type();- if (!buft) buft = ggml_backend_cuda_buffer_type(gpu);-// #ifdef GGML_USE_METAL-// buft = ggml_backend_metal_buffer_type();-// #elif defined(GGML_USE_CUDA)-// buft = ggml_backend_cuda_buffer_type(gpu);-// #elif defined(GGML_USE_VULKAN)-// buft = ggml_backend_vk_buffer_type(gpu);-// #elif defined(GGML_USE_SYCL)-// buft = ggml_backend_sycl_buffer_type(gpu);-// #elif defined(GGML_USE_CLBLAST)-// buft = ggml_backend_opencl_buffer_type();-// #elif defined(GGML_USE_KOMPUTE)-// buft = ggml_backend_kompute_buffer_type(gpu);-// if (buft == nullptr) {-// LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu);-// }-// #endif-- if (buft == nullptr) {- buft = llama_default_buffer_type_cpu(true);- }- return buft;-- GGML_UNUSED(gpu);-}--static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gpu, const float * tensor_split) {- ggml_backend_buffer_type_t buft = nullptr;-- if (ggml_backend_cuda_get_device_count() > 1) {- buft = ggml_backend_cuda_split_buffer_type(tensor_split);- }-- if (buft == nullptr) {- buft = llama_default_buffer_type_offload(fallback_gpu);- }- return buft;-- GGML_UNUSED(tensor_split);-}--static size_t llama_get_device_count() {- size_t res = ggml_backend_cuda_get_device_count();- if (!res) res = 1;- return res;-// #if defined(GGML_USE_CUBLAS)-// return ggml_backend_cuda_get_device_count();-// #elif defined(GGML_USE_VULKAN)-// return ggml_backend_vk_get_device_count();-// #else-// return 1;-// #endif-}--static size_t llama_get_device_memory(int device) {- size_t total = 0;- size_t free = 0;- ggml_backend_cuda_get_device_memory(device, &total, &free);- if (!free) free = 1;- return free;-// #if defined(GGML_USE_CUBLAS)-// size_t total;-// size_t free;-// ggml_backend_cuda_get_device_memory(device, &total, &free);-// return free;-// #elif defined(GGML_USE_VULKAN)-// size_t total;-// size_t free;-// ggml_backend_vk_get_device_memory(device, &total, &free);-// return free;-// #else-// return 1;-// GGML_UNUSED(device);-// #endif-}-//// globals//@@ -1561,6 +1489,8 @@ struct llama_state {// [jart] don't load dso early in initialization// #ifdef GGML_USE_METAL// ggml_backend_metal_log_set_callback(log_callback, log_callback_user_data);+// #elif defined(GGML_USE_CUDA)+// ggml_backend_cuda_log_set_callback(log_callback, log_callback_user_data);// #endif}@@ -1634,6 +1564,7 @@ struct llama_hparams {float f_norm_eps;float f_norm_rms_eps;+ float rope_attn_factor = 1.0f;float rope_freq_base_train;float rope_freq_scale_train;uint32_t n_yarn_orig_ctx;@@ -1649,7 +1580,7 @@ struct llama_hparams {float f_logit_scale = 0.0f;bool causal_attn = true;- bool use_alibi = false; // currently, we need KQ_pos data for ALiBi-based models+ bool use_alibi = false;enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_NONE;enum llama_rope_type rope_type = LLAMA_ROPE_TYPE_NONE;@@ -1682,6 +1613,7 @@ struct llama_hparams {if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true;if (!is_float_close(this->f_norm_rms_eps, other.f_norm_rms_eps, EPSILON)) return true;+ if (!is_float_close(this->rope_attn_factor, other.rope_attn_factor, EPSILON)) return true;if (!is_float_close(this->rope_freq_base_train, other.rope_freq_base_train, EPSILON)) return true;if (!is_float_close(this->rope_freq_scale_train, other.rope_freq_scale_train, EPSILON)) return true;@@ -1987,12 +1919,18 @@ struct llama_model {struct ggml_tensor * output;struct ggml_tensor * output_b;+ // long rope factors+ struct ggml_tensor * rope_long = nullptr;+ struct ggml_tensor * rope_short = nullptr;+std::vector<llama_layer> layers;llama_split_mode split_mode;int main_gpu;int n_gpu_layers;+ std::vector<std::string> rpc_servers;+// gguf metadatastd::unordered_map<std::string, std::string> gguf_kv;@@ -2034,7 +1972,7 @@ struct llama_model {ggml_free(ctx);}for (ggml_backend_buffer_t buf : bufs) {-// #ifdef GGML_USE_CUBLAS+// #ifdef GGML_USE_CUDAif (ggml_backend_buffer_get_type(buf) == ggml_backend_cpu_buffer_type()) {ggml_backend_cuda_unregister_host_buffer(ggml_backend_buffer_get_base(buf));}@@ -2121,7 +2059,6 @@ struct llama_context {struct ggml_tensor * inp_pos; // I32 [n_batch]struct ggml_tensor * inp_out_ids; // I32 [n_outputs]struct ggml_tensor * inp_KQ_mask; // F32 [kv_size, n_batch]- struct ggml_tensor * inp_KQ_pos; // F32 [n_kv]struct ggml_tensor * inp_K_shift; // I32 [kv_size]struct ggml_tensor * inp_mean; // F32 [n_batch, n_batch]struct ggml_tensor * inp_cls; // I32 [n_batch]@@ -2131,11 +2068,119 @@ struct llama_context {// control vectorsstruct llama_control_vector cvec;+};-#ifdef GGML_USE_MPI- ggml_mpi_context * ctx_mpi = NULL;+static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_model & model, int gpu) {+ ggml_backend_buffer_type_t buft = nullptr;++ if (!buft) buft = ggml_backend_metal_buffer_type();+ if (!buft) buft = ggml_backend_cuda_buffer_type(gpu);++// #ifdef GGML_USE_RPC+// std::string endpoint = model.rpc_servers[gpu];+// buft = ggml_backend_rpc_buffer_type(endpoint.c_str());+// #elif defined(GGML_USE_METAL)+// buft = ggml_backend_metal_buffer_type();+// #elif defined(GGML_USE_CUDA)+// buft = ggml_backend_cuda_buffer_type(gpu);+// #elif defined(GGML_USE_VULKAN)+// buft = ggml_backend_vk_buffer_type(gpu);+// #elif defined(GGML_USE_SYCL)+// buft = ggml_backend_sycl_buffer_type(gpu);+// #elif defined(GGML_USE_CLBLAST)+// buft = ggml_backend_opencl_buffer_type();+// #elif defined(GGML_USE_KOMPUTE)+// buft = ggml_backend_kompute_buffer_type(gpu);+// if (buft == nullptr) {+// LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu);+// }+// #endif++ if (buft == nullptr) {+ buft = llama_default_buffer_type_cpu(true);+ }+ return buft;+ GGML_UNUSED(model);+ GGML_UNUSED(gpu);+}++static ggml_backend_buffer_type_t llama_default_buffer_type_split(const llama_model & model, int fallback_gpu, const float * tensor_split) {+ ggml_backend_buffer_type_t buft = nullptr;++// #ifdef GGML_USE_CUDA+ if (ggml_backend_cuda_get_device_count() > 1) {+ buft = ggml_backend_cuda_split_buffer_type(tensor_split);+ }+// #endif++#ifdef GGML_USE_SYCL+ if (ggml_backend_sycl_get_device_count() > 1) {+ buft = ggml_backend_sycl_split_buffer_type(tensor_split);+ }#endif-};++ if (buft == nullptr) {+ buft = llama_default_buffer_type_offload(model, fallback_gpu);+ }+ return buft;++ GGML_UNUSED(tensor_split);+}++static size_t llama_get_device_count(const llama_model & model) {+ size_t res = ggml_backend_cuda_get_device_count();+ if (!res) res = 1;+ return res;+// #if defined(GGML_USE_RPC)+// return model.rpc_servers.size();+// #elif defined(GGML_USE_CUDA)+// return ggml_backend_cuda_get_device_count();+// #elif defined(GGML_USE_SYCL)+// return ggml_backend_sycl_get_device_count();+// #elif defined(GGML_USE_VULKAN)+// return ggml_backend_vk_get_device_count();+// #else+// return 1;+// #endif+ GGML_UNUSED(model);+}++static size_t llama_get_device_memory(const llama_model & model, int device) {+ if (llamafile_has_cuda()) {+ size_t total;+ size_t free;+ ggml_backend_cuda_get_device_memory(device, &free, &total);+ return free;+ } else {+ return 1;+ }+// #if defined(GGML_USE_RPC)+// size_t total;+// size_t free;+// std::string endpoint = model.rpc_servers[device];+// ggml_backend_rpc_get_device_memory(endpoint.c_str(), &free, &total);+// return free;+// #elif defined(GGML_USE_CUDA)+// size_t total;+// size_t free;+// ggml_backend_cuda_get_device_memory(device, &free, &total);+// return free;+// #elif defined(GGML_USE_SYCL)+// size_t total;+// size_t free;+// ggml_backend_sycl_get_device_memory(device, &free, &total);+// return free;+// #elif defined(GGML_USE_VULKAN)+// size_t total;+// size_t free;+// ggml_backend_vk_get_device_memory(device, &free, &total);+// return free;+// #else+// return 1;+// #endif+ GGML_UNUSED(model);+ GGML_UNUSED(device);+}//// kv cache helpers@@ -2163,7 +2208,7 @@ static bool llama_kv_cache_init(cache.recurrent = model.arch == LLM_ARCH_MAMBA;cache.v_trans = !cparams.flash_attn;- // TODO: support mixed reccurent Transformer architectues+ // TODO: support mixed recurrent Transformer architectures// NOTE: (!a || b) is a logical implication (a -> b)GGML_ASSERT(!cache.recurrent || n_embd_k_gqa == hparams.n_embd_k_s());GGML_ASSERT(!cache.recurrent || n_embd_v_gqa == hparams.n_embd_v_s());@@ -2589,6 +2634,11 @@ static void llama_kv_cache_defrag(struct llama_kv_cache & cache) {cache.do_defrag = true;}+static uint32_t llama_kv_cache_get_padding(const struct llama_cparams & cparams) {+ // the FA kernels require padding to avoid extra runtime boundary checks+ return cparams.flash_attn ? 256u : 32u;+}+//// model loading and saving//@@ -3092,6 +3142,39 @@ struct llama_model_loader {return get_arr_n(llm_kv(kid), result, required);}+ template<typename T>+ bool get_arr(const std::string & key, std::vector<T> & result, const bool required = true) {+ const int kid = gguf_find_key(meta, key.c_str());++ if (kid < 0) {+ if (required) {+ throw std::runtime_error(format("key not found in model: %s", key.c_str()));+ }+ return false;+ }++ struct GGUFMeta::ArrayInfo arr_info =+ GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta, kid);++ if (arr_info.gt != GGUF_TYPE_FLOAT32 && arr_info.gt != GGUF_TYPE_INT32) {+ throw std::runtime_error(format("%s is not a float32 or int32 array", key.c_str()));+ }++ // GGML_ASSERT(gguf_type_size(arr_info.gt) == sizeof(T));+ GGML_ASSERT((arr_info.gt != GGUF_TYPE_FLOAT32 || std::is_same<T, float>::value));+ GGML_ASSERT((arr_info.gt != GGUF_TYPE_INT32 || std::is_same<T, int>::value));++ result.resize(arr_info.length);+ result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);++ return true;+ }++ template<typename T>+ bool get_arr(const enum llm_kv kid, T& result, const bool required = true) {+ return get_arr(llm_kv(kid), result, required);+ }+template<typename T>bool get_key(const std::string & key, T & result, const bool required = true) {auto it = kv_overrides.find(key);@@ -3585,6 +3668,12 @@ static void llm_load_hparams(// get hparams kvml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, hparams.n_vocab);++ // everything past this point is not vocab-related+ if (hparams.vocab_only) {+ return;+ }+ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);ml.get_key(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff);@@ -3629,6 +3718,8 @@ static void llm_load_hparams(}hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;+ ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);+// sanity check for n_rot (optional){hparams.n_rot = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head;@@ -3666,7 +3757,7 @@ static void llm_load_hparams(switch (hparams.n_layer) {case 22: model.type = e_model::MODEL_1B; break;case 26: model.type = e_model::MODEL_3B; break;- case 32: model.type = hparams.n_head == hparams.n_head_kv ? e_model::MODEL_7B : e_model::MODEL_8B; break; // LLaMa 8B v3 uses GQA+ case 32: model.type = hparams.n_vocab < 40000 ? e_model::MODEL_7B : e_model::MODEL_8B; break;case 40: model.type = e_model::MODEL_13B; break;case 48: model.type = e_model::MODEL_34B; break;case 60: model.type = e_model::MODEL_30B; break;@@ -3728,14 +3819,6 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;- case LLM_ARCH_PERSIMMON:- {- ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);- switch (hparams.n_layer) {- case 36: model.type = e_model::MODEL_8B; break;- default: model.type = e_model::MODEL_UNKNOWN;- }- } break;case LLM_ARCH_REFACT:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);@@ -3768,6 +3851,19 @@ static void llm_load_hparams(model.type = e_model::MODEL_335M; break; // bge-large}} break;+ case LLM_ARCH_JINA_BERT_V2:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);+ ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type);+ ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);+ hparams.f_max_alibi_bias = 8.0f;++ switch (hparams.n_layer) {+ case 4: model.type = e_model::MODEL_33M; break; // jina-embeddings-small+ case 12: model.type = e_model::MODEL_137M; break; // jina-embeddings-base+ }+ } break;case LLM_ARCH_NOMIC_BERT:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);@@ -4189,7 +4285,11 @@ static void llm_load_vocab(tokenizer_pre == "starcoder") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;} else if (- tokenizer_pre == "gpt-2") {+ tokenizer_pre == "gpt-2" ||+ tokenizer_pre == "jina-es" ||+ tokenizer_pre == "jina-de" ||+ tokenizer_pre == "jina-v2-es" ||+ tokenizer_pre == "jina-v2-de") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;} else if (tokenizer_pre == "refact") {@@ -4197,9 +4297,18 @@ static void llm_load_vocab(} else if (tokenizer_pre == "command-r") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;+ } else if (+ tokenizer_pre == "qwen2") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;+ } else if (+ tokenizer_pre == "stablelm2") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STABLELM2;} else if (tokenizer_pre == "olmo") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_OLMO;+ } else if (+ tokenizer_pre == "dbrx") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DBRX;} else {throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));}@@ -4315,8 +4424,8 @@ static void llm_load_vocab((t.first == "<|eot_id|>" ||t.first == "<|im_end|>" ||t.first == "<|end|>" ||- t.first == "<|end_of_text|>" ||- t.first == "<end_of_turn>"+ t.first == "<end_of_turn>" ||+ t.first == "<|endoftext|>")) {vocab.special_eot_id = t.second;@@ -4544,13 +4653,13 @@ static bool llm_load_tensors(if (split_mode == LLAMA_SPLIT_MODE_LAYER) {// calculate the split points- int device_count = llama_get_device_count();+ int device_count = llama_get_device_count(model);bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; });std::vector<float> splits(device_count);if (all_zero) {// default split, by free memoryfor (int i = 0; i < device_count; ++i) {- splits[i] = llama_get_device_memory(i);+ splits[i] = llama_get_device_memory(model, i);}} else {std::copy(tensor_split, tensor_split + device_count, splits.begin());@@ -4570,35 +4679,35 @@ static bool llm_load_tensors(int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1);for (int64_t i = i_gpu_start; i < n_layer; ++i) {int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits.begin();- model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu);+ model.buft_layer[i] = llama_default_buffer_type_offload(model, layer_gpu);}// assign the output layerif (n_gpu_layers > n_layer) {int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits.begin();- model.buft_output = llama_default_buffer_type_offload(layer_gpu);+ model.buft_output = llama_default_buffer_type_offload(model, layer_gpu);} else {model.buft_output = llama_default_buffer_type_cpu(true);}} else {ggml_backend_buffer_type_t split_buft;if (split_mode == LLAMA_SPLIT_MODE_ROW) {- split_buft = llama_default_buffer_type_split(main_gpu, tensor_split);+ split_buft = llama_default_buffer_type_split(model, main_gpu, tensor_split);} else {// LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_LAYER in backends where it is not supported- split_buft = llama_default_buffer_type_offload(main_gpu);+ split_buft = llama_default_buffer_type_offload(model, main_gpu);}// assign the repeating layersfor (int64_t i = i_gpu_start; i < n_layer; ++i) {model.buft_layer[i] = {split_buft,- llama_default_buffer_type_offload(main_gpu)+ llama_default_buffer_type_offload(model, main_gpu)};}// assign the output layerif (n_gpu_layers > n_layer) {model.buft_output = {split_buft,- llama_default_buffer_type_offload(main_gpu)+ llama_default_buffer_type_offload(model, main_gpu)};} else {model.buft_output = llama_default_buffer_type_cpu(true);@@ -4642,6 +4751,7 @@ static bool llm_load_tensors(// create tensors for the weights{const int64_t n_embd = hparams.n_embd;+ const int64_t n_embd_head = n_embd / hparams.n_head;const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();const int64_t n_embd_gqa = n_embd_v_gqa;@@ -4922,7 +5032,14 @@ static bool llm_load_tensors({model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false);+ if (!model.output) {+ // needs to be on GPU+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ ml.n_created--; // artificial tensor+ ml.size_data += ggml_nbytes(model.output);+ }+}for (int i = 0; i < n_layer; ++i) {@@ -4950,47 +5067,6 @@ static bool llm_load_tensors(layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;- case LLM_ARCH_PERSIMMON:- {- model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});-- {- model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});- model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});- model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});- }-- for (int i = 0; i < n_layer; ++i) {- ggml_context * ctx_layer = ctx_for_layer(i);- ggml_context * ctx_split = ctx_for_layer_split(i);-- auto & layer = model.layers[i];-- layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});-- layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});- layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});-- layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});-- layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});- layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});-- layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});- layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});-- layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});- layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});-- layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64});- layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64});-- layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64});- layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64});- }- } break;case LLM_ARCH_BERT:case LLM_ARCH_NOMIC_BERT:{@@ -5043,6 +5119,50 @@ static bool llm_load_tensors(layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd});}} break;+ case LLM_ARCH_JINA_BERT_V2:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // word_embeddings+ model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); //token_type_embeddings+ model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); // LayerNorm+ model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); //LayerNorm bias++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i]; // JinaBertLayer++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});++ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, false);+ layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, false);++ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});++ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, false);+ layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, false);++ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); //output_dens+ layer.bo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); //output_dens++ layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm+ layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});++ layer.layer_out_norm = ml.create_tensor(ctx_split, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});+ layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd});+ }+ } break;case LLM_ARCH_BLOOM:{model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});@@ -5343,6 +5463,9 @@ static bool llm_load_tensors({model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab });+ model.rope_long = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, false);+ model.rope_short = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, false);+// output{model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd });@@ -5353,12 +5476,12 @@ static bool llm_load_tensors(ggml_context* ctx_layer = ctx_for_layer(i);ggml_context* ctx_split = ctx_for_layer_split(i);- auto& layer = model.layers[i];+ auto & layer = model.layers[i];layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd });layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, false);- layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd });+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd });layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd });@@ -6014,10 +6137,7 @@ static struct ggml_tensor * llm_build_inp_embd(inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens);} else {-#ifdef GGML_USE_MPI- GGML_ASSERT(false && "not implemented");-#endif- lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens);+ lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens);inpL = lctx.inp_embd;ggml_set_input(lctx.inp_embd);}@@ -6119,7 +6239,7 @@ static struct ggml_tensor * llm_build_ffn(llm_ffn_gate_type type_gate,const llm_build_cb & cb,int il) {- struct ggml_tensor * tmp = ggml_mul_mat(ctx, up, cur);+ struct ggml_tensor * tmp = up ? ggml_mul_mat(ctx, up, cur) : cur;cb(tmp, "ffn_up", il);if (up_b) {@@ -6301,7 +6421,6 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * wo_b,struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,- struct ggml_tensor * kq_pos,int32_t n_tokens,int32_t n_kv,float kq_scale,@@ -6313,6 +6432,7 @@ static struct ggml_tensor * llm_build_kqv(const int64_t n_embd_head_k = hparams.n_embd_head_k;const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();const int64_t n_embd_head_v = hparams.n_embd_head_v;+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3);cb(q, "q", il);@@ -6331,26 +6451,22 @@ static struct ggml_tensor * llm_build_kqv(GGML_UNUSED(model);GGML_UNUSED(n_ctx);- // note: if this assert triggers, then some check has failed earlier- // the idea is to detect during context creation that ALiBi would be used and disable Flash Attention- GGML_ASSERT(kq_pos == nullptr && "ALiBi is not yet supported with Flash Attention");-// split cached v into n_head heads (not transposed)struct ggml_tensor * v =ggml_view_3d(ctx, kv.v_l[il],n_embd_head_v, n_kv, n_head_kv,- ggml_row_size(kv.v_l[il]->type, n_embd_k_gqa),- ggml_row_size(kv.v_l[il]->type, n_embd_head_k),+ ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa),+ ggml_row_size(kv.v_l[il]->type, n_embd_head_v),0);cb(v, "v", il);- cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale);+ cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias);if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) {ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);}- cur = ggml_reshape_2d(ctx, cur, n_embd_head_k*n_head, n_tokens);+ cur = ggml_reshape_2d(ctx, cur, n_embd_head_v*n_head, n_tokens);} else {struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);cb(kq, "kq", il);@@ -6375,28 +6491,8 @@ static struct ggml_tensor * llm_build_kqv(kq = ggml_scale(ctx, kq, 30);}-#if defined(GGML_USE_KOMPUTE)-#pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Kompute")-#pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024")-#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")- if (hparams.use_alibi) {- kq = ggml_scale(ctx, kq, kq_scale);- cb(kq, "kq_scaled", il);-- kq = ggml_alibi(ctx, kq, /*n_past*/ 0, n_head, hparams.f_max_alibi_bias);- cb(kq, "kq_scaled_alibi", il);-- kq = ggml_add(ctx, kq, kq_mask);- cb(kq, "kq_masked", il);-- kq = ggml_soft_max(ctx, kq);- cb(kq, "kq_soft_max", il);- } else-#endif- {- kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_pos, kq_scale, hparams.f_max_alibi_bias);- cb(kq, "kq_soft_max_ext", il);- }+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);GGML_ASSERT(kv.size == n_ctx);@@ -6415,7 +6511,7 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);cb(kqv_merged, "kqv_merged", il);- cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);+ cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_v*n_head, n_tokens);cb(cur, "kqv_merged_cont", il);}@@ -6446,7 +6542,6 @@ static struct ggml_tensor * llm_build_kv(struct ggml_tensor * v_cur,struct ggml_tensor * q_cur,struct ggml_tensor * kq_mask,- struct ggml_tensor * kq_pos,int32_t n_tokens,int32_t kv_head,int32_t n_kv,@@ -6465,7 +6560,7 @@ static struct ggml_tensor * llm_build_kv(struct ggml_tensor * cur;cur = llm_build_kqv(ctx, model, hparams, cparams, kv, graph, wo, wo_b,- q_cur, kq_mask, kq_pos, n_tokens, n_kv, kq_scale, cb, il);+ q_cur, kq_mask, n_tokens, n_kv, kq_scale, cb, il);cb(cur, "kqv_out", il);return cur;@@ -6572,18 +6667,17 @@ struct llm_build_context {ctx0 = ggml_init(params);- lctx.inp_tokens = nullptr;- lctx.inp_embd = nullptr;- lctx.inp_pos = nullptr;+ lctx.inp_tokens = nullptr;+ lctx.inp_embd = nullptr;+ lctx.inp_pos = nullptr;lctx.inp_out_ids = nullptr;lctx.inp_KQ_mask = nullptr;- lctx.inp_KQ_pos = nullptr;lctx.inp_K_shift = nullptr;- lctx.inp_mean = nullptr;- lctx.inp_cls = nullptr;- lctx.inp_s_copy = nullptr;- lctx.inp_s_mask = nullptr;- lctx.inp_s_seq = nullptr;+ lctx.inp_mean = nullptr;+ lctx.inp_cls = nullptr;+ lctx.inp_s_copy = nullptr;+ lctx.inp_s_mask = nullptr;+ lctx.inp_s_seq = nullptr;}void free() {@@ -6602,17 +6696,20 @@ struct llm_build_context {cb(lctx.inp_K_shift, "K_shift", -1);ggml_set_input(lctx.inp_K_shift);+ struct ggml_tensor * rope_factors = build_rope_factors();+for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * tmp =// we rotate only the first n_rot dimensions- ggml_rope_custom_inplace(ctx0,+ ggml_rope_ext_inplace(ctx0,ggml_view_3d(ctx0, kv_self.k_l[il],n_embd_head_k, n_head_kv, n_ctx,ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),0),- lctx.inp_K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,+ lctx.inp_K_shift, rope_factors, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);+cb(tmp, "K_shifted", il);ggml_build_forward_expand(gf, tmp);}@@ -6715,6 +6812,17 @@ struct llm_build_context {return lctx.inp_pos;}+ struct ggml_tensor * build_rope_factors() {+ // choose long/short freq factors based on the context size+ const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max;++ if (n_ctx_pre_seq > hparams.n_yarn_orig_ctx) {+ return model.rope_long;+ }++ return model.rope_short;+ }+struct ggml_tensor * build_inp_out_ids() {lctx.inp_out_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs);cb(lctx.inp_out_ids, "inp_out_ids", -1);@@ -6733,19 +6841,6 @@ struct llm_build_context {return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_mask, GGML_TYPE_F16) : lctx.inp_KQ_mask;}- struct ggml_tensor * build_inp_KQ_pos(bool causal = true) {- if (causal) {- lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_kv);- } else {- // TODO: this will be needed for ALiBi-based BERT models- // https://github.com/ggerganov/llama.cpp/pull/6826- lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_tokens);- }- cb(lctx.inp_KQ_pos, "KQ_pos", -1);- ggml_set_input(lctx.inp_KQ_pos);- return flash_attn ? ggml_cast(ctx0, lctx.inp_KQ_pos, GGML_TYPE_F16) : lctx.inp_KQ_pos;- }-struct ggml_tensor * build_inp_mean() {lctx.inp_mean = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens);cb(lctx.inp_mean, "inp_mean", -1);@@ -6835,15 +6930,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -6851,7 +6946,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -6944,9 +7039,6 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- // positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = build_inp_KQ_pos();-for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -6968,13 +7060,13 @@ struct llm_build_context {switch (model.type) {case MODEL_7B:- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -6991,7 +7083,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7061,9 +7153,6 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- // positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = build_inp_KQ_pos();-for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -7083,22 +7172,22 @@ struct llm_build_context {struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7204,21 +7293,21 @@ struct llm_build_context {Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);// using mode = 2 for neox mode- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7327,15 +7416,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -7343,7 +7432,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -7479,15 +7568,15 @@ struct llm_build_context {cb(Kcur, "Kcur", il);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -7495,7 +7584,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7527,128 +7616,28 @@ struct llm_build_context {cb(cur, "ffn_moe_out", il);cur = ggml_add(ctx0, cur, ffn_inp);- cb(cur, "ffn_out", il);-- ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);- if (layer_dir != nullptr) {- cur = ggml_add(ctx0, cur, layer_dir);- }- cb(cur, "l_out", il);-- // input for next layer- inpL = cur;- }-- cur = inpL;-- cur = llm_build_norm(ctx0, cur, hparams,- model.output_norm, NULL,- LLM_NORM, cb, -1);- cb(cur, "result_norm", -1);-- // lm_head- cur = ggml_mul_mat(ctx0, model.output, cur);-- cb(cur, "result_output", -1);-- ggml_build_forward_expand(gf, cur);-- return gf;- }-- struct ggml_cgraph * build_starcoder() {- struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);-- const int64_t n_embd_head = hparams.n_embd_head_v;- const int64_t n_embd_gqa = hparams.n_embd_v_gqa();- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);-- struct ggml_tensor * cur;- struct ggml_tensor * inpL;-- inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);-- // inp_pos - contains the positions- struct ggml_tensor * inp_pos = build_inp_pos();-- // KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = build_inp_KQ_mask();-- struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);- cb(pos, "pos_embd", -1);-- inpL = ggml_add(ctx0, inpL, pos);- cb(inpL, "inpL", -1);-- for (int il = 0; il < n_layer; ++il) {- cur = llm_build_norm(ctx0, inpL, hparams,- model.layers[il].attn_norm,- model.layers[il].attn_norm_b,- LLM_NORM, cb, il);- cb(cur, "attn_norm", il);-- // self-attention- {- cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);- cb(cur, "wqkv", il);-- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);- cb(cur, "bqkv", il);-- struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));- struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));- struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));-- cb(Qcur, "Qcur", il);- cb(Kcur, "Kcur", il);- cb(Vcur, "Vcur", il);-- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);-- cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,- model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);- }-- if (il == n_layer - 1) {- // skip computing output for unused tokens- struct ggml_tensor * inp_out_ids = build_inp_out_ids();- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);- }-- // add the input- struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);- cb(ffn_inp, "ffn_inp", il);-- // FF- {- cur = llm_build_norm(ctx0, ffn_inp, hparams,- model.layers[il].ffn_norm,- model.layers[il].ffn_norm_b,- LLM_NORM, cb, il);- cb(cur, "ffn_norm", il);+ cb(cur, "ffn_out", il);- cur = llm_build_ffn(ctx0, cur,- model.layers[il].ffn_up, model.layers[il].ffn_up_b,- NULL, NULL,- model.layers[il].ffn_down, model.layers[il].ffn_down_b,- NULL,- LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);- cb(cur, "ffn_out", il);+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);+ if (layer_dir != nullptr) {+ cur = ggml_add(ctx0, cur, layer_dir);}+ cb(cur, "l_out", il);- inpL = ggml_add(ctx0, cur, ffn_inp);- cb(inpL, "l_out", il);+ // input for next layer+ inpL = cur;}- cur = llm_build_norm(ctx0, inpL, hparams,- model.output_norm,- model.output_norm_b,- LLM_NORM, cb, -1);+ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM, cb, -1);cb(cur, "result_norm", -1);+ // lm_headcur = ggml_mul_mat(ctx0, model.output, cur);+cb(cur, "result_output", -1);ggml_build_forward_expand(gf, cur);@@ -7656,12 +7645,12 @@ struct llm_build_context {return gf;}- struct ggml_cgraph * build_persimmon() {+ struct ggml_cgraph * build_starcoder() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);const int64_t n_embd_head = hparams.n_embd_head_v;- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);- GGML_ASSERT(n_embd_head/2 == hparams.n_rot);+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -7674,16 +7663,20 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- for (int il = 0; il < n_layer; ++il) {- struct ggml_tensor * residual = inpL;+ struct ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);+ cb(pos, "pos_embd", -1);++ inpL = ggml_add(ctx0, inpL, pos);+ cb(inpL, "inpL", -1);+ for (int il = 0; il < n_layer; ++il) {cur = llm_build_norm(ctx0, inpL, hparams,model.layers[il].attn_norm,model.layers[il].attn_norm_b,LLM_NORM, cb, il);cb(cur, "attn_norm", il);- // self attention+ // self-attention{cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);cb(cur, "wqkv", il);@@ -7691,140 +7684,33 @@ struct llm_build_context {cur = ggml_add(ctx0, cur, model.layers[il].bqkv);cb(cur, "bqkv", il);- // split qkv- GGML_ASSERT(n_head_kv == n_head);-- struct ggml_tensor * tmpqkv = ggml_reshape_4d(ctx0, cur, n_embd_head, 3, n_head, n_tokens);- cb(tmpqkv, "tmpqkv", il);-- struct ggml_tensor * tmpqkv_perm = ggml_cont(ctx0, ggml_permute(ctx0, tmpqkv, 0, 3, 1, 2));- cb(tmpqkv_perm, "tmpqkv", il);-- struct ggml_tensor * tmpq = ggml_view_3d(- ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens,- ggml_element_size(tmpqkv_perm) * n_embd_head,- ggml_element_size(tmpqkv_perm) * n_embd_head * n_head,- 0- );- cb(tmpq, "tmpq", il);-- struct ggml_tensor * tmpk = ggml_view_3d(- ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens,- ggml_element_size(tmpqkv_perm) * n_embd_head,- ggml_element_size(tmpqkv_perm) * n_embd_head * n_head,- ggml_element_size(tmpqkv_perm) * n_embd_head * n_head * n_tokens- );- cb(tmpk, "tmpk", il);-- // Q/K Layernorm- tmpq = llm_build_norm(ctx0, tmpq, hparams,- model.layers[il].attn_q_norm,- model.layers[il].attn_q_norm_b,- LLM_NORM, cb, il);- cb(tmpq, "tmpq", il);-- tmpk = llm_build_norm(ctx0, tmpk, hparams,- model.layers[il].attn_k_norm,- model.layers[il].attn_k_norm_b,- LLM_NORM, cb, il);- cb(tmpk, "tmpk", il);-- // RoPE the first n_rot of q/k, pass the other half, and concat.- struct ggml_tensor * qrot = ggml_view_3d(- ctx0, tmpq, n_rot, n_head, n_tokens,- ggml_element_size(tmpq) * n_embd_head,- ggml_element_size(tmpq) * n_embd_head * n_head,- 0- );- cb(qrot, "qrot", il);-- struct ggml_tensor * krot = ggml_view_3d(- ctx0, tmpk, n_rot, n_head, n_tokens,- ggml_element_size(tmpk) * n_embd_head,- ggml_element_size(tmpk) * n_embd_head * n_head,- 0- );- cb(krot, "krot", il);-- // get the second half of tmpq, e.g tmpq[n_rot:, :, :]- struct ggml_tensor * qpass = ggml_view_3d(- ctx0, tmpq, n_rot, n_head, n_tokens,- ggml_element_size(tmpq) * n_embd_head,- ggml_element_size(tmpq) * n_embd_head * n_head,- ggml_element_size(tmpq) * n_rot- );- cb(qpass, "qpass", il);-- struct ggml_tensor * kpass = ggml_view_3d(- ctx0, tmpk, n_rot, n_head, n_tokens,- ggml_element_size(tmpk) * n_embd_head,- ggml_element_size(tmpk) * n_embd_head * n_head,- ggml_element_size(tmpk) * n_rot- );- cb(kpass, "kpass", il);-- struct ggml_tensor * qrotated = ggml_rope_custom(- ctx0, qrot, inp_pos, n_rot, rope_type, 0, n_orig_ctx,- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow- );- cb(qrotated, "qrotated", il);-- struct ggml_tensor * krotated = ggml_rope_custom(- ctx0, krot, inp_pos, n_rot, rope_type, 0, n_orig_ctx,- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow- );- cb(krotated, "krotated", il);-- // ggml currently only supports concatenation on dim=2- // so we need to permute qrot, qpass, concat, then permute back.- qrotated = ggml_cont(ctx0, ggml_permute(ctx0, qrotated, 2, 1, 0, 3));- cb(qrotated, "qrotated", il);-- krotated = ggml_cont(ctx0, ggml_permute(ctx0, krotated, 2, 1, 0, 3));- cb(krotated, "krotated", il);-- qpass = ggml_cont(ctx0, ggml_permute(ctx0, qpass, 2, 1, 0, 3));- cb(qpass, "qpass", il);-- kpass = ggml_cont(ctx0, ggml_permute(ctx0, kpass, 2, 1, 0, 3));- cb(kpass, "kpass", il);+ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));- struct ggml_tensor * Qcur = ggml_concat(ctx0, qrotated, qpass);cb(Qcur, "Qcur", il);-- struct ggml_tensor * Kcur = ggml_concat(ctx0, krotated, kpass);- cb(Kcur, "Kcur", il);-- struct ggml_tensor * Q = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 2, 1, 0, 3));- cb(Q, "Q", il);-- Kcur = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 2, 1, 0, 3));cb(Kcur, "Kcur", il);-- struct ggml_tensor * Vcur = ggml_view_3d(- ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens,- ggml_element_size(tmpqkv_perm) * n_embd_head,- ggml_element_size(tmpqkv_perm) * n_embd_head * n_head,- ggml_element_size(tmpqkv_perm) * n_embd_head * n_head * n_tokens * 2- );cb(Vcur, "Vcur", il);+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Q, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {// skip computing output for unused tokensstruct ggml_tensor * inp_out_ids = build_inp_out_ids();- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- residual = ggml_get_rows(ctx0, residual, inp_out_ids);+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);}- struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur);+ // add the input+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);cb(ffn_inp, "ffn_inp", il);- // feed-forward network+ // FF{cur = llm_build_norm(ctx0, ffn_inp, hparams,model.layers[il].ffn_norm,@@ -7837,19 +7723,15 @@ struct llm_build_context {NULL, NULL,model.layers[il].ffn_down, model.layers[il].ffn_down_b,NULL,- LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il);+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}- cur = ggml_add(ctx0, cur, ffn_inp);- cb(cur, "l_out", il);-- inpL = cur;+ inpL = ggml_add(ctx0, cur, ffn_inp);+ cb(inpL, "l_out", il);}- cur = inpL;-- cur = llm_build_norm(ctx0, cur, hparams,+ cur = llm_build_norm(ctx0, inpL, hparams,model.output_norm,model.output_norm_b,LLM_NORM, cb, -1);@@ -7877,9 +7759,6 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- // positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = build_inp_KQ_pos();-for (int il = 0; il < n_layer; ++il) {struct ggml_tensor * inpSA = inpL;@@ -7907,7 +7786,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -7969,8 +7848,11 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;+ struct ggml_tensor * inp_pos = nullptr;- struct ggml_tensor * inp_pos = build_inp_pos();+ if (model.arch != LLM_ARCH_JINA_BERT_V2) {+ inp_pos = build_inp_pos();+ }struct ggml_tensor * inp_mean = build_inp_mean();struct ggml_tensor * inp_cls = build_inp_cls();@@ -8001,13 +7883,26 @@ struct llm_build_context {struct ggml_tensor * Vcur;// self-attention- if (model.arch == LLM_ARCH_BERT) {+ if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT_V2) {Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq);cb(Qcur, "Qcur", il);+ if (model.layers[il].attn_q_norm) {+ Qcur = llm_build_norm(ctx0, Qcur, hparams,+ model.layers[il].attn_q_norm,+ model.layers[il].attn_q_norm_b,+ LLM_NORM, cb, il);+ }+Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk);cb(Kcur, "Kcur", il);+ if (model.layers[il].attn_k_norm) {+ Kcur = llm_build_norm(ctx0, Kcur, hparams,+ model.layers[il].attn_k_norm,+ model.layers[il].attn_k_norm_b,+ LLM_NORM, cb, il);+ }Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv);cb(Vcur, "Vcur", il);@@ -8026,15 +7921,15 @@ struct llm_build_context {cb(Kcur, "Kcur", il);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -8047,7 +7942,7 @@ struct llm_build_context {struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);cb(kq, "kq", il);- kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, nullptr, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias);+ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias);cb(kq, "kq_soft_max_ext", il);struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));@@ -8098,6 +7993,13 @@ struct llm_build_context {model.layers[il].ffn_down, model.layers[il].ffn_down_b,NULL,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);+ } else if (model.arch == LLM_ARCH_JINA_BERT_V2) {+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,+ LLM_FFN_GELU, LLM_FFN_PAR, cb, il);} else {cur = llm_build_ffn(ctx0, cur,model.layers[il].ffn_up, NULL,@@ -8164,9 +8066,6 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- // positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = build_inp_KQ_pos();-inpL = llm_build_norm(ctx0, inpL, hparams,model.tok_norm,model.tok_norm_b,@@ -8200,7 +8099,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8265,9 +8164,6 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();- // positions of the tokens in the KV cache- struct ggml_tensor * KQ_pos = build_inp_KQ_pos();-if (model.pos_embd) {// inp_pos - contains the positionsstruct ggml_tensor * inp_pos = build_inp_pos();@@ -8331,13 +8227,13 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);} else {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, KQ_pos, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}}@@ -8465,15 +8361,15 @@ struct llm_build_context {}- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -8481,7 +8377,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8585,21 +8481,21 @@ struct llm_build_context {Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);// using mode = 2 for neox mode- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8696,15 +8592,15 @@ struct llm_build_context {Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -8712,7 +8608,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8810,15 +8706,15 @@ struct llm_build_context {Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -8826,7 +8722,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -8962,8 +8858,8 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);@@ -8973,15 +8869,15 @@ struct llm_build_context {Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -9045,6 +8941,9 @@ struct llm_build_context {// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = build_inp_KQ_mask();+ // rope freq factors for 128k context+ struct ggml_tensor * rope_factors = build_rope_factors();+for (int il = 0; il < n_layer; ++il) {auto residual = inpL;@@ -9081,8 +8980,8 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Qcur = ggml_rope_ext(+ ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);@@ -9090,15 +8989,15 @@ struct llm_build_context {Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head)));cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx,+ Kcur = ggml_rope_ext(+ ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -9197,21 +9096,21 @@ struct llm_build_context {struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr,n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr,n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}struct ggml_tensor * sa_out = cur;@@ -9314,7 +9213,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9405,15 +9304,15 @@ struct llm_build_context {cb(tmpk, "tmpk", il);cb(Vcur, "Vcur", il);- struct ggml_tensor * Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos,+ struct ggml_tensor * Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- struct ggml_tensor * Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos,+ struct ggml_tensor * Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -9421,7 +9320,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9521,15 +9420,15 @@ struct llm_build_context {// cb(Vcur, "Vcur", il);// }- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -9537,7 +9436,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9638,15 +9537,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -9654,7 +9553,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9768,15 +9667,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -9784,7 +9683,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -9888,8 +9787,8 @@ struct llm_build_context {struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);cb(Vcur, "Vcur", il);- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr,n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);@@ -9897,15 +9796,15 @@ struct llm_build_context {Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k)));cb(Qcur, "Qcur_scaled", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr,n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, NULL,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f, cb, il);}if (il == n_layer - 1) {@@ -10008,15 +9907,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -10024,7 +9923,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10298,15 +10197,15 @@ struct llm_build_context {cb(Kcur, "Kcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -10314,7 +10213,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10429,15 +10328,15 @@ struct llm_build_context {cb(Vcur, "Vcur", il);}- Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);- Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -10445,7 +10344,7 @@ struct llm_build_context {cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,model.layers[il].wo, nullptr,- Kcur, Vcur, Qcur, KQ_mask, nullptr, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);}if (il == n_layer - 1) {@@ -10617,15 +10516,12 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_starcoder();} break;- case LLM_ARCH_PERSIMMON:- {- result = llm.build_persimmon();- } break;case LLM_ARCH_REFACT:{result = llm.build_refact();} break;case LLM_ARCH_BERT:+ case LLM_ARCH_JINA_BERT_V2:case LLM_ARCH_NOMIC_BERT:{result = llm.build_bert();@@ -10833,11 +10729,21 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {f = -INFINITY;} else {- f = 0.0f;+ if (hparams.use_alibi) {+ f = -fabs(lctx.kv_self.cells[i].pos - pos);+ } else {+ f = 0.0f;+ }}data[h*(n_kv*n_tokens) + j*n_kv + i] = f;}}++ for (int i = n_tokens; i < GGML_PAD(n_tokens, GGML_KQ_MASK_PAD); ++i) {+ for (int j = 0; j < n_kv; ++j) {+ data[h*(n_kv*n_tokens) + i*n_kv + j] = -INFINITY;+ }+ }}} else {// when using kv cache, the mask needs to match the kv cache size@@ -10856,7 +10762,11 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {float f = -INFINITY;for (int s = 0; s < batch.n_seq_id[i]; ++s) {if (batch.seq_id[i][s] == seq_id) {- f = 0.0f;+ if (hparams.use_alibi) {+ f = -fabs(batch.pos[i] - batch.pos[j]);+ } else {+ f = 0.0f;+ }break;}}@@ -10872,21 +10782,6 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {}}- // ALiBi requires the KQ_pos tensor to provide the sequence position of each token in the batch- // this allows to process multiple sequences in parallel with ALiBi-based models- if (hparams.use_alibi) {- const int64_t n_kv = kv_self.n;-- GGML_ASSERT(lctx.inp_KQ_pos);- GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer));-- float * data = (float *) lctx.inp_KQ_pos->data;-- for (int i = 0; i < n_kv; ++i) {- data[i] = float(lctx.kv_self.cells[i].pos);- }- }-if (cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN) {const int64_t n_tokens = batch.n_tokens;@@ -11060,11 +10955,6 @@ static void llama_graph_compute(llama_context & lctx,ggml_cgraph * gf,int n_threads) {-#ifdef GGML_USE_MPI- const int64_t n_layer = lctx.model.hparams.n_layer;- ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);-#endif-// #ifdef GGML_USE_METALif (ggml_backend_is_metal(lctx.backend_metal)) {ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads);@@ -11079,10 +10969,6 @@ static void llama_graph_compute(ggml_backend_sched_graph_compute_async(lctx.sched, gf);// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));--#ifdef GGML_USE_MPI- ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer);-#endif}// decode a batch of tokens by evaluating the transformer@@ -11120,12 +11006,6 @@ static int llama_decode_internal(}lctx.n_queued_tokens += n_tokens_all;-#ifdef GGML_USE_MPI- // TODO: needs fix after #3228- GGML_ASSERT(false && "not implemented");- //ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads);-#endif-auto & kv_self = lctx.kv_self;const int64_t n_embd = hparams.n_embd;@@ -11256,7 +11136,8 @@ static int llama_decode_internal(// a heuristic, to avoid attending the full cache if it is not yet utilized// after enough generations, the benefit from this heuristic disappears// if we start defragmenting the cache, the benefit from this will be more important- kv_self.n = std::min(kv_self.size, std::max(256u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 256)));+ const uint32_t pad = llama_kv_cache_get_padding(cparams);+ kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad)));//kv_self.n = llama_kv_cache_cell_max(kv_self);}}@@ -12009,12 +11890,14 @@ struct llm_tokenizer_bpe {void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {int final_prev_index = -1;+ bool ignore_merges = false;std::vector<std::string> word_collection;switch (vocab.type) {case LLAMA_VOCAB_TYPE_BPE:switch (vocab.type_pre) {case LLAMA_VOCAB_PRE_TYPE_LLAMA3:+ ignore_merges = true;word_collection = unicode_regex_split(text, {// original regex from tokenizer.json//"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",@@ -12023,6 +11906,12 @@ struct llm_tokenizer_bpe {"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",});break;+ case LLAMA_VOCAB_PRE_TYPE_DBRX:+ word_collection = unicode_regex_split(text, {+ // same as llama3+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ });+ break;case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:word_collection = unicode_regex_split(text, {"[\r\n]",@@ -12074,6 +11963,14 @@ struct llm_tokenizer_bpe {"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",});break;+ case LLAMA_VOCAB_PRE_TYPE_STABLELM2:+ case LLAMA_VOCAB_PRE_TYPE_QWEN2:+ word_collection = unicode_regex_split(text, {+ // original regex from tokenizer.json+ // "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",+ });+ break;default:// default regex for BPE tokenization pre-processingword_collection = unicode_regex_split(text, {@@ -12099,6 +11996,11 @@ struct llm_tokenizer_bpe {int index = 0;size_t offset = 0;+ if (ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {+ symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});+ offset = word.size();+ }+while (offset < word.size()) {llm_symbol sym;size_t char_len = std::min(word.size() - offset, (size_t) ::utf8_len(word[offset]));@@ -12284,16 +12186,16 @@ struct llm_tokenizer_wpm {// to lowercase, pad chinese characters, pad punctuationstd::string new_str = "";for (uint32_t code : cpts_nfd) {- int type = unicode_cpt_type(code);- if (type == CODEPOINT_TYPE_ACCENT_MARK || type == CODEPOINT_TYPE_CONTROL) {+ const codepoint_flags flags = unicode_cpt_flags(code);+ if (flags.is_accent_mark || flags.is_control) {continue;}code = unicode_tolower(code);- if (type == CODEPOINT_TYPE_WHITESPACE) {+ if (flags.is_separator || flags.is_whitespace) { //####FIXME: is_separator ?code = ' ';}std::string s = unicode_cpt_to_utf8(code);- if (type == CODEPOINT_TYPE_PUNCTUATION || is_ascii_punct(code) || is_chinese_char(code)) {+ if (flags.is_punctuation || is_ascii_punct(code) || is_chinese_char(code)) {new_str += " ";new_str += s;new_str += " ";@@ -12496,9 +12398,14 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &// tokenizer.encode('', add_special_tokens=True) returns [1]// tokenizer.encode('', add_special_tokens=False) returns []+ static const bool rtrim = true; //TODO: as param+ bool is_prev_special = false;+ bool special_token_rtrim = false;+if (add_special && vocab.special_add_bos != 0) {GGML_ASSERT(vocab.special_bos_id != -1);output.push_back(vocab.special_bos_id);+ is_prev_special = true;}for (const auto & fragment : fragment_buffer) {@@ -12510,9 +12417,21 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &// and passing 'add space prefix' as bool argument//auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);- if (&fragment == &fragment_buffer.front()) {- if (vocab.add_space_prefix) {- raw_text = " " + raw_text; // prefix with space if the first token is not special++ if (special_token_rtrim) {+ size_t num_whitespaces = 0;+ while (isspace(raw_text[num_whitespaces])) {+ num_whitespaces++;+ }+ if (num_whitespaces == raw_text.size()) {+ continue; // skip if all whitespaces+ }+ raw_text = raw_text.substr(num_whitespaces);+ }++ if (vocab.add_space_prefix) {+ if (!output.size() || is_prev_special) { // prefix with space if first token+ raw_text = " " + raw_text;}}@@ -12524,9 +12443,22 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &tokenizer.tokenize(raw_text, output);} else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)output.push_back(fragment.token);+ is_prev_special = true;+ // phi-3 special tokens without rtrim, works fine for llama-spm too+ special_token_rtrim = rtrim+ && fragment.token != vocab.special_bos_id+ && fragment.token != vocab.special_unk_id+ && fragment.token != vocab.special_eos_id;}}+ if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {+ LLAMA_LOG_WARN(+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "+ "Are you sure this is what you want?\n", __FUNCTION__);+ }+if (add_special && vocab.special_add_eos == 1) {GGML_ASSERT(vocab.special_eos_id != -1);output.push_back(vocab.special_eos_id);@@ -12553,7 +12485,17 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &}}- GGML_ASSERT(vocab.special_add_eos != 1);+ if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) {+ LLAMA_LOG_WARN(+ "%s: Added a BOS token to the prompt as specified by the model but the prompt "+ "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "+ "Are you sure this is what you want?\n", __FUNCTION__);+ }++ if (add_special && vocab.special_add_eos == 1) {+ GGML_ASSERT(vocab.special_add_eos != -1);+ output.push_back(vocab.special_eos_id);+ }} break;case LLAMA_VOCAB_TYPE_WPM:{@@ -12907,6 +12849,58 @@ static std::vector<llama_grammar_candidate> llama_grammar_reject_candidates(return rejects;}+static bool llama_grammar_detect_left_recursion(+ const std::vector<std::vector<llama_grammar_element>> & rules,+ size_t rule_index,+ std::vector<bool> * rules_visited,+ std::vector<bool> * rules_in_progress,+ std::vector<bool> * rules_may_be_empty) {+ if ((*rules_in_progress)[rule_index]) {+ return true;+ }++ (*rules_in_progress)[rule_index] = true;++ const std::vector<llama_grammar_element> & rule = rules[rule_index];++ // First check if the rule might produce the empty string. This could be done combined with the second+ // step but it's more readable as two steps.+ bool at_rule_start = true;+ for (size_t i = 0; i < rule.size(); i++) {+ if (llama_grammar_is_end_of_sequence(&rule[i])) {+ if (at_rule_start) {+ (*rules_may_be_empty)[rule_index] = true;+ break;+ }+ at_rule_start = true;+ } else {+ at_rule_start = false;+ }+ }++ // Second, recurse into leftmost nonterminals (or next-leftmost as long as the previous nonterminal may+ // be empty)+ bool recurse_into_nonterminal = true;+ for (size_t i = 0; i < rule.size(); i++) {+ if (rule[i].type == LLAMA_GRETYPE_RULE_REF && recurse_into_nonterminal) {+ if (llama_grammar_detect_left_recursion(rules, (size_t)rule[i].value, rules_visited, rules_in_progress, rules_may_be_empty)) {+ return true;+ }+ if (!((*rules_may_be_empty)[(size_t)rule[i].value])) {+ recurse_into_nonterminal = false;+ }+ } else if (llama_grammar_is_end_of_sequence(&rule[i])) {+ recurse_into_nonterminal = true;+ } else {+ recurse_into_nonterminal = false;+ }+ }++ (*rules_in_progress)[rule_index] = false;+ (*rules_visited)[rule_index] = true;+ return false;+}+//// grammar - external//@@ -12926,6 +12920,19 @@ struct llama_grammar * llama_grammar_init(vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});}+ // Check for left recursion+ std::vector<bool> rules_visited(n_rules);+ std::vector<bool> rules_in_progress(n_rules);+ std::vector<bool> rules_may_be_empty(n_rules);+ for (size_t i = 0; i < n_rules; i++) {+ if (rules_visited[i]) {+ continue;+ }+ if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) {+ throw std::runtime_error(format("unsupported grammar, left recursion detected for nonterminal at index %zu", i));+ }+ }+// loop over alternates of start rule to build initial stacksstd::vector<std::vector<const llama_grammar_element *>> stacks;pos = vec_rules[start_rule_index].data();@@ -12948,6 +12955,9 @@ struct llama_grammar * llama_grammar_init(}} while (true);+ // Important: vec_rules has to be moved here, not copied, because stacks contains+ // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar+ // then the pointers would be invalidated when the local vec_rules goes out of scope.return new llama_grammar{ std::move(vec_rules), std::move(stacks), {} };}@@ -13542,9 +13552,7 @@ llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_// Sample the next word X using top-k samplingllama_sample_top_k(nullptr, candidates, int(k), 1);- if (ctx) {- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;- }+ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;llama_token X = llama_sample_token(ctx, candidates);t_start_sample_us = ggml_time_us();@@ -13558,9 +13566,7 @@ llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_// Update mu using the learning rate and error*mu = *mu - eta * e;- if (ctx) {- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;- }+ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;return X;}@@ -13976,13 +13982,16 @@ static void llama_tensor_dequantize_internal(if (qtype.to_float == NULL) {throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));}- } else if (tensor->type != GGML_TYPE_F16) {+ } else if (tensor->type != GGML_TYPE_F16 &&+ tensor->type != GGML_TYPE_BF16) {throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));}if (nthread < 2) {if (tensor->type == GGML_TYPE_F16) {ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);+ } else if (tensor->type == GGML_TYPE_BF16) {+ ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);} else if (ggml_is_quantized(tensor->type)) {qtype.to_float(tensor->data, f32_output, nelements);} else {@@ -13991,7 +14000,14 @@ static void llama_tensor_dequantize_internal(return;}- size_t block_size = tensor->type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor->type);+ size_t block_size;+ if (tensor->type == GGML_TYPE_F16 ||+ tensor->type == GGML_TYPE_BF16) {+ block_size = 1;+ } else {+ block_size = (size_t)ggml_blck_size(tensor->type);+ }+size_t block_size_bytes = ggml_type_size(tensor->type);GGML_ASSERT(nelements % block_size == 0);@@ -14010,6 +14026,8 @@ static void llama_tensor_dequantize_internal(auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {if (typ == GGML_TYPE_F16) {ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);+ } else if (typ == GGML_TYPE_BF16) {+ ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);} else {qtype.to_float(inbuf, outbuf, nels);}@@ -14369,8 +14387,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const scase LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break;case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break;case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break;- case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break;case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break;+ case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break;case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break;// K-quants@@ -15035,6 +15053,7 @@ struct llama_model_params llama_model_default_params() {/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,+ /*.rpc_servers =*/ nullptr,/*.progress_callback =*/ nullptr,/*.progress_callback_user_data =*/ nullptr,/*.kv_overrides =*/ nullptr,@@ -15105,10 +15124,14 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {}size_t llama_max_devices(void) {- return GGML_CUDA_MAX_DEVICES;+ if (llamafile_has_cuda()) {+ return GGML_CUDA_MAX_DEVICES;+ } else {+ return 1;+ }// #if defined(GGML_USE_METAL)// return 1;-// #elif defined(GGML_USE_CUBLAS)+// #elif defined(GGML_USE_CUDA)// return GGML_CUDA_MAX_DEVICES;// #elif defined(GGML_USE_SYCL)// return GGML_SYCL_MAX_DEVICES;@@ -15140,10 +15163,6 @@ void llama_backend_init(void) {struct ggml_context * ctx = ggml_init(params);ggml_free(ctx);}--#ifdef GGML_USE_MPI- ggml_mpi_backend_init();-#endif}void llama_numa_init(enum ggml_numa_strategy numa) {@@ -15153,9 +15172,6 @@ void llama_numa_init(enum ggml_numa_strategy numa) {}void llama_backend_free(void) {-#ifdef GGML_USE_MPI- ggml_mpi_backend_free();-#endifggml_quantize_free();}@@ -15186,7 +15202,17 @@ struct llama_model * llama_load_model_from_file(return true;};}-+ if (params.rpc_servers != nullptr) {+ // split the servers set them into model->rpc_servers+ std::string servers(params.rpc_servers);+ size_t pos = 0;+ while ((pos = servers.find(",")) != std::string::npos) {+ std::string server = servers.substr(0, pos);+ model->rpc_servers.push_back(server);+ servers.erase(0, pos + 1);+ }+ model->rpc_servers.push_back(servers);+ }int status = llama_model_load(path_model, *model, params);GGML_ASSERT(status <= 0);if (status < 0) {@@ -15225,6 +15251,11 @@ struct llama_context * llama_new_context_with_model(return nullptr;}+ if (params.flash_attn && model->arch == LLM_ARCH_GROK) {+ LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);+ params.flash_attn = false;+ }+llama_context * ctx = new llama_context(*model);const auto & hparams = model->hparams;@@ -15248,7 +15279,7 @@ struct llama_context * llama_new_context_with_model(cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;// this is necessary due to kv_self.n being padded later during inference- cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);+ cparams.n_ctx = GGML_PAD(cparams.n_ctx, llama_kv_cache_get_padding(cparams));// with causal attention, the batch size is limited by the context sizecparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;@@ -15283,6 +15314,7 @@ struct llama_context * llama_new_context_with_model(cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;}+ cparams.yarn_attn_factor *= hparams.rope_attn_factor;cparams.causal_attn = hparams.causal_attn;if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {@@ -15293,23 +15325,6 @@ struct llama_context * llama_new_context_with_model(}}- if (cparams.flash_attn && hparams.use_alibi) {- LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with ALiBi - forcing off\n", __func__);- cparams.flash_attn = false;- }-- if (cparams.flash_attn && model->arch == LLM_ARCH_GROK) {- LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);- cparams.flash_attn = false;- }--// #ifdef GGML_USE_HIPBLAS [jart]- if (cparams.flash_attn && llamafile_has_amd_gpu()) {- LLAMA_LOG_WARN("%s: flash_attn is not yet compatible with AMD GPUs builds - forcing off\n", __func__);- cparams.flash_attn = false;- }-// #endif-if (params.seed == LLAMA_DEFAULT_SEED) {params.seed = _rand64();}@@ -15345,6 +15360,17 @@ struct llama_context * llama_new_context_with_model(if (!hparams.vocab_only) {// initialize backends+// #if defined(GGML_USE_RPC)+// for (auto & server : model->rpc_servers) {+// ggml_backend_t backend = ggml_backend_rpc_init(server.c_str());+// if (backend == nullptr) {+// LLAMA_LOG_ERROR("%s: failed to connect RPC backend to %s\n", __func__, server.c_str());+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// }+// #elif defined(GGML_USE_METAL)if (llamafile_has_metal()) {if (model->n_gpu_layers > 0) {ctx->backend_metal = ggml_backend_metal_init();@@ -15378,7 +15404,20 @@ if (llamafile_has_metal()) {}}// #elif defined(GGML_USE_VULKAN)-// if (model->n_gpu_layers > 0) {+// if (model->split_mode == LLAMA_SPLIT_MODE_ROW) {+// LLAMA_LOG_ERROR("%s: Row split not supported. Failed to initialize Vulkan backend\n", __func__);+// llama_free(ctx);+// return nullptr;+// }+// if (model->split_mode == LLAMA_SPLIT_MODE_NONE) {+// ggml_backend_t backend = ggml_backend_vk_init(0);+// if (backend == nullptr) {+// LLAMA_LOG_ERROR("%s: failed to initialize Vulkan backend\n", __func__);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// } else {// for (int device = 0; device < ggml_backend_vk_get_device_count(); ++device) {// ggml_backend_t backend = ggml_backend_vk_init(device);// if (backend == nullptr) {@@ -15390,30 +15429,28 @@ if (llamafile_has_metal()) {// }// }// #elif defined(GGML_USE_SYCL)-// if (model->n_gpu_layers > 0) {-// // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used-// if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {-// ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);+// // with split_mode LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_ROW, only the main GPU backend is used+// if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {+// ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);+// if (backend == nullptr) {+// int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu);+// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu);+// llama_free(ctx);+// return nullptr;+// }+// ctx->backends.push_back(backend);+// } else {+// // LLAMA_SPLIT_LAYER requires a backend for each GPU+// for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) {+// ggml_backend_t backend = ggml_backend_sycl_init(i);// if (backend == nullptr) {-// int main_gpu_id = ggml_backend_sycl_get_device_id(model->main_gpu);-// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu);+// int id_list[GGML_SYCL_MAX_DEVICES];+// ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES);+// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i);// llama_free(ctx);// return nullptr;// }// ctx->backends.push_back(backend);-// } else {-// // LLAMA_SPLIT_LAYER requires a backend for each GPU-// for (int i = 0; i < ggml_backend_sycl_get_device_count(); ++i) {-// ggml_backend_t backend = ggml_backend_sycl_init(i);-// if (backend == nullptr) {-// int id_list[GGML_SYCL_MAX_DEVICES];-// ggml_sycl_get_gpu_list(id_list, GGML_SYCL_MAX_DEVICES);-// LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, id_list[i], i);-// llama_free(ctx);-// return nullptr;-// }-// ctx->backends.push_back(backend);-// }// }// }// #elif defined(GGML_USE_KOMPUTE)@@ -15427,7 +15464,7 @@ if (llamafile_has_metal()) {// ctx->backends.push_back(backend);// }// #endif-}+} // [jart]ctx->backend_cpu = ggml_backend_cpu_init();if (ctx->backend_cpu == nullptr) {LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);@@ -15491,9 +15528,13 @@ if (llamafile_has_metal()) {ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false));// enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary- bool pipeline_parallel = llama_get_device_count() > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER;+ bool pipeline_parallel =+ llama_get_device_count(*model) > 1 &&+ model->n_gpu_layers > (int)model->hparams.n_layer &&+ model->split_mode == LLAMA_SPLIT_MODE_LAYER &&+ params.offload_kqv;// #ifndef GGML_USE_CUDA- if (llamafile_has_cuda()) {+ if (!llamafile_has_cuda()) {// pipeline parallelism requires support for async compute and events// currently this is only implemented in the CUDA backendpipeline_parallel = false;@@ -15536,20 +15577,6 @@ if (llamafile_has_metal()) {}}-#ifdef GGML_USE_MPI- ctx->ctx_mpi = ggml_mpi_init();-- if (ggml_mpi_rank(ctx->ctx_mpi) > 0) {- // Enter a blocking eval loop with dummy input, letting rank=0 drive the process- // TODO: needs fix after #3228- GGML_ASSERT(false && "not implemented");- //const std::vector<llama_token> tmp(ctx->model.hparams.n_ctx, llama_token_bos(ctx));- //while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {};- llama_backend_free();- exit(1);- }-#endif-return ctx;}@@ -15591,6 +15618,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_REFACT:case LLM_ARCH_BLOOM:case LLM_ARCH_MAMBA:+ case LLM_ARCH_JINA_BERT_V2:return LLAMA_ROPE_TYPE_NONE;// use what we call a normal RoPE, operating on pairs of consecutive head values@@ -15611,7 +15639,6 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_FALCON:case LLM_ARCH_GROK:case LLM_ARCH_DBRX:- case LLM_ARCH_PERSIMMON:case LLM_ARCH_BERT:case LLM_ARCH_NOMIC_BERT:case LLM_ARCH_STABLELM:@@ -15781,6 +15808,7 @@ static bool llama_control_vector_init(struct llama_control_vector & cvec, const}// make tensors+ cvec.tensors.reserve(model.hparams.n_layer);cvec.tensors.push_back(nullptr); // there's never a tensor for layer 0for (size_t il = 1; il < model.hparams.n_layer; il++) {struct ggml_context * ctx = ctx_map.at(model.buft_layer[il].buft);@@ -15789,6 +15817,8 @@ static bool llama_control_vector_init(struct llama_control_vector & cvec, const}// allocate tensors / buffers and zero+ cvec.ctxs.reserve(ctx_map.size());+ cvec.bufs.reserve(ctx_map.size());for (auto it : ctx_map) {ggml_backend_buffer_type_t buft = it.first;ggml_context * ctx = it.second;@@ -16612,13 +16642,13 @@ static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llam}else {if (cell_range_begin != kv_self.size) {- cell_ranges.push_back({ cell_range_begin, i });+ cell_ranges.emplace_back(cell_range_begin, i);cell_range_begin = kv_self.size;}}}if (cell_range_begin != kv_self.size) {- cell_ranges.push_back({ cell_range_begin, kv_self.size });+ cell_ranges.emplace_back(cell_range_begin, kv_self.size);}// DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count@@ -17654,7 +17684,7 @@ struct llama_timings llama_get_timings(struct llama_context * ctx) {/*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us,/*.n_sample =*/ std::max(1, ctx->n_sample),- /*.n_p_eval =*/ std::max(1, ctx->n_p_eval),+ /*.n_p_eval =*/ std::max(0, ctx->n_p_eval),/*.n_eval =*/ std::max(1, ctx->n_eval),};@@ -17695,6 +17725,7 @@ const char * llama_print_system_info(void) {s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | ";s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | ";s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | ";+ s += "AVX512_BF16 = " + std::to_string(ggml_cpu_has_avx512_bf16()) + " | ";s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | ";s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | ";s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | ";@@ -17755,6 +17786,8 @@ void llama_log_set(ggml_log_callback log_callback, void * user_data) {g_state.log_callback_user_data = user_data;#ifdef GGML_USE_METAL // [jart] do not wantggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);+#elif defined(GGML_USE_CUDA)+ ggml_backend_cuda_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);#endif}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex fdb95f58..e4b51582 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -83,7 +83,10 @@ extern "C" {LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,LLAMA_VOCAB_PRE_TYPE_REFACT = 8,LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,- LLAMA_VOCAB_PRE_TYPE_OLMO = 10,+ LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,+ LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,+ LLAMA_VOCAB_PRE_TYPE_OLMO = 12,+ LLAMA_VOCAB_PRE_TYPE_DBRX = 13,};// note: these values should be synchronized with ggml_rope@@ -242,6 +245,9 @@ extern "C" {// proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()const float * tensor_split;+ // comma separated list of RPC servers to use for offloading+ const char * rpc_servers;+// Called with a progress value between 0.0 and 1.0. Pass NULL to disable.// If the provided progress_callback returns true, model loading continues.// If it returns false, model loading is immediately aborted.diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex fdcc8b93..90745781 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -98,6 +98,7 @@ static std::string format(const char * fmt, ...) {#define TN_POS_EMBD "%s.position_embd.weight"#define TN_CLASS_EMBD "v.class_embd"#define TN_PATCH_EMBD "v.patch_embd.weight"+#define TN_PATCH_BIAS "v.patch_embd.bias"#define TN_ATTN_K "%s.blk.%d.attn_k.%s"#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"#define TN_ATTN_V "%s.blk.%d.attn_v.%s"@@ -420,6 +421,7 @@ struct clip_vision_model {// embeddingsstruct ggml_tensor * class_embedding;struct ggml_tensor * patch_embeddings;+ struct ggml_tensor * patch_bias;struct ggml_tensor * position_embeddings;struct ggml_tensor * pre_ln_w;@@ -496,6 +498,11 @@ struct clip_ctx {bool use_gelu = false;int32_t ftype = 1;+ bool has_class_embedding = true;+ bool has_pre_norm = true;+ bool has_post_norm = false;+ bool has_patch_bias = false;+struct gguf_context * ctx_gguf;struct ggml_context * ctx_data;@@ -521,7 +528,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32const int patch_size = hparams.patch_size;const int num_patches = ((image_size / patch_size) * (image_size / patch_size));const int num_patches_per_side = image_size / patch_size; GGML_UNUSED(num_patches_per_side);- const int num_positions = num_patches + 1;+ const int num_positions = num_patches + (ctx->has_class_embedding ? 1 : 0);const int hidden_size = hparams.hidden_size;const int n_head = hparams.n_head;const int d_head = hidden_size / n_head;@@ -552,17 +559,23 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));- // [jart] revert https://github.com/ggerganov/llama.cpp/pull/6899/files- // do not sync these lines until https://github.com/ggerganov/llama.cpp/issues/7060 is resolved- struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);- ggml_set_name(embeddings, "embeddings");- ggml_set_input(embeddings);+ if (ctx->has_patch_bias) {+ // inp = ggml_add(ctx0, inp, ggml_repeat(ctx0, model.patch_bias, inp));+ inp = ggml_add(ctx0, inp, model.patch_bias);+ }- embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,- embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);+ // concat class_embeddings and patch_embeddings+ struct ggml_tensor * embeddings = inp;+ if (ctx->has_class_embedding) {+ embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);+ ggml_set_name(embeddings, "embeddings");+ ggml_set_input(embeddings);+ embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);+ embeddings = ggml_acc(ctx0, embeddings, inp,+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);+ }- embeddings = ggml_acc(ctx0, embeddings, inp,- embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);ggml_set_name(positions, "positions");@@ -572,7 +585,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions));// pre-layernorm- {+ if (ctx->has_pre_norm) {embeddings = ggml_norm(ctx0, embeddings, eps);ggml_set_name(embeddings, "pre_ln");@@ -660,6 +673,14 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32embeddings = cur;}+ // post-layernorm+ if (ctx->has_post_norm) {+ embeddings = ggml_norm(ctx0, embeddings, eps);+ ggml_set_name(embeddings, "post_ln");++ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.post_ln_w), model.post_ln_b);+ }+// llava projector{embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);@@ -1142,12 +1163,39 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}+ try {+ vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);+ new_clip->has_class_embedding = true;+ } catch (const std::exception& e) {+ new_clip->has_class_embedding = false;+ }++ try {+ vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));+ vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));+ new_clip->has_pre_norm = true;+ } catch (std::exception & e) {+ new_clip->has_pre_norm = false;+ }++ try {+ vision_model.post_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_POST, "v", "weight"));+ vision_model.post_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_POST, "v", "bias"));+ new_clip->has_post_norm = true;+ } catch (std::exception & e) {+ new_clip->has_post_norm = false;+ }++ try {+ vision_model.patch_bias = get_tensor(new_clip->ctx_data, TN_PATCH_BIAS);+ new_clip->has_patch_bias = true;+ } catch (std::exception & e) {+ new_clip->has_patch_bias = false;+ }+try {vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD);- vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));- vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));- vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));} catch(const std::exception& e) {LOG_TEE("%s: failed to load vision model tensors\n", __func__);}@@ -1791,7 +1839,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_imaconst int image_size = hparams.image_size;const int patch_size = hparams.patch_size;const int num_patches = ((image_size / patch_size) * (image_size / patch_size));- const int num_positions = num_patches + 1;+ const int num_positions = num_patches + (ctx->has_class_embedding ? 1 : 0);{struct ggml_tensor * inp_raw = ggml_graph_get_tensor(gf, "inp_raw");@@ -1819,12 +1867,14 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima}{- struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings");+ if (ctx->has_class_embedding) {+ struct ggml_tensor * embeddings = ggml_graph_get_tensor(gf, "embeddings");- void* zero_mem = malloc(ggml_nbytes(embeddings));- memset(zero_mem, 0, ggml_nbytes(embeddings));- ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));- free(zero_mem);+ void* zero_mem = malloc(ggml_nbytes(embeddings));+ memset(zero_mem, 0, ggml_nbytes(embeddings));+ ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));+ free(zero_mem);+ }}{diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex 6b5c8657..b855d945 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -202,7 +202,7 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_LOG_TEE("\n");struct llama_sampling_context * ctx_sampling = llama_sampling_init(params->sparams);- if (!ctx_sampling) { // [jart] fixes crash+ if (!ctx_sampling) {fprintf(stderr, "%s: failed to initialize sampling subsystem\n", __func__);exit(1);}@@ -312,14 +312,10 @@ int llava_cli(int argc, char ** argv, gpt_params & params) {return 1;}- for (auto & image : params.image) {+ if (prompt_contains_image(params.prompt)) {auto ctx_llava = llava_init_context(¶ms, model);- auto image_embed = load_image(ctx_llava, ¶ms, image);- if (!image_embed) {- std::cerr << "error: failed to load image " << image << ". Terminating\n\n";- return 1;- }+ auto image_embed = load_image(ctx_llava, ¶ms, "");// process the promptprocess_prompt(ctx_llava, image_embed, ¶ms, params.prompt);@@ -328,7 +324,26 @@ int llava_cli(int argc, char ** argv, gpt_params & params) {llava_image_embed_free(image_embed);ctx_llava->model = NULL;llava_free(ctx_llava);+ } else {+ for (auto & image : params.image) {+ auto ctx_llava = llava_init_context(¶ms, model);++ auto image_embed = load_image(ctx_llava, ¶ms, image);+ if (!image_embed) {+ std::cerr << "error: failed to load image " << image << ". Terminating\n\n";+ return 1;+ }++ // process the prompt+ process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);++ llama_print_timings(ctx_llava->ctx_llama);+ llava_image_embed_free(image_embed);+ ctx_llava->model = NULL;+ llava_free(ctx_llava);+ }}+llama_free_model(model);return 0;diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cppindex 82c7905c..98cdaedf 100644--- a/llama.cpp/llava/llava.cpp+++ b/llama.cpp/llava/llava.cpp@@ -90,7 +90,6 @@ static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair<// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out)static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *> & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) {struct {- struct ggml_tensor * newline;struct ggml_context * ctx;} model;@@ -152,20 +151,6 @@ static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *>model.ctx = ggml_init(params);- ggml_tensor * newline_tmp = clip_get_newline_tensor(ctx_clip);- model.newline = ggml_new_tensor_1d(model.ctx, GGML_TYPE_F32, newline_tmp->ne[0]);- if (newline_tmp->backend != GGML_BACKEND_TYPE_CPU) {- if (newline_tmp->buffer == NULL) {- LOG_TEE("newline_tmp tensor buffer is NULL\n");- }- ggml_backend_tensor_get(newline_tmp, model.newline->data, 0, ggml_nbytes(newline_tmp));- } else {- model.newline->data = newline_tmp->data;- if (model.newline->data == NULL) {- LOG_TEE("newline_tmp tensor data is NULL\n");- }- }-struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4// ggml_tensor_printf(image_features,"image_features",__LINE__,false,false);// fill it with the image embeddings, ignoring the basediff --git a/llama.cpp/log.h b/llama.cpp/log.hindex db1ab355..376daefd 100644--- a/llama.cpp/log.h+++ b/llama.cpp/log.h@@ -214,7 +214,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::#define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__#else#define LOG_FLF_FMT "[%24s:%5ld][%24s] "- #define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__+ #define LOG_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__#endif#else#define LOG_FLF_FMT "%s"@@ -227,7 +227,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::#define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__#else#define LOG_TEE_FLF_FMT "[%24s:%5ld][%24s] "- #define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__+ #define LOG_TEE_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__#endif#else#define LOG_TEE_FLF_FMT "%s"@@ -237,7 +237,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// INTERNAL, DO NOT USE// USE LOG() INSTEAD//-#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)#define LOG_IMPL(str, ...) \do { \if (LOG_TARGET != nullptr) \@@ -260,7 +260,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// INTERNAL, DO NOT USE// USE LOG_TEE() INSTEAD//-#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER)+#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)#define LOG_TEE_IMPL(str, ...) \do { \if (LOG_TARGET != nullptr) \@@ -297,7 +297,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// Main LOG macro.// behaves like printf, and supports arguments the exact same way.//-#ifndef _MSC_VER+#if !defined(_MSC_VER) || defined(__clang__)#define LOG(...) LOG_IMPL(__VA_ARGS__, "")#else#define LOG(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "")@@ -311,14 +311,14 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std::// Secondary target can be changed just like LOG_TARGET// by defining LOG_TEE_TARGET//-#ifndef _MSC_VER+#if !defined(_MSC_VER) || defined(__clang__)#define LOG_TEE(...) LOG_TEE_IMPL(__VA_ARGS__, "")#else#define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "")#endif// LOG macro variants with auto endline.-#ifndef _MSC_VER+#if !defined(_MSC_VER) || defined(__clang__)#define LOGLN(...) LOG_IMPL(__VA_ARGS__, "\n")#define LOG_TEELN(...) LOG_TEE_IMPL(__VA_ARGS__, "\n")#elsediff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 4c0eb97d..8064b294 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -446,6 +446,9 @@ int main(int argc, char ** argv) {params.interactive_first = true;params.antiprompt.emplace_back("<|im_start|>user\n");}+ else if (params.conversation) {+ params.interactive_first = true;+ }// enable interactive mode if interactive start is specifiedif (params.interactive_first) {@@ -593,11 +596,10 @@ int main(int argc, char ** argv) {antiprompt_ids.emplace_back(::llama_tokenize(ctx, antiprompt, false, true));}- bool should_show_special_tokens = sparams.grammar.empty(); // [jart] for shell scriptabilitystruct llama_sampling_context * ctx_sampling = llama_sampling_init(sparams);- if (!ctx_sampling) { // [jart] fixes crash+ if (!ctx_sampling) {fprintf(stderr, "%s: failed to initialize sampling subsystem\n", __func__);- return 1;+ exit(1);}while ((n_remain != 0 && !is_antiprompt) || params.interactive) {@@ -623,7 +625,7 @@ int main(int argc, char ** argv) {// if we run out of context:// - take the n_keep first tokens from the original prompt (via n_past)// - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches- if (n_past + (int) embd.size() + std::max<int>(0, guidance_offset) > n_ctx) {+ if (n_past + (int) embd.size() + std::max<int>(0, guidance_offset) >= n_ctx) {if (params.n_predict == -2) {LOG_TEE("\n\n%s: context full and n_predict == -%d => stopping\n", __func__, params.n_predict);break;@@ -779,7 +781,7 @@ int main(int argc, char ** argv) {const llama_token id = llama_sampling_sample(ctx_sampling, ctx, ctx_guidance);- llama_sampling_accept(ctx_sampling, ctx, id, true);+ llama_sampling_accept(ctx_sampling, ctx, id, /* apply_grammar= */ true);LOG("last: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, ctx_sampling->prev).c_str());@@ -800,7 +802,7 @@ int main(int argc, char ** argv) {// push the prompt in the sampling context in order to apply repetition penalties later// for the prompt, we don't apply grammar rules- llama_sampling_accept(ctx_sampling, ctx, embd_inp[n_consumed], false);+ llama_sampling_accept(ctx_sampling, ctx, embd_inp[n_consumed], /* apply_grammar= */ false);++n_consumed;if ((int) embd.size() >= params.n_batch) {@@ -812,12 +814,7 @@ int main(int argc, char ** argv) {// display textif (input_echo && display) {for (auto id : embd) {-- // [jart] shell scriptability- const std::string token_str =- llama_token_to_piece(- ctx, id, should_show_special_tokens);-+ const std::string token_str = llama_token_to_piece(ctx, id, !params.conversation && sparams.grammar.empty());printf("%s", token_str.c_str());if (embd.size() > 1) {@@ -880,7 +877,7 @@ int main(int argc, char ** argv) {// deal with end of generation tokens in interactive modeif (llama_token_is_eog(model, llama_sampling_last(ctx_sampling))) {- LOG("found EOS token\n");+ LOG("found an EOG token\n");if (params.interactive) {if (!params.antiprompt.empty()) {@@ -900,7 +897,7 @@ int main(int argc, char ** argv) {if (n_past > 0 && is_interacting) {LOG("waiting for user input\n");- if (params.instruct || params.chatml) {+ if (params.conversation || params.instruct || params.chatml) {printf("\n> ");}@@ -910,7 +907,7 @@ int main(int argc, char ** argv) {}std::string buffer;- if (!params.input_prefix.empty()) {+ if (!params.input_prefix.empty() && !params.conversation) {LOG("appending input prefix: '%s'\n", params.input_prefix.c_str());printf("%s", params.input_prefix.c_str());}@@ -934,7 +931,7 @@ int main(int argc, char ** argv) {// Entering a empty line lets the user pass control backif (buffer.length() > 1) {// append input suffix if any- if (!params.input_suffix.empty()) {+ if (!params.input_suffix.empty() && !params.conversation) {LOG("appending input suffix: '%s'\n", params.input_suffix.c_str());printf("%s", params.input_suffix.c_str());}@@ -960,7 +957,7 @@ int main(int argc, char ** argv) {}const auto line_pfx = ::llama_tokenize(ctx, params.input_prefix, false, true);- const auto line_inp = ::llama_tokenize(ctx, buffer, false, false);+ const auto line_inp = ::llama_tokenize(ctx, buffer, false, params.interactive_specials);const auto line_sfx = ::llama_tokenize(ctx, params.input_suffix, false, true);LOG("input tokens: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, line_inp).c_str());diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cppindex 4e0931a3..26242da9 100644--- a/llama.cpp/perplexity/perplexity.cpp+++ b/llama.cpp/perplexity/perplexity.cpp@@ -1427,7 +1427,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params// Use all taskstasks.resize(n_task);printf("%s: reading tasks", __func__);- int n_dot = n_task/100;+ int n_dot = std::max((int) n_task/100, 1);int i = 0;for (auto& task : tasks) {++i;@@ -1677,7 +1677,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & paramsllama_batch_free(batch);- if (n_done < 100) return;+ if (n_done < 100 && (params.multiple_choice_tasks != 0 && params.multiple_choice_tasks < (size_t)n_task)) return;float p = 1.f*n_correct/n_done;float sigma = sqrt(p*(1-p)/(n_done-1));diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 249e346a..8659a942 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -52,8 +52,8 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 4.45G, +0.0122 ppl @ LLaMA-v1-7B", },{ "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 5.15G, +0.0008 ppl @ LLaMA-v1-7B", },{ "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 6.70G, +0.0004 ppl @ LLaMA-v1-7B", },- { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "Google Brain Floating Point", },- { "F16", LLAMA_FTYPE_MOSTLY_F16, "13.00G @ 7B", },+ { "F16", LLAMA_FTYPE_MOSTLY_F16, "14.00G, -0.0020 ppl @ Mistral-7B", },+ { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "14.00G, -0.0050 ppl @ Mistral-7B", },{ "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },// Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.{ "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },@@ -64,13 +64,6 @@ static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrixstatic const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";-static bool is_integer_str(const char *s) {- if (*s == '-') ++s;- if (!*s) return false;- while (isdigit(*s)) ++s;- return !*s;-}-static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {std::string ftype_str;@@ -84,7 +77,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftypreturn true;}}- if (is_integer_str(ftype_str.c_str())) {+ try {int ftype_int = std::stoi(ftype_str);for (auto & it : QUANT_OPTIONS) {if (it.ftype == ftype_int) {@@ -94,6 +87,9 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp}}}+ catch (...) {+ // stoi failed+ }return false;}@@ -309,7 +305,7 @@ int main(int argc, char ** argv) {} else {usage(argv[0]);}- } else if (strcmp(argv[arg_idx], "--keep-split")) {+ } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {params.keep_split = true;} else {usage(argv[0]);diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex dd19c1e2..080ad591 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -38,7 +38,7 @@ struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_result->prev.resize(params.n_prev);- result->n_considered = 0;+ result->n_valid = 0;llama_sampling_set_rng_seed(result, params.seed);@@ -69,7 +69,7 @@ void llama_sampling_reset(llama_sampling_context * ctx) {std::fill(ctx->prev.begin(), ctx->prev.end(), 0);ctx->cur.clear();- ctx->n_considered = 0;+ ctx->n_valid = 0;}void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {@@ -182,7 +182,7 @@ static llama_token llama_sampling_sample_impl(struct llama_context * ctx_main,struct llama_context * ctx_cfg,const int idx,- bool is_resampling) { // Add a parameter to indicate if we are resampling+ bool is_resampling) {const llama_sampling_params & params = ctx_sampling->params;const float temp = params.temp;@@ -191,8 +191,8 @@ static llama_token llama_sampling_sample_impl(const float mirostat_eta = params.mirostat_eta;std::vector<float> original_logits;- auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, !is_resampling, &original_logits);- if (!is_resampling) {+ auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, /* apply_grammar= */ is_resampling, &original_logits);+ if (ctx_sampling->grammar != NULL && !is_resampling) {GGML_ASSERT(!original_logits.empty());}llama_token id = 0;@@ -255,11 +255,11 @@ static llama_token llama_sampling_sample_impl(// Restore logits from the copystd::copy(original_logits.begin(), original_logits.end(), logits);- return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, true); // Pass true for is_resampling+ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ true);}}- ctx_sampling->n_considered = cur_p.size;+ ctx_sampling->n_valid = temp == 0.0f ? 0 : cur_p.size;return id;}@@ -288,7 +288,8 @@ static llama_token_data_array llama_sampling_prepare_impl(// Get a pointer to the logitsfloat * logits = llama_get_logits_ith(ctx_main, idx);- if (apply_grammar && original_logits != NULL) {+ if (ctx_sampling->grammar != NULL && !apply_grammar) {+ GGML_ASSERT(original_logits != NULL);// Only make a copy of the original logits if we are not applying grammar checks, not sure if I actually have to do this.*original_logits = {logits, logits + llama_n_vocab(llama_get_model(ctx_main))};}@@ -345,7 +346,7 @@ llama_token llama_sampling_sample(struct llama_context * ctx_cfg,const int idx) {// Call the implementation function with is_resampling set to false by default- return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false);+ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ false);}llama_token_data_array llama_sampling_prepare(diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex 5b73ecdc..655732ad 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -81,7 +81,7 @@ struct llama_sampling_context {// TODO: replace with ring-bufferstd::vector<llama_token> prev;std::vector<llama_token_data> cur;- size_t n_considered;+ size_t n_valid; // Number of correct top tokens with correct probabilities.std::mt19937 rng;};diff --git a/llama.cpp/unicode-data.cpp b/llama.cpp/unicode-data.cppindex 07bf02c4..d7c1c898 100644--- a/llama.cpp/unicode-data.cpp+++ b/llama.cpp/unicode-data.cpp@@ -1,1693 +1,6983 @@-#include "unicode-data.h"--#include <cstdint>-#include <map>-#include <utility>-#include <vector>-// generated with scripts/gen-unicode-data.py-//-// TODO: generate unicode_map_lowercase-// TODO: generate unicode_map_nfd-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_number = {-{0x00000030, 0x00000039}, {0x000000B2, 0x000000B3}, {0x000000B9, 0x000000B9}, {0x000000BC, 0x000000BE},-{0x00000660, 0x00000669}, {0x000006F0, 0x000006F9}, {0x000007C0, 0x000007C9}, {0x00000966, 0x0000096F},-{0x000009E6, 0x000009EF}, {0x000009F4, 0x000009F9}, {0x00000A66, 0x00000A6F}, {0x00000AE6, 0x00000AEF},-{0x00000B66, 0x00000B6F}, {0x00000B72, 0x00000B77}, {0x00000BE6, 0x00000BF2}, {0x00000C66, 0x00000C6F},-{0x00000C78, 0x00000C7E}, {0x00000CE6, 0x00000CEF}, {0x00000D58, 0x00000D5E}, {0x00000D66, 0x00000D78},-{0x00000DE6, 0x00000DEF}, {0x00000E50, 0x00000E59}, {0x00000ED0, 0x00000ED9}, {0x00000F20, 0x00000F33},-{0x00001040, 0x00001049}, {0x00001090, 0x00001099}, {0x00001369, 0x0000137C}, {0x000016EE, 0x000016F0},-{0x000017E0, 0x000017E9}, {0x000017F0, 0x000017F9}, {0x00001810, 0x00001819}, {0x00001946, 0x0000194F},-{0x000019D0, 0x000019DA}, {0x00001A80, 0x00001A89}, {0x00001A90, 0x00001A99}, {0x00001B50, 0x00001B59},-{0x00001BB0, 0x00001BB9}, {0x00001C40, 0x00001C49}, {0x00001C50, 0x00001C59}, {0x00002070, 0x00002070},-{0x00002074, 0x00002079}, {0x00002080, 0x00002089}, {0x00002150, 0x00002182}, {0x00002185, 0x00002189},-{0x00002460, 0x0000249B}, {0x000024EA, 0x000024FF}, {0x00002776, 0x00002793}, {0x00002CFD, 0x00002CFD},-{0x00003007, 0x00003007}, {0x00003021, 0x00003029}, {0x00003038, 0x0000303A}, {0x00003192, 0x00003195},-{0x00003220, 0x00003229}, {0x00003248, 0x0000324F}, {0x00003251, 0x0000325F}, {0x00003280, 0x00003289},-{0x000032B1, 0x000032BF}, {0x0000A620, 0x0000A629}, {0x0000A6E6, 0x0000A6EF}, {0x0000A830, 0x0000A835},-{0x0000A8D0, 0x0000A8D9}, {0x0000A900, 0x0000A909}, {0x0000A9D0, 0x0000A9D9}, {0x0000A9F0, 0x0000A9F9},-{0x0000AA50, 0x0000AA59}, {0x0000ABF0, 0x0000ABF9}, {0x0000FF10, 0x0000FF19}, {0x00010107, 0x00010133},-{0x00010140, 0x00010178}, {0x0001018A, 0x0001018B}, {0x000102E1, 0x000102FB}, {0x00010320, 0x00010323},-{0x00010341, 0x00010341}, {0x0001034A, 0x0001034A}, {0x000103D1, 0x000103D5}, {0x000104A0, 0x000104A9},-{0x00010858, 0x0001085F}, {0x00010879, 0x0001087F}, {0x000108A7, 0x000108AF}, {0x000108FB, 0x000108FF},-{0x00010916, 0x0001091B}, {0x000109BC, 0x000109BD}, {0x000109C0, 0x000109CF}, {0x000109D2, 0x000109FF},-{0x00010A40, 0x00010A48}, {0x00010A7D, 0x00010A7E}, {0x00010A9D, 0x00010A9F}, {0x00010AEB, 0x00010AEF},-{0x00010B58, 0x00010B5F}, {0x00010B78, 0x00010B7F}, {0x00010BA9, 0x00010BAF}, {0x00010CFA, 0x00010CFF},-{0x00010D30, 0x00010D39}, {0x00010E60, 0x00010E7E}, {0x00010F1D, 0x00010F26}, {0x00010F51, 0x00010F54},-{0x00010FC5, 0x00010FCB}, {0x00011052, 0x0001106F}, {0x000110F0, 0x000110F9}, {0x00011136, 0x0001113F},-{0x000111D0, 0x000111D9}, {0x000111E1, 0x000111F4}, {0x000112F0, 0x000112F9}, {0x00011450, 0x00011459},-{0x000114D0, 0x000114D9}, {0x00011650, 0x00011659}, {0x000116C0, 0x000116C9}, {0x00011730, 0x0001173B},-{0x000118E0, 0x000118F2}, {0x00011950, 0x00011959}, {0x00011C50, 0x00011C6C}, {0x00011D50, 0x00011D59},-{0x00011DA0, 0x00011DA9}, {0x00011F50, 0x00011F59}, {0x00011FC0, 0x00011FD4}, {0x00012400, 0x0001246E},-{0x00016A60, 0x00016A69}, {0x00016AC0, 0x00016AC9}, {0x00016B50, 0x00016B59}, {0x00016B5B, 0x00016B61},-{0x00016E80, 0x00016E96}, {0x0001D2C0, 0x0001D2D3}, {0x0001D2E0, 0x0001D2F3}, {0x0001D360, 0x0001D378},-{0x0001D7CE, 0x0001D7FF}, {0x0001E140, 0x0001E149}, {0x0001E2F0, 0x0001E2F9}, {0x0001E4F0, 0x0001E4F9},-{0x0001E8C7, 0x0001E8CF}, {0x0001E950, 0x0001E959}, {0x0001EC71, 0x0001ECAB}, {0x0001ECAD, 0x0001ECAF},-{0x0001ECB1, 0x0001ECB4}, {0x0001ED01, 0x0001ED2D}, {0x0001ED2F, 0x0001ED3D}, {0x0001F100, 0x0001F10C},-{0x0001FBF0, 0x0001FBF9},-};+#include "unicode-data.h"-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_letter = {-{0x00000041, 0x0000005A}, {0x00000061, 0x0000007A}, {0x000000AA, 0x000000AA}, {0x000000B5, 0x000000B5},-{0x000000BA, 0x000000BA}, {0x000000C0, 0x000000D6}, {0x000000D8, 0x000000F6}, {0x000000F8, 0x000002C1},-{0x000002C6, 0x000002D1}, {0x000002E0, 0x000002E4}, {0x000002EC, 0x000002EC}, {0x000002EE, 0x000002EE},-{0x00000370, 0x00000374}, {0x00000376, 0x00000377}, {0x0000037A, 0x0000037D}, {0x0000037F, 0x0000037F},-{0x00000386, 0x00000386}, {0x00000388, 0x0000038A}, {0x0000038C, 0x0000038C}, {0x0000038E, 0x000003A1},-{0x000003A3, 0x000003F5}, {0x000003F7, 0x00000481}, {0x0000048A, 0x0000052F}, {0x00000531, 0x00000556},-{0x00000559, 0x00000559}, {0x00000560, 0x00000588}, {0x000005D0, 0x000005EA}, {0x000005EF, 0x000005F2},-{0x00000620, 0x0000064A}, {0x0000066E, 0x0000066F}, {0x00000671, 0x000006D3}, {0x000006D5, 0x000006D5},-{0x000006E5, 0x000006E6}, {0x000006EE, 0x000006EF}, {0x000006FA, 0x000006FC}, {0x000006FF, 0x000006FF},-{0x00000710, 0x00000710}, {0x00000712, 0x0000072F}, {0x0000074D, 0x000007A5}, {0x000007B1, 0x000007B1},-{0x000007CA, 0x000007EA}, {0x000007F4, 0x000007F5}, {0x000007FA, 0x000007FA}, {0x00000800, 0x00000815},-{0x0000081A, 0x0000081A}, {0x00000824, 0x00000824}, {0x00000828, 0x00000828}, {0x00000840, 0x00000858},-{0x00000860, 0x0000086A}, {0x00000870, 0x00000887}, {0x00000889, 0x0000088E}, {0x000008A0, 0x000008C9},-{0x00000904, 0x00000939}, {0x0000093D, 0x0000093D}, {0x00000950, 0x00000950}, {0x00000958, 0x00000961},-{0x00000971, 0x00000980}, {0x00000985, 0x0000098C}, {0x0000098F, 0x00000990}, {0x00000993, 0x000009A8},-{0x000009AA, 0x000009B0}, {0x000009B2, 0x000009B2}, {0x000009B6, 0x000009B9}, {0x000009BD, 0x000009BD},-{0x000009CE, 0x000009CE}, {0x000009DC, 0x000009DD}, {0x000009DF, 0x000009E1}, {0x000009F0, 0x000009F1},-{0x000009FC, 0x000009FC}, {0x00000A05, 0x00000A0A}, {0x00000A0F, 0x00000A10}, {0x00000A13, 0x00000A28},-{0x00000A2A, 0x00000A30}, {0x00000A32, 0x00000A33}, {0x00000A35, 0x00000A36}, {0x00000A38, 0x00000A39},-{0x00000A59, 0x00000A5C}, {0x00000A5E, 0x00000A5E}, {0x00000A72, 0x00000A74}, {0x00000A85, 0x00000A8D},-{0x00000A8F, 0x00000A91}, {0x00000A93, 0x00000AA8}, {0x00000AAA, 0x00000AB0}, {0x00000AB2, 0x00000AB3},-{0x00000AB5, 0x00000AB9}, {0x00000ABD, 0x00000ABD}, {0x00000AD0, 0x00000AD0}, {0x00000AE0, 0x00000AE1},-{0x00000AF9, 0x00000AF9}, {0x00000B05, 0x00000B0C}, {0x00000B0F, 0x00000B10}, {0x00000B13, 0x00000B28},-{0x00000B2A, 0x00000B30}, {0x00000B32, 0x00000B33}, {0x00000B35, 0x00000B39}, {0x00000B3D, 0x00000B3D},-{0x00000B5C, 0x00000B5D}, {0x00000B5F, 0x00000B61}, {0x00000B71, 0x00000B71}, {0x00000B83, 0x00000B83},-{0x00000B85, 0x00000B8A}, {0x00000B8E, 0x00000B90}, {0x00000B92, 0x00000B95}, {0x00000B99, 0x00000B9A},-{0x00000B9C, 0x00000B9C}, {0x00000B9E, 0x00000B9F}, {0x00000BA3, 0x00000BA4}, {0x00000BA8, 0x00000BAA},-{0x00000BAE, 0x00000BB9}, {0x00000BD0, 0x00000BD0}, {0x00000C05, 0x00000C0C}, {0x00000C0E, 0x00000C10},-{0x00000C12, 0x00000C28}, {0x00000C2A, 0x00000C39}, {0x00000C3D, 0x00000C3D}, {0x00000C58, 0x00000C5A},-{0x00000C5D, 0x00000C5D}, {0x00000C60, 0x00000C61}, {0x00000C80, 0x00000C80}, {0x00000C85, 0x00000C8C},-{0x00000C8E, 0x00000C90}, {0x00000C92, 0x00000CA8}, {0x00000CAA, 0x00000CB3}, {0x00000CB5, 0x00000CB9},-{0x00000CBD, 0x00000CBD}, {0x00000CDD, 0x00000CDE}, {0x00000CE0, 0x00000CE1}, {0x00000CF1, 0x00000CF2},-{0x00000D04, 0x00000D0C}, {0x00000D0E, 0x00000D10}, {0x00000D12, 0x00000D3A}, {0x00000D3D, 0x00000D3D},-{0x00000D4E, 0x00000D4E}, {0x00000D54, 0x00000D56}, {0x00000D5F, 0x00000D61}, {0x00000D7A, 0x00000D7F},-{0x00000D85, 0x00000D96}, {0x00000D9A, 0x00000DB1}, {0x00000DB3, 0x00000DBB}, {0x00000DBD, 0x00000DBD},-{0x00000DC0, 0x00000DC6}, {0x00000E01, 0x00000E30}, {0x00000E32, 0x00000E33}, {0x00000E40, 0x00000E46},-{0x00000E81, 0x00000E82}, {0x00000E84, 0x00000E84}, {0x00000E86, 0x00000E8A}, {0x00000E8C, 0x00000EA3},-{0x00000EA5, 0x00000EA5}, {0x00000EA7, 0x00000EB0}, {0x00000EB2, 0x00000EB3}, {0x00000EBD, 0x00000EBD},-{0x00000EC0, 0x00000EC4}, {0x00000EC6, 0x00000EC6}, {0x00000EDC, 0x00000EDF}, {0x00000F00, 0x00000F00},-{0x00000F40, 0x00000F47}, {0x00000F49, 0x00000F6C}, {0x00000F88, 0x00000F8C}, {0x00001000, 0x0000102A},-{0x0000103F, 0x0000103F}, {0x00001050, 0x00001055}, {0x0000105A, 0x0000105D}, {0x00001061, 0x00001061},-{0x00001065, 0x00001066}, {0x0000106E, 0x00001070}, {0x00001075, 0x00001081}, {0x0000108E, 0x0000108E},-{0x000010A0, 0x000010C5}, {0x000010C7, 0x000010C7}, {0x000010CD, 0x000010CD}, {0x000010D0, 0x000010FA},-{0x000010FC, 0x00001248}, {0x0000124A, 0x0000124D}, {0x00001250, 0x00001256}, {0x00001258, 0x00001258},-{0x0000125A, 0x0000125D}, {0x00001260, 0x00001288}, {0x0000128A, 0x0000128D}, {0x00001290, 0x000012B0},-{0x000012B2, 0x000012B5}, {0x000012B8, 0x000012BE}, {0x000012C0, 0x000012C0}, {0x000012C2, 0x000012C5},-{0x000012C8, 0x000012D6}, {0x000012D8, 0x00001310}, {0x00001312, 0x00001315}, {0x00001318, 0x0000135A},-{0x00001380, 0x0000138F}, {0x000013A0, 0x000013F5}, {0x000013F8, 0x000013FD}, {0x00001401, 0x0000166C},-{0x0000166F, 0x0000167F}, {0x00001681, 0x0000169A}, {0x000016A0, 0x000016EA}, {0x000016F1, 0x000016F8},-{0x00001700, 0x00001711}, {0x0000171F, 0x00001731}, {0x00001740, 0x00001751}, {0x00001760, 0x0000176C},-{0x0000176E, 0x00001770}, {0x00001780, 0x000017B3}, {0x000017D7, 0x000017D7}, {0x000017DC, 0x000017DC},-{0x00001820, 0x00001878}, {0x00001880, 0x00001884}, {0x00001887, 0x000018A8}, {0x000018AA, 0x000018AA},-{0x000018B0, 0x000018F5}, {0x00001900, 0x0000191E}, {0x00001950, 0x0000196D}, {0x00001970, 0x00001974},-{0x00001980, 0x000019AB}, {0x000019B0, 0x000019C9}, {0x00001A00, 0x00001A16}, {0x00001A20, 0x00001A54},-{0x00001AA7, 0x00001AA7}, {0x00001B05, 0x00001B33}, {0x00001B45, 0x00001B4C}, {0x00001B83, 0x00001BA0},-{0x00001BAE, 0x00001BAF}, {0x00001BBA, 0x00001BE5}, {0x00001C00, 0x00001C23}, {0x00001C4D, 0x00001C4F},-{0x00001C5A, 0x00001C7D}, {0x00001C80, 0x00001C88}, {0x00001C90, 0x00001CBA}, {0x00001CBD, 0x00001CBF},-{0x00001CE9, 0x00001CEC}, {0x00001CEE, 0x00001CF3}, {0x00001CF5, 0x00001CF6}, {0x00001CFA, 0x00001CFA},-{0x00001D00, 0x00001DBF}, {0x00001E00, 0x00001F15}, {0x00001F18, 0x00001F1D}, {0x00001F20, 0x00001F45},-{0x00001F48, 0x00001F4D}, {0x00001F50, 0x00001F57}, {0x00001F59, 0x00001F59}, {0x00001F5B, 0x00001F5B},-{0x00001F5D, 0x00001F5D}, {0x00001F5F, 0x00001F7D}, {0x00001F80, 0x00001FB4}, {0x00001FB6, 0x00001FBC},-{0x00001FBE, 0x00001FBE}, {0x00001FC2, 0x00001FC4}, {0x00001FC6, 0x00001FCC}, {0x00001FD0, 0x00001FD3},-{0x00001FD6, 0x00001FDB}, {0x00001FE0, 0x00001FEC}, {0x00001FF2, 0x00001FF4}, {0x00001FF6, 0x00001FFC},-{0x00002071, 0x00002071}, {0x0000207F, 0x0000207F}, {0x00002090, 0x0000209C}, {0x00002102, 0x00002102},-{0x00002107, 0x00002107}, {0x0000210A, 0x00002113}, {0x00002115, 0x00002115}, {0x00002119, 0x0000211D},-{0x00002124, 0x00002124}, {0x00002126, 0x00002126}, {0x00002128, 0x00002128}, {0x0000212A, 0x0000212D},-{0x0000212F, 0x00002139}, {0x0000213C, 0x0000213F}, {0x00002145, 0x00002149}, {0x0000214E, 0x0000214E},-{0x00002183, 0x00002184}, {0x00002C00, 0x00002CE4}, {0x00002CEB, 0x00002CEE}, {0x00002CF2, 0x00002CF3},-{0x00002D00, 0x00002D25}, {0x00002D27, 0x00002D27}, {0x00002D2D, 0x00002D2D}, {0x00002D30, 0x00002D67},-{0x00002D6F, 0x00002D6F}, {0x00002D80, 0x00002D96}, {0x00002DA0, 0x00002DA6}, {0x00002DA8, 0x00002DAE},-{0x00002DB0, 0x00002DB6}, {0x00002DB8, 0x00002DBE}, {0x00002DC0, 0x00002DC6}, {0x00002DC8, 0x00002DCE},-{0x00002DD0, 0x00002DD6}, {0x00002DD8, 0x00002DDE}, {0x00002E2F, 0x00002E2F}, {0x00003005, 0x00003006},-{0x00003031, 0x00003035}, {0x0000303B, 0x0000303C}, {0x00003041, 0x00003096}, {0x0000309D, 0x0000309F},-{0x000030A1, 0x000030FA}, {0x000030FC, 0x000030FF}, {0x00003105, 0x0000312F}, {0x00003131, 0x0000318E},-{0x000031A0, 0x000031BF}, {0x000031F0, 0x000031FF}, {0x00003400, 0x00004DBF}, {0x00004E00, 0x0000A48C},-{0x0000A4D0, 0x0000A4FD}, {0x0000A500, 0x0000A60C}, {0x0000A610, 0x0000A61F}, {0x0000A62A, 0x0000A62B},-{0x0000A640, 0x0000A66E}, {0x0000A67F, 0x0000A69D}, {0x0000A6A0, 0x0000A6E5}, {0x0000A717, 0x0000A71F},-{0x0000A722, 0x0000A788}, {0x0000A78B, 0x0000A7CA}, {0x0000A7D0, 0x0000A7D1}, {0x0000A7D3, 0x0000A7D3},-{0x0000A7D5, 0x0000A7D9}, {0x0000A7F2, 0x0000A801}, {0x0000A803, 0x0000A805}, {0x0000A807, 0x0000A80A},-{0x0000A80C, 0x0000A822}, {0x0000A840, 0x0000A873}, {0x0000A882, 0x0000A8B3}, {0x0000A8F2, 0x0000A8F7},-{0x0000A8FB, 0x0000A8FB}, {0x0000A8FD, 0x0000A8FE}, {0x0000A90A, 0x0000A925}, {0x0000A930, 0x0000A946},-{0x0000A960, 0x0000A97C}, {0x0000A984, 0x0000A9B2}, {0x0000A9CF, 0x0000A9CF}, {0x0000A9E0, 0x0000A9E4},-{0x0000A9E6, 0x0000A9EF}, {0x0000A9FA, 0x0000A9FE}, {0x0000AA00, 0x0000AA28}, {0x0000AA40, 0x0000AA42},-{0x0000AA44, 0x0000AA4B}, {0x0000AA60, 0x0000AA76}, {0x0000AA7A, 0x0000AA7A}, {0x0000AA7E, 0x0000AAAF},-{0x0000AAB1, 0x0000AAB1}, {0x0000AAB5, 0x0000AAB6}, {0x0000AAB9, 0x0000AABD}, {0x0000AAC0, 0x0000AAC0},-{0x0000AAC2, 0x0000AAC2}, {0x0000AADB, 0x0000AADD}, {0x0000AAE0, 0x0000AAEA}, {0x0000AAF2, 0x0000AAF4},-{0x0000AB01, 0x0000AB06}, {0x0000AB09, 0x0000AB0E}, {0x0000AB11, 0x0000AB16}, {0x0000AB20, 0x0000AB26},-{0x0000AB28, 0x0000AB2E}, {0x0000AB30, 0x0000AB5A}, {0x0000AB5C, 0x0000AB69}, {0x0000AB70, 0x0000ABE2},-{0x0000AC00, 0x0000D7A3}, {0x0000D7B0, 0x0000D7C6}, {0x0000D7CB, 0x0000D7FB}, {0x0000F900, 0x0000FA6D},-{0x0000FA70, 0x0000FAD9}, {0x0000FB00, 0x0000FB06}, {0x0000FB13, 0x0000FB17}, {0x0000FB1D, 0x0000FB1D},-{0x0000FB1F, 0x0000FB28}, {0x0000FB2A, 0x0000FB36}, {0x0000FB38, 0x0000FB3C}, {0x0000FB3E, 0x0000FB3E},-{0x0000FB40, 0x0000FB41}, {0x0000FB43, 0x0000FB44}, {0x0000FB46, 0x0000FBB1}, {0x0000FBD3, 0x0000FD3D},-{0x0000FD50, 0x0000FD8F}, {0x0000FD92, 0x0000FDC7}, {0x0000FDF0, 0x0000FDFB}, {0x0000FE70, 0x0000FE74},-{0x0000FE76, 0x0000FEFC}, {0x0000FF21, 0x0000FF3A}, {0x0000FF41, 0x0000FF5A}, {0x0000FF66, 0x0000FFBE},-{0x0000FFC2, 0x0000FFC7}, {0x0000FFCA, 0x0000FFCF}, {0x0000FFD2, 0x0000FFD7}, {0x0000FFDA, 0x0000FFDC},-{0x00010000, 0x0001000B}, {0x0001000D, 0x00010026}, {0x00010028, 0x0001003A}, {0x0001003C, 0x0001003D},-{0x0001003F, 0x0001004D}, {0x00010050, 0x0001005D}, {0x00010080, 0x000100FA}, {0x00010280, 0x0001029C},-{0x000102A0, 0x000102D0}, {0x00010300, 0x0001031F}, {0x0001032D, 0x00010340}, {0x00010342, 0x00010349},-{0x00010350, 0x00010375}, {0x00010380, 0x0001039D}, {0x000103A0, 0x000103C3}, {0x000103C8, 0x000103CF},-{0x00010400, 0x0001049D}, {0x000104B0, 0x000104D3}, {0x000104D8, 0x000104FB}, {0x00010500, 0x00010527},-{0x00010530, 0x00010563}, {0x00010570, 0x0001057A}, {0x0001057C, 0x0001058A}, {0x0001058C, 0x00010592},-{0x00010594, 0x00010595}, {0x00010597, 0x000105A1}, {0x000105A3, 0x000105B1}, {0x000105B3, 0x000105B9},-{0x000105BB, 0x000105BC}, {0x00010600, 0x00010736}, {0x00010740, 0x00010755}, {0x00010760, 0x00010767},-{0x00010780, 0x00010785}, {0x00010787, 0x000107B0}, {0x000107B2, 0x000107BA}, {0x00010800, 0x00010805},-{0x00010808, 0x00010808}, {0x0001080A, 0x00010835}, {0x00010837, 0x00010838}, {0x0001083C, 0x0001083C},-{0x0001083F, 0x00010855}, {0x00010860, 0x00010876}, {0x00010880, 0x0001089E}, {0x000108E0, 0x000108F2},-{0x000108F4, 0x000108F5}, {0x00010900, 0x00010915}, {0x00010920, 0x00010939}, {0x00010980, 0x000109B7},-{0x000109BE, 0x000109BF}, {0x00010A00, 0x00010A00}, {0x00010A10, 0x00010A13}, {0x00010A15, 0x00010A17},-{0x00010A19, 0x00010A35}, {0x00010A60, 0x00010A7C}, {0x00010A80, 0x00010A9C}, {0x00010AC0, 0x00010AC7},-{0x00010AC9, 0x00010AE4}, {0x00010B00, 0x00010B35}, {0x00010B40, 0x00010B55}, {0x00010B60, 0x00010B72},-{0x00010B80, 0x00010B91}, {0x00010C00, 0x00010C48}, {0x00010C80, 0x00010CB2}, {0x00010CC0, 0x00010CF2},-{0x00010D00, 0x00010D23}, {0x00010E80, 0x00010EA9}, {0x00010EB0, 0x00010EB1}, {0x00010F00, 0x00010F1C},-{0x00010F27, 0x00010F27}, {0x00010F30, 0x00010F45}, {0x00010F70, 0x00010F81}, {0x00010FB0, 0x00010FC4},-{0x00010FE0, 0x00010FF6}, {0x00011003, 0x00011037}, {0x00011071, 0x00011072}, {0x00011075, 0x00011075},-{0x00011083, 0x000110AF}, {0x000110D0, 0x000110E8}, {0x00011103, 0x00011126}, {0x00011144, 0x00011144},-{0x00011147, 0x00011147}, {0x00011150, 0x00011172}, {0x00011176, 0x00011176}, {0x00011183, 0x000111B2},-{0x000111C1, 0x000111C4}, {0x000111DA, 0x000111DA}, {0x000111DC, 0x000111DC}, {0x00011200, 0x00011211},-{0x00011213, 0x0001122B}, {0x0001123F, 0x00011240}, {0x00011280, 0x00011286}, {0x00011288, 0x00011288},-{0x0001128A, 0x0001128D}, {0x0001128F, 0x0001129D}, {0x0001129F, 0x000112A8}, {0x000112B0, 0x000112DE},-{0x00011305, 0x0001130C}, {0x0001130F, 0x00011310}, {0x00011313, 0x00011328}, {0x0001132A, 0x00011330},-{0x00011332, 0x00011333}, {0x00011335, 0x00011339}, {0x0001133D, 0x0001133D}, {0x00011350, 0x00011350},-{0x0001135D, 0x00011361}, {0x00011400, 0x00011434}, {0x00011447, 0x0001144A}, {0x0001145F, 0x00011461},-{0x00011480, 0x000114AF}, {0x000114C4, 0x000114C5}, {0x000114C7, 0x000114C7}, {0x00011580, 0x000115AE},-{0x000115D8, 0x000115DB}, {0x00011600, 0x0001162F}, {0x00011644, 0x00011644}, {0x00011680, 0x000116AA},-{0x000116B8, 0x000116B8}, {0x00011700, 0x0001171A}, {0x00011740, 0x00011746}, {0x00011800, 0x0001182B},-{0x000118A0, 0x000118DF}, {0x000118FF, 0x00011906}, {0x00011909, 0x00011909}, {0x0001190C, 0x00011913},-{0x00011915, 0x00011916}, {0x00011918, 0x0001192F}, {0x0001193F, 0x0001193F}, {0x00011941, 0x00011941},-{0x000119A0, 0x000119A7}, {0x000119AA, 0x000119D0}, {0x000119E1, 0x000119E1}, {0x000119E3, 0x000119E3},-{0x00011A00, 0x00011A00}, {0x00011A0B, 0x00011A32}, {0x00011A3A, 0x00011A3A}, {0x00011A50, 0x00011A50},-{0x00011A5C, 0x00011A89}, {0x00011A9D, 0x00011A9D}, {0x00011AB0, 0x00011AF8}, {0x00011C00, 0x00011C08},-{0x00011C0A, 0x00011C2E}, {0x00011C40, 0x00011C40}, {0x00011C72, 0x00011C8F}, {0x00011D00, 0x00011D06},-{0x00011D08, 0x00011D09}, {0x00011D0B, 0x00011D30}, {0x00011D46, 0x00011D46}, {0x00011D60, 0x00011D65},-{0x00011D67, 0x00011D68}, {0x00011D6A, 0x00011D89}, {0x00011D98, 0x00011D98}, {0x00011EE0, 0x00011EF2},-{0x00011F02, 0x00011F02}, {0x00011F04, 0x00011F10}, {0x00011F12, 0x00011F33}, {0x00011FB0, 0x00011FB0},-{0x00012000, 0x00012399}, {0x00012480, 0x00012543}, {0x00012F90, 0x00012FF0}, {0x00013000, 0x0001342F},-{0x00013441, 0x00013446}, {0x00014400, 0x00014646}, {0x00016800, 0x00016A38}, {0x00016A40, 0x00016A5E},-{0x00016A70, 0x00016ABE}, {0x00016AD0, 0x00016AED}, {0x00016B00, 0x00016B2F}, {0x00016B40, 0x00016B43},-{0x00016B63, 0x00016B77}, {0x00016B7D, 0x00016B8F}, {0x00016E40, 0x00016E7F}, {0x00016F00, 0x00016F4A},-{0x00016F50, 0x00016F50}, {0x00016F93, 0x00016F9F}, {0x00016FE0, 0x00016FE1}, {0x00016FE3, 0x00016FE3},-{0x00017000, 0x000187F7}, {0x00018800, 0x00018CD5}, {0x00018D00, 0x00018D08}, {0x0001AFF0, 0x0001AFF3},-{0x0001AFF5, 0x0001AFFB}, {0x0001AFFD, 0x0001AFFE}, {0x0001B000, 0x0001B122}, {0x0001B132, 0x0001B132},-{0x0001B150, 0x0001B152}, {0x0001B155, 0x0001B155}, {0x0001B164, 0x0001B167}, {0x0001B170, 0x0001B2FB},-{0x0001BC00, 0x0001BC6A}, {0x0001BC70, 0x0001BC7C}, {0x0001BC80, 0x0001BC88}, {0x0001BC90, 0x0001BC99},-{0x0001D400, 0x0001D454}, {0x0001D456, 0x0001D49C}, {0x0001D49E, 0x0001D49F}, {0x0001D4A2, 0x0001D4A2},-{0x0001D4A5, 0x0001D4A6}, {0x0001D4A9, 0x0001D4AC}, {0x0001D4AE, 0x0001D4B9}, {0x0001D4BB, 0x0001D4BB},-{0x0001D4BD, 0x0001D4C3}, {0x0001D4C5, 0x0001D505}, {0x0001D507, 0x0001D50A}, {0x0001D50D, 0x0001D514},-{0x0001D516, 0x0001D51C}, {0x0001D51E, 0x0001D539}, {0x0001D53B, 0x0001D53E}, {0x0001D540, 0x0001D544},-{0x0001D546, 0x0001D546}, {0x0001D54A, 0x0001D550}, {0x0001D552, 0x0001D6A5}, {0x0001D6A8, 0x0001D6C0},-{0x0001D6C2, 0x0001D6DA}, {0x0001D6DC, 0x0001D6FA}, {0x0001D6FC, 0x0001D714}, {0x0001D716, 0x0001D734},-{0x0001D736, 0x0001D74E}, {0x0001D750, 0x0001D76E}, {0x0001D770, 0x0001D788}, {0x0001D78A, 0x0001D7A8},-{0x0001D7AA, 0x0001D7C2}, {0x0001D7C4, 0x0001D7CB}, {0x0001DF00, 0x0001DF1E}, {0x0001DF25, 0x0001DF2A},-{0x0001E030, 0x0001E06D}, {0x0001E100, 0x0001E12C}, {0x0001E137, 0x0001E13D}, {0x0001E14E, 0x0001E14E},-{0x0001E290, 0x0001E2AD}, {0x0001E2C0, 0x0001E2EB}, {0x0001E4D0, 0x0001E4EB}, {0x0001E7E0, 0x0001E7E6},-{0x0001E7E8, 0x0001E7EB}, {0x0001E7ED, 0x0001E7EE}, {0x0001E7F0, 0x0001E7FE}, {0x0001E800, 0x0001E8C4},-{0x0001E900, 0x0001E943}, {0x0001E94B, 0x0001E94B}, {0x0001EE00, 0x0001EE03}, {0x0001EE05, 0x0001EE1F},-{0x0001EE21, 0x0001EE22}, {0x0001EE24, 0x0001EE24}, {0x0001EE27, 0x0001EE27}, {0x0001EE29, 0x0001EE32},-{0x0001EE34, 0x0001EE37}, {0x0001EE39, 0x0001EE39}, {0x0001EE3B, 0x0001EE3B}, {0x0001EE42, 0x0001EE42},-{0x0001EE47, 0x0001EE47}, {0x0001EE49, 0x0001EE49}, {0x0001EE4B, 0x0001EE4B}, {0x0001EE4D, 0x0001EE4F},-{0x0001EE51, 0x0001EE52}, {0x0001EE54, 0x0001EE54}, {0x0001EE57, 0x0001EE57}, {0x0001EE59, 0x0001EE59},-{0x0001EE5B, 0x0001EE5B}, {0x0001EE5D, 0x0001EE5D}, {0x0001EE5F, 0x0001EE5F}, {0x0001EE61, 0x0001EE62},-{0x0001EE64, 0x0001EE64}, {0x0001EE67, 0x0001EE6A}, {0x0001EE6C, 0x0001EE72}, {0x0001EE74, 0x0001EE77},-{0x0001EE79, 0x0001EE7C}, {0x0001EE7E, 0x0001EE7E}, {0x0001EE80, 0x0001EE89}, {0x0001EE8B, 0x0001EE9B},-{0x0001EEA1, 0x0001EEA3}, {0x0001EEA5, 0x0001EEA9}, {0x0001EEAB, 0x0001EEBB}, {0x00020000, 0x0002A6DF},-{0x0002A700, 0x0002B739}, {0x0002B740, 0x0002B81D}, {0x0002B820, 0x0002CEA1}, {0x0002CEB0, 0x0002EBE0},-{0x0002F800, 0x0002FA1D}, {0x00030000, 0x0003134A}, {0x00031350, 0x000323AF},-};--const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_whitespace = {-{0x00000020, 0x00000020}, {0x000000A0, 0x000000A0}, {0x00001680, 0x00001680}, {0x00002000, 0x0000200A},-{0x00002028, 0x00002029}, {0x0000202F, 0x0000202F}, {0x0000205F, 0x0000205F}, {0x00003000, 0x00003000},-};+#include <cstdint>+#include <vector>+#include <unordered_map>+#include <unordered_set>-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_mark = {-{0x00000300, 0x0000036F}, {0x00000483, 0x00000489}, {0x00000591, 0x000005BD}, {0x000005BF, 0x000005BF},-{0x000005C1, 0x000005C2}, {0x000005C4, 0x000005C5}, {0x000005C7, 0x000005C7}, {0x00000610, 0x0000061A},-{0x0000064B, 0x0000065F}, {0x00000670, 0x00000670}, {0x000006D6, 0x000006DC}, {0x000006DF, 0x000006E4},-{0x000006E7, 0x000006E8}, {0x000006EA, 0x000006ED}, {0x00000711, 0x00000711}, {0x00000730, 0x0000074A},-{0x000007A6, 0x000007B0}, {0x000007EB, 0x000007F3}, {0x000007FD, 0x000007FD}, {0x00000816, 0x00000819},-{0x0000081B, 0x00000823}, {0x00000825, 0x00000827}, {0x00000829, 0x0000082D}, {0x00000859, 0x0000085B},-{0x00000898, 0x0000089F}, {0x000008CA, 0x000008E1}, {0x000008E3, 0x00000903}, {0x0000093A, 0x0000093C},-{0x0000093E, 0x0000094F}, {0x00000951, 0x00000957}, {0x00000962, 0x00000963}, {0x00000981, 0x00000983},-{0x000009BC, 0x000009BC}, {0x000009BE, 0x000009C4}, {0x000009C7, 0x000009C8}, {0x000009CB, 0x000009CD},-{0x000009D7, 0x000009D7}, {0x000009E2, 0x000009E3}, {0x000009FE, 0x000009FE}, {0x00000A01, 0x00000A03},-{0x00000A3C, 0x00000A3C}, {0x00000A3E, 0x00000A42}, {0x00000A47, 0x00000A48}, {0x00000A4B, 0x00000A4D},-{0x00000A51, 0x00000A51}, {0x00000A70, 0x00000A71}, {0x00000A75, 0x00000A75}, {0x00000A81, 0x00000A83},-{0x00000ABC, 0x00000ABC}, {0x00000ABE, 0x00000AC5}, {0x00000AC7, 0x00000AC9}, {0x00000ACB, 0x00000ACD},-{0x00000AE2, 0x00000AE3}, {0x00000AFA, 0x00000AFF}, {0x00000B01, 0x00000B03}, {0x00000B3C, 0x00000B3C},-{0x00000B3E, 0x00000B44}, {0x00000B47, 0x00000B48}, {0x00000B4B, 0x00000B4D}, {0x00000B55, 0x00000B57},-{0x00000B62, 0x00000B63}, {0x00000B82, 0x00000B82}, {0x00000BBE, 0x00000BC2}, {0x00000BC6, 0x00000BC8},-{0x00000BCA, 0x00000BCD}, {0x00000BD7, 0x00000BD7}, {0x00000C00, 0x00000C04}, {0x00000C3C, 0x00000C3C},-{0x00000C3E, 0x00000C44}, {0x00000C46, 0x00000C48}, {0x00000C4A, 0x00000C4D}, {0x00000C55, 0x00000C56},-{0x00000C62, 0x00000C63}, {0x00000C81, 0x00000C83}, {0x00000CBC, 0x00000CBC}, {0x00000CBE, 0x00000CC4},-{0x00000CC6, 0x00000CC8}, {0x00000CCA, 0x00000CCD}, {0x00000CD5, 0x00000CD6}, {0x00000CE2, 0x00000CE3},-{0x00000CF3, 0x00000CF3}, {0x00000D00, 0x00000D03}, {0x00000D3B, 0x00000D3C}, {0x00000D3E, 0x00000D44},-{0x00000D46, 0x00000D48}, {0x00000D4A, 0x00000D4D}, {0x00000D57, 0x00000D57}, {0x00000D62, 0x00000D63},-{0x00000D81, 0x00000D83}, {0x00000DCA, 0x00000DCA}, {0x00000DCF, 0x00000DD4}, {0x00000DD6, 0x00000DD6},-{0x00000DD8, 0x00000DDF}, {0x00000DF2, 0x00000DF3}, {0x00000E31, 0x00000E31}, {0x00000E34, 0x00000E3A},-{0x00000E47, 0x00000E4E}, {0x00000EB1, 0x00000EB1}, {0x00000EB4, 0x00000EBC}, {0x00000EC8, 0x00000ECE},-{0x00000F18, 0x00000F19}, {0x00000F35, 0x00000F35}, {0x00000F37, 0x00000F37}, {0x00000F39, 0x00000F39},-{0x00000F3E, 0x00000F3F}, {0x00000F71, 0x00000F84}, {0x00000F86, 0x00000F87}, {0x00000F8D, 0x00000F97},-{0x00000F99, 0x00000FBC}, {0x00000FC6, 0x00000FC6}, {0x0000102B, 0x0000103E}, {0x00001056, 0x00001059},-{0x0000105E, 0x00001060}, {0x00001062, 0x00001064}, {0x00001067, 0x0000106D}, {0x00001071, 0x00001074},-{0x00001082, 0x0000108D}, {0x0000108F, 0x0000108F}, {0x0000109A, 0x0000109D}, {0x0000135D, 0x0000135F},-{0x00001712, 0x00001715}, {0x00001732, 0x00001734}, {0x00001752, 0x00001753}, {0x00001772, 0x00001773},-{0x000017B4, 0x000017D3}, {0x000017DD, 0x000017DD}, {0x0000180B, 0x0000180D}, {0x0000180F, 0x0000180F},-{0x00001885, 0x00001886}, {0x000018A9, 0x000018A9}, {0x00001920, 0x0000192B}, {0x00001930, 0x0000193B},-{0x00001A17, 0x00001A1B}, {0x00001A55, 0x00001A5E}, {0x00001A60, 0x00001A7C}, {0x00001A7F, 0x00001A7F},-{0x00001AB0, 0x00001ACE}, {0x00001B00, 0x00001B04}, {0x00001B34, 0x00001B44}, {0x00001B6B, 0x00001B73},-{0x00001B80, 0x00001B82}, {0x00001BA1, 0x00001BAD}, {0x00001BE6, 0x00001BF3}, {0x00001C24, 0x00001C37},-{0x00001CD0, 0x00001CD2}, {0x00001CD4, 0x00001CE8}, {0x00001CED, 0x00001CED}, {0x00001CF4, 0x00001CF4},-{0x00001CF7, 0x00001CF9}, {0x00001DC0, 0x00001DFF}, {0x000020D0, 0x000020F0}, {0x00002CEF, 0x00002CF1},-{0x00002D7F, 0x00002D7F}, {0x00002DE0, 0x00002DFF}, {0x0000302A, 0x0000302F}, {0x00003099, 0x0000309A},-{0x0000A66F, 0x0000A672}, {0x0000A674, 0x0000A67D}, {0x0000A69E, 0x0000A69F}, {0x0000A6F0, 0x0000A6F1},-{0x0000A802, 0x0000A802}, {0x0000A806, 0x0000A806}, {0x0000A80B, 0x0000A80B}, {0x0000A823, 0x0000A827},-{0x0000A82C, 0x0000A82C}, {0x0000A880, 0x0000A881}, {0x0000A8B4, 0x0000A8C5}, {0x0000A8E0, 0x0000A8F1},-{0x0000A8FF, 0x0000A8FF}, {0x0000A926, 0x0000A92D}, {0x0000A947, 0x0000A953}, {0x0000A980, 0x0000A983},-{0x0000A9B3, 0x0000A9C0}, {0x0000A9E5, 0x0000A9E5}, {0x0000AA29, 0x0000AA36}, {0x0000AA43, 0x0000AA43},-{0x0000AA4C, 0x0000AA4D}, {0x0000AA7B, 0x0000AA7D}, {0x0000AAB0, 0x0000AAB0}, {0x0000AAB2, 0x0000AAB4},-{0x0000AAB7, 0x0000AAB8}, {0x0000AABE, 0x0000AABF}, {0x0000AAC1, 0x0000AAC1}, {0x0000AAEB, 0x0000AAEF},-{0x0000AAF5, 0x0000AAF6}, {0x0000ABE3, 0x0000ABEA}, {0x0000ABEC, 0x0000ABED}, {0x0000FB1E, 0x0000FB1E},-{0x0000FE00, 0x0000FE0F}, {0x0000FE20, 0x0000FE2F}, {0x000101FD, 0x000101FD}, {0x000102E0, 0x000102E0},-{0x00010376, 0x0001037A}, {0x00010A01, 0x00010A03}, {0x00010A05, 0x00010A06}, {0x00010A0C, 0x00010A0F},-{0x00010A38, 0x00010A3A}, {0x00010A3F, 0x00010A3F}, {0x00010AE5, 0x00010AE6}, {0x00010D24, 0x00010D27},-{0x00010EAB, 0x00010EAC}, {0x00010EFD, 0x00010EFF}, {0x00010F46, 0x00010F50}, {0x00010F82, 0x00010F85},-{0x00011000, 0x00011002}, {0x00011038, 0x00011046}, {0x00011070, 0x00011070}, {0x00011073, 0x00011074},-{0x0001107F, 0x00011082}, {0x000110B0, 0x000110BA}, {0x000110C2, 0x000110C2}, {0x00011100, 0x00011102},-{0x00011127, 0x00011134}, {0x00011145, 0x00011146}, {0x00011173, 0x00011173}, {0x00011180, 0x00011182},-{0x000111B3, 0x000111C0}, {0x000111C9, 0x000111CC}, {0x000111CE, 0x000111CF}, {0x0001122C, 0x00011237},-{0x0001123E, 0x0001123E}, {0x00011241, 0x00011241}, {0x000112DF, 0x000112EA}, {0x00011300, 0x00011303},-{0x0001133B, 0x0001133C}, {0x0001133E, 0x00011344}, {0x00011347, 0x00011348}, {0x0001134B, 0x0001134D},-{0x00011357, 0x00011357}, {0x00011362, 0x00011363}, {0x00011366, 0x0001136C}, {0x00011370, 0x00011374},-{0x00011435, 0x00011446}, {0x0001145E, 0x0001145E}, {0x000114B0, 0x000114C3}, {0x000115AF, 0x000115B5},-{0x000115B8, 0x000115C0}, {0x000115DC, 0x000115DD}, {0x00011630, 0x00011640}, {0x000116AB, 0x000116B7},-{0x0001171D, 0x0001172B}, {0x0001182C, 0x0001183A}, {0x00011930, 0x00011935}, {0x00011937, 0x00011938},-{0x0001193B, 0x0001193E}, {0x00011940, 0x00011940}, {0x00011942, 0x00011943}, {0x000119D1, 0x000119D7},-{0x000119DA, 0x000119E0}, {0x000119E4, 0x000119E4}, {0x00011A01, 0x00011A0A}, {0x00011A33, 0x00011A39},-{0x00011A3B, 0x00011A3E}, {0x00011A47, 0x00011A47}, {0x00011A51, 0x00011A5B}, {0x00011A8A, 0x00011A99},-{0x00011C2F, 0x00011C36}, {0x00011C38, 0x00011C3F}, {0x00011C92, 0x00011CA7}, {0x00011CA9, 0x00011CB6},-{0x00011D31, 0x00011D36}, {0x00011D3A, 0x00011D3A}, {0x00011D3C, 0x00011D3D}, {0x00011D3F, 0x00011D45},-{0x00011D47, 0x00011D47}, {0x00011D8A, 0x00011D8E}, {0x00011D90, 0x00011D91}, {0x00011D93, 0x00011D97},-{0x00011EF3, 0x00011EF6}, {0x00011F00, 0x00011F01}, {0x00011F03, 0x00011F03}, {0x00011F34, 0x00011F3A},-{0x00011F3E, 0x00011F42}, {0x00013440, 0x00013440}, {0x00013447, 0x00013455}, {0x00016AF0, 0x00016AF4},-{0x00016B30, 0x00016B36}, {0x00016F4F, 0x00016F4F}, {0x00016F51, 0x00016F87}, {0x00016F8F, 0x00016F92},-{0x00016FE4, 0x00016FE4}, {0x00016FF0, 0x00016FF1}, {0x0001BC9D, 0x0001BC9E}, {0x0001CF00, 0x0001CF2D},-{0x0001CF30, 0x0001CF46}, {0x0001D165, 0x0001D169}, {0x0001D16D, 0x0001D172}, {0x0001D17B, 0x0001D182},-{0x0001D185, 0x0001D18B}, {0x0001D1AA, 0x0001D1AD}, {0x0001D242, 0x0001D244}, {0x0001DA00, 0x0001DA36},-{0x0001DA3B, 0x0001DA6C}, {0x0001DA75, 0x0001DA75}, {0x0001DA84, 0x0001DA84}, {0x0001DA9B, 0x0001DA9F},-{0x0001DAA1, 0x0001DAAF}, {0x0001E000, 0x0001E006}, {0x0001E008, 0x0001E018}, {0x0001E01B, 0x0001E021},-{0x0001E023, 0x0001E024}, {0x0001E026, 0x0001E02A}, {0x0001E08F, 0x0001E08F}, {0x0001E130, 0x0001E136},-{0x0001E2AE, 0x0001E2AE}, {0x0001E2EC, 0x0001E2EF}, {0x0001E4EC, 0x0001E4EF}, {0x0001E8D0, 0x0001E8D6},-{0x0001E944, 0x0001E94A}, {0x000E0100, 0x000E01EF},+const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags = { // start, flags // last=next_start-1+{0x000000, 0x0080},+{0x000020, 0x0008},+{0x000021, 0x0020},+{0x000024, 0x0040},+{0x000025, 0x0020},+{0x00002B, 0x0040},+{0x00002C, 0x0020},+{0x000030, 0x0002},+{0x00003A, 0x0020},+{0x00003C, 0x0040},+{0x00003F, 0x0020},+{0x000041, 0x0004},+{0x00005B, 0x0020},+{0x00005E, 0x0040},+{0x00005F, 0x0020},+{0x000060, 0x0040},+{0x000061, 0x0004},+{0x00007B, 0x0020},+{0x00007C, 0x0040},+{0x00007D, 0x0020},+{0x00007E, 0x0040},+{0x00007F, 0x0080},+{0x0000A0, 0x0008},+{0x0000A1, 0x0020},+{0x0000A2, 0x0040},+{0x0000A7, 0x0020},+{0x0000A8, 0x0040},+{0x0000AA, 0x0004},+{0x0000AB, 0x0020},+{0x0000AC, 0x0040},+{0x0000AD, 0x0080},+{0x0000AE, 0x0040},+{0x0000B2, 0x0002},+{0x0000B4, 0x0040},+{0x0000B5, 0x0004},+{0x0000B6, 0x0020},+{0x0000B8, 0x0040},+{0x0000B9, 0x0002},+{0x0000BA, 0x0004},+{0x0000BB, 0x0020},+{0x0000BC, 0x0002},+{0x0000BF, 0x0020},+{0x0000C0, 0x0004},+{0x0000D7, 0x0040},+{0x0000D8, 0x0004},+{0x0000F7, 0x0040},+{0x0000F8, 0x0004},+{0x0002C2, 0x0040},+{0x0002C6, 0x0004},+{0x0002D2, 0x0040},+{0x0002E0, 0x0004},+{0x0002E5, 0x0040},+{0x0002EC, 0x0004},+{0x0002ED, 0x0040},+{0x0002EE, 0x0004},+{0x0002EF, 0x0040},+{0x000300, 0x0010},+{0x000370, 0x0004},+{0x000375, 0x0040},+{0x000376, 0x0004},+{0x000378, 0x0080},+{0x00037A, 0x0004},+{0x00037E, 0x0020},+{0x00037F, 0x0004},+{0x000380, 0x0080},+{0x000384, 0x0040},+{0x000386, 0x0004},+{0x000387, 0x0020},+{0x000388, 0x0004},+{0x00038B, 0x0080},+{0x00038C, 0x0004},+{0x00038D, 0x0080},+{0x00038E, 0x0004},+{0x0003A2, 0x0080},+{0x0003A3, 0x0004},+{0x0003F6, 0x0040},+{0x0003F7, 0x0004},+{0x000482, 0x0040},+{0x000483, 0x0010},+{0x00048A, 0x0004},+{0x000530, 0x0080},+{0x000531, 0x0004},+{0x000557, 0x0080},+{0x000559, 0x0004},+{0x00055A, 0x0020},+{0x000560, 0x0004},+{0x000589, 0x0020},+{0x00058B, 0x0080},+{0x00058D, 0x0040},+{0x000590, 0x0080},+{0x000591, 0x0010},+{0x0005BE, 0x0020},+{0x0005BF, 0x0010},+{0x0005C0, 0x0020},+{0x0005C1, 0x0010},+{0x0005C3, 0x0020},+{0x0005C4, 0x0010},+{0x0005C6, 0x0020},+{0x0005C7, 0x0010},+{0x0005C8, 0x0080},+{0x0005D0, 0x0004},+{0x0005EB, 0x0080},+{0x0005EF, 0x0004},+{0x0005F3, 0x0020},+{0x0005F5, 0x0080},+{0x000606, 0x0040},+{0x000609, 0x0020},+{0x00060B, 0x0040},+{0x00060C, 0x0020},+{0x00060E, 0x0040},+{0x000610, 0x0010},+{0x00061B, 0x0020},+{0x00061C, 0x0080},+{0x00061D, 0x0020},+{0x000620, 0x0004},+{0x00064B, 0x0010},+{0x000660, 0x0002},+{0x00066A, 0x0020},+{0x00066E, 0x0004},+{0x000670, 0x0010},+{0x000671, 0x0004},+{0x0006D4, 0x0020},+{0x0006D5, 0x0004},+{0x0006D6, 0x0010},+{0x0006DD, 0x0080},+{0x0006DE, 0x0040},+{0x0006DF, 0x0010},+{0x0006E5, 0x0004},+{0x0006E7, 0x0010},+{0x0006E9, 0x0040},+{0x0006EA, 0x0010},+{0x0006EE, 0x0004},+{0x0006F0, 0x0002},+{0x0006FA, 0x0004},+{0x0006FD, 0x0040},+{0x0006FF, 0x0004},+{0x000700, 0x0020},+{0x00070E, 0x0080},+{0x000710, 0x0004},+{0x000711, 0x0010},+{0x000712, 0x0004},+{0x000730, 0x0010},+{0x00074B, 0x0080},+{0x00074D, 0x0004},+{0x0007A6, 0x0010},+{0x0007B1, 0x0004},+{0x0007B2, 0x0080},+{0x0007C0, 0x0002},+{0x0007CA, 0x0004},+{0x0007EB, 0x0010},+{0x0007F4, 0x0004},+{0x0007F6, 0x0040},+{0x0007F7, 0x0020},+{0x0007FA, 0x0004},+{0x0007FB, 0x0080},+{0x0007FD, 0x0010},+{0x0007FE, 0x0040},+{0x000800, 0x0004},+{0x000816, 0x0010},+{0x00081A, 0x0004},+{0x00081B, 0x0010},+{0x000824, 0x0004},+{0x000825, 0x0010},+{0x000828, 0x0004},+{0x000829, 0x0010},+{0x00082E, 0x0080},+{0x000830, 0x0020},+{0x00083F, 0x0080},+{0x000840, 0x0004},+{0x000859, 0x0010},+{0x00085C, 0x0080},+{0x00085E, 0x0020},+{0x00085F, 0x0080},+{0x000860, 0x0004},+{0x00086B, 0x0080},+{0x000870, 0x0004},+{0x000888, 0x0040},+{0x000889, 0x0004},+{0x00088F, 0x0080},+{0x000898, 0x0010},+{0x0008A0, 0x0004},+{0x0008CA, 0x0010},+{0x0008E2, 0x0080},+{0x0008E3, 0x0010},+{0x000904, 0x0004},+{0x00093A, 0x0010},+{0x00093D, 0x0004},+{0x00093E, 0x0010},+{0x000950, 0x0004},+{0x000951, 0x0010},+{0x000958, 0x0004},+{0x000962, 0x0010},+{0x000964, 0x0020},+{0x000966, 0x0002},+{0x000970, 0x0020},+{0x000971, 0x0004},+{0x000981, 0x0010},+{0x000984, 0x0080},+{0x000985, 0x0004},+{0x00098D, 0x0080},+{0x00098F, 0x0004},+{0x000991, 0x0080},+{0x000993, 0x0004},+{0x0009A9, 0x0080},+{0x0009AA, 0x0004},+{0x0009B1, 0x0080},+{0x0009B2, 0x0004},+{0x0009B3, 0x0080},+{0x0009B6, 0x0004},+{0x0009BA, 0x0080},+{0x0009BC, 0x0010},+{0x0009BD, 0x0004},+{0x0009BE, 0x0010},+{0x0009C5, 0x0080},+{0x0009C7, 0x0010},+{0x0009C9, 0x0080},+{0x0009CB, 0x0010},+{0x0009CE, 0x0004},+{0x0009CF, 0x0080},+{0x0009D7, 0x0010},+{0x0009D8, 0x0080},+{0x0009DC, 0x0004},+{0x0009DE, 0x0080},+{0x0009DF, 0x0004},+{0x0009E2, 0x0010},+{0x0009E4, 0x0080},+{0x0009E6, 0x0002},+{0x0009F0, 0x0004},+{0x0009F2, 0x0040},+{0x0009F4, 0x0002},+{0x0009FA, 0x0040},+{0x0009FC, 0x0004},+{0x0009FD, 0x0020},+{0x0009FE, 0x0010},+{0x0009FF, 0x0080},+{0x000A01, 0x0010},+{0x000A04, 0x0080},+{0x000A05, 0x0004},+{0x000A0B, 0x0080},+{0x000A0F, 0x0004},+{0x000A11, 0x0080},+{0x000A13, 0x0004},+{0x000A29, 0x0080},+{0x000A2A, 0x0004},+{0x000A31, 0x0080},+{0x000A32, 0x0004},+{0x000A34, 0x0080},+{0x000A35, 0x0004},+{0x000A37, 0x0080},+{0x000A38, 0x0004},+{0x000A3A, 0x0080},+{0x000A3C, 0x0010},+{0x000A3D, 0x0080},+{0x000A3E, 0x0010},+{0x000A43, 0x0080},+{0x000A47, 0x0010},+{0x000A49, 0x0080},+{0x000A4B, 0x0010},+{0x000A4E, 0x0080},+{0x000A51, 0x0010},+{0x000A52, 0x0080},+{0x000A59, 0x0004},+{0x000A5D, 0x0080},+{0x000A5E, 0x0004},+{0x000A5F, 0x0080},+{0x000A66, 0x0002},+{0x000A70, 0x0010},+{0x000A72, 0x0004},+{0x000A75, 0x0010},+{0x000A76, 0x0020},+{0x000A77, 0x0080},+{0x000A81, 0x0010},+{0x000A84, 0x0080},+{0x000A85, 0x0004},+{0x000A8E, 0x0080},+{0x000A8F, 0x0004},+{0x000A92, 0x0080},+{0x000A93, 0x0004},+{0x000AA9, 0x0080},+{0x000AAA, 0x0004},+{0x000AB1, 0x0080},+{0x000AB2, 0x0004},+{0x000AB4, 0x0080},+{0x000AB5, 0x0004},+{0x000ABA, 0x0080},+{0x000ABC, 0x0010},+{0x000ABD, 0x0004},+{0x000ABE, 0x0010},+{0x000AC6, 0x0080},+{0x000AC7, 0x0010},+{0x000ACA, 0x0080},+{0x000ACB, 0x0010},+{0x000ACE, 0x0080},+{0x000AD0, 0x0004},+{0x000AD1, 0x0080},+{0x000AE0, 0x0004},+{0x000AE2, 0x0010},+{0x000AE4, 0x0080},+{0x000AE6, 0x0002},+{0x000AF0, 0x0020},+{0x000AF1, 0x0040},+{0x000AF2, 0x0080},+{0x000AF9, 0x0004},+{0x000AFA, 0x0010},+{0x000B00, 0x0080},+{0x000B01, 0x0010},+{0x000B04, 0x0080},+{0x000B05, 0x0004},+{0x000B0D, 0x0080},+{0x000B0F, 0x0004},+{0x000B11, 0x0080},+{0x000B13, 0x0004},+{0x000B29, 0x0080},+{0x000B2A, 0x0004},+{0x000B31, 0x0080},+{0x000B32, 0x0004},+{0x000B34, 0x0080},+{0x000B35, 0x0004},+{0x000B3A, 0x0080},+{0x000B3C, 0x0010},+{0x000B3D, 0x0004},+{0x000B3E, 0x0010},+{0x000B45, 0x0080},+{0x000B47, 0x0010},+{0x000B49, 0x0080},+{0x000B4B, 0x0010},+{0x000B4E, 0x0080},+{0x000B55, 0x0010},+{0x000B58, 0x0080},+{0x000B5C, 0x0004},+{0x000B5E, 0x0080},+{0x000B5F, 0x0004},+{0x000B62, 0x0010},+{0x000B64, 0x0080},+{0x000B66, 0x0002},+{0x000B70, 0x0040},+{0x000B71, 0x0004},+{0x000B72, 0x0002},+{0x000B78, 0x0080},+{0x000B82, 0x0010},+{0x000B83, 0x0004},+{0x000B84, 0x0080},+{0x000B85, 0x0004},+{0x000B8B, 0x0080},+{0x000B8E, 0x0004},+{0x000B91, 0x0080},+{0x000B92, 0x0004},+{0x000B96, 0x0080},+{0x000B99, 0x0004},+{0x000B9B, 0x0080},+{0x000B9C, 0x0004},+{0x000B9D, 0x0080},+{0x000B9E, 0x0004},+{0x000BA0, 0x0080},+{0x000BA3, 0x0004},+{0x000BA5, 0x0080},+{0x000BA8, 0x0004},+{0x000BAB, 0x0080},+{0x000BAE, 0x0004},+{0x000BBA, 0x0080},+{0x000BBE, 0x0010},+{0x000BC3, 0x0080},+{0x000BC6, 0x0010},+{0x000BC9, 0x0080},+{0x000BCA, 0x0010},+{0x000BCE, 0x0080},+{0x000BD0, 0x0004},+{0x000BD1, 0x0080},+{0x000BD7, 0x0010},+{0x000BD8, 0x0080},+{0x000BE6, 0x0002},+{0x000BF3, 0x0040},+{0x000BFB, 0x0080},+{0x000C00, 0x0010},+{0x000C05, 0x0004},+{0x000C0D, 0x0080},+{0x000C0E, 0x0004},+{0x000C11, 0x0080},+{0x000C12, 0x0004},+{0x000C29, 0x0080},+{0x000C2A, 0x0004},+{0x000C3A, 0x0080},+{0x000C3C, 0x0010},+{0x000C3D, 0x0004},+{0x000C3E, 0x0010},+{0x000C45, 0x0080},+{0x000C46, 0x0010},+{0x000C49, 0x0080},+{0x000C4A, 0x0010},+{0x000C4E, 0x0080},+{0x000C55, 0x0010},+{0x000C57, 0x0080},+{0x000C58, 0x0004},+{0x000C5B, 0x0080},+{0x000C5D, 0x0004},+{0x000C5E, 0x0080},+{0x000C60, 0x0004},+{0x000C62, 0x0010},+{0x000C64, 0x0080},+{0x000C66, 0x0002},+{0x000C70, 0x0080},+{0x000C77, 0x0020},+{0x000C78, 0x0002},+{0x000C7F, 0x0040},+{0x000C80, 0x0004},+{0x000C81, 0x0010},+{0x000C84, 0x0020},+{0x000C85, 0x0004},+{0x000C8D, 0x0080},+{0x000C8E, 0x0004},+{0x000C91, 0x0080},+{0x000C92, 0x0004},+{0x000CA9, 0x0080},+{0x000CAA, 0x0004},+{0x000CB4, 0x0080},+{0x000CB5, 0x0004},+{0x000CBA, 0x0080},+{0x000CBC, 0x0010},+{0x000CBD, 0x0004},+{0x000CBE, 0x0010},+{0x000CC5, 0x0080},+{0x000CC6, 0x0010},+{0x000CC9, 0x0080},+{0x000CCA, 0x0010},+{0x000CCE, 0x0080},+{0x000CD5, 0x0010},+{0x000CD7, 0x0080},+{0x000CDD, 0x0004},+{0x000CDF, 0x0080},+{0x000CE0, 0x0004},+{0x000CE2, 0x0010},+{0x000CE4, 0x0080},+{0x000CE6, 0x0002},+{0x000CF0, 0x0080},+{0x000CF1, 0x0004},+{0x000CF3, 0x0010},+{0x000CF4, 0x0080},+{0x000D00, 0x0010},+{0x000D04, 0x0004},+{0x000D0D, 0x0080},+{0x000D0E, 0x0004},+{0x000D11, 0x0080},+{0x000D12, 0x0004},+{0x000D3B, 0x0010},+{0x000D3D, 0x0004},+{0x000D3E, 0x0010},+{0x000D45, 0x0080},+{0x000D46, 0x0010},+{0x000D49, 0x0080},+{0x000D4A, 0x0010},+{0x000D4E, 0x0004},+{0x000D4F, 0x0040},+{0x000D50, 0x0080},+{0x000D54, 0x0004},+{0x000D57, 0x0010},+{0x000D58, 0x0002},+{0x000D5F, 0x0004},+{0x000D62, 0x0010},+{0x000D64, 0x0080},+{0x000D66, 0x0002},+{0x000D79, 0x0040},+{0x000D7A, 0x0004},+{0x000D80, 0x0080},+{0x000D81, 0x0010},+{0x000D84, 0x0080},+{0x000D85, 0x0004},+{0x000D97, 0x0080},+{0x000D9A, 0x0004},+{0x000DB2, 0x0080},+{0x000DB3, 0x0004},+{0x000DBC, 0x0080},+{0x000DBD, 0x0004},+{0x000DBE, 0x0080},+{0x000DC0, 0x0004},+{0x000DC7, 0x0080},+{0x000DCA, 0x0010},+{0x000DCB, 0x0080},+{0x000DCF, 0x0010},+{0x000DD5, 0x0080},+{0x000DD6, 0x0010},+{0x000DD7, 0x0080},+{0x000DD8, 0x0010},+{0x000DE0, 0x0080},+{0x000DE6, 0x0002},+{0x000DF0, 0x0080},+{0x000DF2, 0x0010},+{0x000DF4, 0x0020},+{0x000DF5, 0x0080},+{0x000E01, 0x0004},+{0x000E31, 0x0010},+{0x000E32, 0x0004},+{0x000E34, 0x0010},+{0x000E3B, 0x0080},+{0x000E3F, 0x0040},+{0x000E40, 0x0004},+{0x000E47, 0x0010},+{0x000E4F, 0x0020},+{0x000E50, 0x0002},+{0x000E5A, 0x0020},+{0x000E5C, 0x0080},+{0x000E81, 0x0004},+{0x000E83, 0x0080},+{0x000E84, 0x0004},+{0x000E85, 0x0080},+{0x000E86, 0x0004},+{0x000E8B, 0x0080},+{0x000E8C, 0x0004},+{0x000EA4, 0x0080},+{0x000EA5, 0x0004},+{0x000EA6, 0x0080},+{0x000EA7, 0x0004},+{0x000EB1, 0x0010},+{0x000EB2, 0x0004},+{0x000EB4, 0x0010},+{0x000EBD, 0x0004},+{0x000EBE, 0x0080},+{0x000EC0, 0x0004},+{0x000EC5, 0x0080},+{0x000EC6, 0x0004},+{0x000EC7, 0x0080},+{0x000EC8, 0x0010},+{0x000ECF, 0x0080},+{0x000ED0, 0x0002},+{0x000EDA, 0x0080},+{0x000EDC, 0x0004},+{0x000EE0, 0x0080},+{0x000F00, 0x0004},+{0x000F01, 0x0040},+{0x000F04, 0x0020},+{0x000F13, 0x0040},+{0x000F14, 0x0020},+{0x000F15, 0x0040},+{0x000F18, 0x0010},+{0x000F1A, 0x0040},+{0x000F20, 0x0002},+{0x000F34, 0x0040},+{0x000F35, 0x0010},+{0x000F36, 0x0040},+{0x000F37, 0x0010},+{0x000F38, 0x0040},+{0x000F39, 0x0010},+{0x000F3A, 0x0020},+{0x000F3E, 0x0010},+{0x000F40, 0x0004},+{0x000F48, 0x0080},+{0x000F49, 0x0004},+{0x000F6D, 0x0080},+{0x000F71, 0x0010},+{0x000F85, 0x0020},+{0x000F86, 0x0010},+{0x000F88, 0x0004},+{0x000F8D, 0x0010},+{0x000F98, 0x0080},+{0x000F99, 0x0010},+{0x000FBD, 0x0080},+{0x000FBE, 0x0040},+{0x000FC6, 0x0010},+{0x000FC7, 0x0040},+{0x000FCD, 0x0080},+{0x000FCE, 0x0040},+{0x000FD0, 0x0020},+{0x000FD5, 0x0040},+{0x000FD9, 0x0020},+{0x000FDB, 0x0080},+{0x001000, 0x0004},+{0x00102B, 0x0010},+{0x00103F, 0x0004},+{0x001040, 0x0002},+{0x00104A, 0x0020},+{0x001050, 0x0004},+{0x001056, 0x0010},+{0x00105A, 0x0004},+{0x00105E, 0x0010},+{0x001061, 0x0004},+{0x001062, 0x0010},+{0x001065, 0x0004},+{0x001067, 0x0010},+{0x00106E, 0x0004},+{0x001071, 0x0010},+{0x001075, 0x0004},+{0x001082, 0x0010},+{0x00108E, 0x0004},+{0x00108F, 0x0010},+{0x001090, 0x0002},+{0x00109A, 0x0010},+{0x00109E, 0x0040},+{0x0010A0, 0x0004},+{0x0010C6, 0x0080},+{0x0010C7, 0x0004},+{0x0010C8, 0x0080},+{0x0010CD, 0x0004},+{0x0010CE, 0x0080},+{0x0010D0, 0x0004},+{0x0010FB, 0x0020},+{0x0010FC, 0x0004},+{0x001249, 0x0080},+{0x00124A, 0x0004},+{0x00124E, 0x0080},+{0x001250, 0x0004},+{0x001257, 0x0080},+{0x001258, 0x0004},+{0x001259, 0x0080},+{0x00125A, 0x0004},+{0x00125E, 0x0080},+{0x001260, 0x0004},+{0x001289, 0x0080},+{0x00128A, 0x0004},+{0x00128E, 0x0080},+{0x001290, 0x0004},+{0x0012B1, 0x0080},+{0x0012B2, 0x0004},+{0x0012B6, 0x0080},+{0x0012B8, 0x0004},+{0x0012BF, 0x0080},+{0x0012C0, 0x0004},+{0x0012C1, 0x0080},+{0x0012C2, 0x0004},+{0x0012C6, 0x0080},+{0x0012C8, 0x0004},+{0x0012D7, 0x0080},+{0x0012D8, 0x0004},+{0x001311, 0x0080},+{0x001312, 0x0004},+{0x001316, 0x0080},+{0x001318, 0x0004},+{0x00135B, 0x0080},+{0x00135D, 0x0010},+{0x001360, 0x0020},+{0x001369, 0x0002},+{0x00137D, 0x0080},+{0x001380, 0x0004},+{0x001390, 0x0040},+{0x00139A, 0x0080},+{0x0013A0, 0x0004},+{0x0013F6, 0x0080},+{0x0013F8, 0x0004},+{0x0013FE, 0x0080},+{0x001400, 0x0020},+{0x001401, 0x0004},+{0x00166D, 0x0040},+{0x00166E, 0x0020},+{0x00166F, 0x0004},+{0x001680, 0x0008},+{0x001681, 0x0004},+{0x00169B, 0x0020},+{0x00169D, 0x0080},+{0x0016A0, 0x0004},+{0x0016EB, 0x0020},+{0x0016EE, 0x0002},+{0x0016F1, 0x0004},+{0x0016F9, 0x0080},+{0x001700, 0x0004},+{0x001712, 0x0010},+{0x001716, 0x0080},+{0x00171F, 0x0004},+{0x001732, 0x0010},+{0x001735, 0x0020},+{0x001737, 0x0080},+{0x001740, 0x0004},+{0x001752, 0x0010},+{0x001754, 0x0080},+{0x001760, 0x0004},+{0x00176D, 0x0080},+{0x00176E, 0x0004},+{0x001771, 0x0080},+{0x001772, 0x0010},+{0x001774, 0x0080},+{0x001780, 0x0004},+{0x0017B4, 0x0010},+{0x0017D4, 0x0020},+{0x0017D7, 0x0004},+{0x0017D8, 0x0020},+{0x0017DB, 0x0040},+{0x0017DC, 0x0004},+{0x0017DD, 0x0010},+{0x0017DE, 0x0080},+{0x0017E0, 0x0002},+{0x0017EA, 0x0080},+{0x0017F0, 0x0002},+{0x0017FA, 0x0080},+{0x001800, 0x0020},+{0x00180B, 0x0010},+{0x00180E, 0x0080},+{0x00180F, 0x0010},+{0x001810, 0x0002},+{0x00181A, 0x0080},+{0x001820, 0x0004},+{0x001879, 0x0080},+{0x001880, 0x0004},+{0x001885, 0x0010},+{0x001887, 0x0004},+{0x0018A9, 0x0010},+{0x0018AA, 0x0004},+{0x0018AB, 0x0080},+{0x0018B0, 0x0004},+{0x0018F6, 0x0080},+{0x001900, 0x0004},+{0x00191F, 0x0080},+{0x001920, 0x0010},+{0x00192C, 0x0080},+{0x001930, 0x0010},+{0x00193C, 0x0080},+{0x001940, 0x0040},+{0x001941, 0x0080},+{0x001944, 0x0020},+{0x001946, 0x0002},+{0x001950, 0x0004},+{0x00196E, 0x0080},+{0x001970, 0x0004},+{0x001975, 0x0080},+{0x001980, 0x0004},+{0x0019AC, 0x0080},+{0x0019B0, 0x0004},+{0x0019CA, 0x0080},+{0x0019D0, 0x0002},+{0x0019DB, 0x0080},+{0x0019DE, 0x0040},+{0x001A00, 0x0004},+{0x001A17, 0x0010},+{0x001A1C, 0x0080},+{0x001A1E, 0x0020},+{0x001A20, 0x0004},+{0x001A55, 0x0010},+{0x001A5F, 0x0080},+{0x001A60, 0x0010},+{0x001A7D, 0x0080},+{0x001A7F, 0x0010},+{0x001A80, 0x0002},+{0x001A8A, 0x0080},+{0x001A90, 0x0002},+{0x001A9A, 0x0080},+{0x001AA0, 0x0020},+{0x001AA7, 0x0004},+{0x001AA8, 0x0020},+{0x001AAE, 0x0080},+{0x001AB0, 0x0010},+{0x001ACF, 0x0080},+{0x001B00, 0x0010},+{0x001B05, 0x0004},+{0x001B34, 0x0010},+{0x001B45, 0x0004},+{0x001B4D, 0x0080},+{0x001B50, 0x0002},+{0x001B5A, 0x0020},+{0x001B61, 0x0040},+{0x001B6B, 0x0010},+{0x001B74, 0x0040},+{0x001B7D, 0x0020},+{0x001B7F, 0x0080},+{0x001B80, 0x0010},+{0x001B83, 0x0004},+{0x001BA1, 0x0010},+{0x001BAE, 0x0004},+{0x001BB0, 0x0002},+{0x001BBA, 0x0004},+{0x001BE6, 0x0010},+{0x001BF4, 0x0080},+{0x001BFC, 0x0020},+{0x001C00, 0x0004},+{0x001C24, 0x0010},+{0x001C38, 0x0080},+{0x001C3B, 0x0020},+{0x001C40, 0x0002},+{0x001C4A, 0x0080},+{0x001C4D, 0x0004},+{0x001C50, 0x0002},+{0x001C5A, 0x0004},+{0x001C7E, 0x0020},+{0x001C80, 0x0004},+{0x001C89, 0x0080},+{0x001C90, 0x0004},+{0x001CBB, 0x0080},+{0x001CBD, 0x0004},+{0x001CC0, 0x0020},+{0x001CC8, 0x0080},+{0x001CD0, 0x0010},+{0x001CD3, 0x0020},+{0x001CD4, 0x0010},+{0x001CE9, 0x0004},+{0x001CED, 0x0010},+{0x001CEE, 0x0004},+{0x001CF4, 0x0010},+{0x001CF5, 0x0004},+{0x001CF7, 0x0010},+{0x001CFA, 0x0004},+{0x001CFB, 0x0080},+{0x001D00, 0x0004},+{0x001DC0, 0x0010},+{0x001E00, 0x0004},+{0x001F16, 0x0080},+{0x001F18, 0x0004},+{0x001F1E, 0x0080},+{0x001F20, 0x0004},+{0x001F46, 0x0080},+{0x001F48, 0x0004},+{0x001F4E, 0x0080},+{0x001F50, 0x0004},+{0x001F58, 0x0080},+{0x001F59, 0x0004},+{0x001F5A, 0x0080},+{0x001F5B, 0x0004},+{0x001F5C, 0x0080},+{0x001F5D, 0x0004},+{0x001F5E, 0x0080},+{0x001F5F, 0x0004},+{0x001F7E, 0x0080},+{0x001F80, 0x0004},+{0x001FB5, 0x0080},+{0x001FB6, 0x0004},+{0x001FBD, 0x0040},+{0x001FBE, 0x0004},+{0x001FBF, 0x0040},+{0x001FC2, 0x0004},+{0x001FC5, 0x0080},+{0x001FC6, 0x0004},+{0x001FCD, 0x0040},+{0x001FD0, 0x0004},+{0x001FD4, 0x0080},+{0x001FD6, 0x0004},+{0x001FDC, 0x0080},+{0x001FDD, 0x0040},+{0x001FE0, 0x0004},+{0x001FED, 0x0040},+{0x001FF0, 0x0080},+{0x001FF2, 0x0004},+{0x001FF5, 0x0080},+{0x001FF6, 0x0004},+{0x001FFD, 0x0040},+{0x001FFF, 0x0080},+{0x002000, 0x0008},+{0x00200B, 0x0080},+{0x002010, 0x0020},+{0x002028, 0x0008},+{0x00202A, 0x0080},+{0x00202F, 0x0008},+{0x002030, 0x0020},+{0x002044, 0x0040},+{0x002045, 0x0020},+{0x002052, 0x0040},+{0x002053, 0x0020},+{0x00205F, 0x0008},+{0x002060, 0x0080},+{0x002070, 0x0002},+{0x002071, 0x0004},+{0x002072, 0x0080},+{0x002074, 0x0002},+{0x00207A, 0x0040},+{0x00207D, 0x0020},+{0x00207F, 0x0004},+{0x002080, 0x0002},+{0x00208A, 0x0040},+{0x00208D, 0x0020},+{0x00208F, 0x0080},+{0x002090, 0x0004},+{0x00209D, 0x0080},+{0x0020A0, 0x0040},+{0x0020C1, 0x0080},+{0x0020D0, 0x0010},+{0x0020F1, 0x0080},+{0x002100, 0x0040},+{0x002102, 0x0004},+{0x002103, 0x0040},+{0x002107, 0x0004},+{0x002108, 0x0040},+{0x00210A, 0x0004},+{0x002114, 0x0040},+{0x002115, 0x0004},+{0x002116, 0x0040},+{0x002119, 0x0004},+{0x00211E, 0x0040},+{0x002124, 0x0004},+{0x002125, 0x0040},+{0x002126, 0x0004},+{0x002127, 0x0040},+{0x002128, 0x0004},+{0x002129, 0x0040},+{0x00212A, 0x0004},+{0x00212E, 0x0040},+{0x00212F, 0x0004},+{0x00213A, 0x0040},+{0x00213C, 0x0004},+{0x002140, 0x0040},+{0x002145, 0x0004},+{0x00214A, 0x0040},+{0x00214E, 0x0004},+{0x00214F, 0x0040},+{0x002150, 0x0002},+{0x002183, 0x0004},+{0x002185, 0x0002},+{0x00218A, 0x0040},+{0x00218C, 0x0080},+{0x002190, 0x0040},+{0x002308, 0x0020},+{0x00230C, 0x0040},+{0x002329, 0x0020},+{0x00232B, 0x0040},+{0x002427, 0x0080},+{0x002440, 0x0040},+{0x00244B, 0x0080},+{0x002460, 0x0002},+{0x00249C, 0x0040},+{0x0024EA, 0x0002},+{0x002500, 0x0040},+{0x002768, 0x0020},+{0x002776, 0x0002},+{0x002794, 0x0040},+{0x0027C5, 0x0020},+{0x0027C7, 0x0040},+{0x0027E6, 0x0020},+{0x0027F0, 0x0040},+{0x002983, 0x0020},+{0x002999, 0x0040},+{0x0029D8, 0x0020},+{0x0029DC, 0x0040},+{0x0029FC, 0x0020},+{0x0029FE, 0x0040},+{0x002B74, 0x0080},+{0x002B76, 0x0040},+{0x002B96, 0x0080},+{0x002B97, 0x0040},+{0x002C00, 0x0004},+{0x002CE5, 0x0040},+{0x002CEB, 0x0004},+{0x002CEF, 0x0010},+{0x002CF2, 0x0004},+{0x002CF4, 0x0080},+{0x002CF9, 0x0020},+{0x002CFD, 0x0002},+{0x002CFE, 0x0020},+{0x002D00, 0x0004},+{0x002D26, 0x0080},+{0x002D27, 0x0004},+{0x002D28, 0x0080},+{0x002D2D, 0x0004},+{0x002D2E, 0x0080},+{0x002D30, 0x0004},+{0x002D68, 0x0080},+{0x002D6F, 0x0004},+{0x002D70, 0x0020},+{0x002D71, 0x0080},+{0x002D7F, 0x0010},+{0x002D80, 0x0004},+{0x002D97, 0x0080},+{0x002DA0, 0x0004},+{0x002DA7, 0x0080},+{0x002DA8, 0x0004},+{0x002DAF, 0x0080},+{0x002DB0, 0x0004},+{0x002DB7, 0x0080},+{0x002DB8, 0x0004},+{0x002DBF, 0x0080},+{0x002DC0, 0x0004},+{0x002DC7, 0x0080},+{0x002DC8, 0x0004},+{0x002DCF, 0x0080},+{0x002DD0, 0x0004},+{0x002DD7, 0x0080},+{0x002DD8, 0x0004},+{0x002DDF, 0x0080},+{0x002DE0, 0x0010},+{0x002E00, 0x0020},+{0x002E2F, 0x0004},+{0x002E30, 0x0020},+{0x002E50, 0x0040},+{0x002E52, 0x0020},+{0x002E5E, 0x0080},+{0x002E80, 0x0040},+{0x002E9A, 0x0080},+{0x002E9B, 0x0040},+{0x002EF4, 0x0080},+{0x002F00, 0x0040},+{0x002FD6, 0x0080},+{0x002FF0, 0x0040},+{0x003000, 0x0008},+{0x003001, 0x0020},+{0x003004, 0x0040},+{0x003005, 0x0004},+{0x003007, 0x0002},+{0x003008, 0x0020},+{0x003012, 0x0040},+{0x003014, 0x0020},+{0x003020, 0x0040},+{0x003021, 0x0002},+{0x00302A, 0x0010},+{0x003030, 0x0020},+{0x003031, 0x0004},+{0x003036, 0x0040},+{0x003038, 0x0002},+{0x00303B, 0x0004},+{0x00303D, 0x0020},+{0x00303E, 0x0040},+{0x003040, 0x0080},+{0x003041, 0x0004},+{0x003097, 0x0080},+{0x003099, 0x0010},+{0x00309B, 0x0040},+{0x00309D, 0x0004},+{0x0030A0, 0x0020},+{0x0030A1, 0x0004},+{0x0030FB, 0x0020},+{0x0030FC, 0x0004},+{0x003100, 0x0080},+{0x003105, 0x0004},+{0x003130, 0x0080},+{0x003131, 0x0004},+{0x00318F, 0x0080},+{0x003190, 0x0040},+{0x003192, 0x0002},+{0x003196, 0x0040},+{0x0031A0, 0x0004},+{0x0031C0, 0x0040},+{0x0031E4, 0x0080},+{0x0031EF, 0x0040},+{0x0031F0, 0x0004},+{0x003200, 0x0040},+{0x00321F, 0x0080},+{0x003220, 0x0002},+{0x00322A, 0x0040},+{0x003248, 0x0002},+{0x003250, 0x0040},+{0x003251, 0x0002},+{0x003260, 0x0040},+{0x003280, 0x0002},+{0x00328A, 0x0040},+{0x0032B1, 0x0002},+{0x0032C0, 0x0040},+{0x003400, 0x0004},+{0x004DC0, 0x0040},+{0x004E00, 0x0004},+{0x00A48D, 0x0080},+{0x00A490, 0x0040},+{0x00A4C7, 0x0080},+{0x00A4D0, 0x0004},+{0x00A4FE, 0x0020},+{0x00A500, 0x0004},+{0x00A60D, 0x0020},+{0x00A610, 0x0004},+{0x00A620, 0x0002},+{0x00A62A, 0x0004},+{0x00A62C, 0x0080},+{0x00A640, 0x0004},+{0x00A66F, 0x0010},+{0x00A673, 0x0020},+{0x00A674, 0x0010},+{0x00A67E, 0x0020},+{0x00A67F, 0x0004},+{0x00A69E, 0x0010},+{0x00A6A0, 0x0004},+{0x00A6E6, 0x0002},+{0x00A6F0, 0x0010},+{0x00A6F2, 0x0020},+{0x00A6F8, 0x0080},+{0x00A700, 0x0040},+{0x00A717, 0x0004},+{0x00A720, 0x0040},+{0x00A722, 0x0004},+{0x00A789, 0x0040},+{0x00A78B, 0x0004},+{0x00A7CB, 0x0080},+{0x00A7D0, 0x0004},+{0x00A7D2, 0x0080},+{0x00A7D3, 0x0004},+{0x00A7D4, 0x0080},+{0x00A7D5, 0x0004},+{0x00A7DA, 0x0080},+{0x00A7F2, 0x0004},+{0x00A802, 0x0010},+{0x00A803, 0x0004},+{0x00A806, 0x0010},+{0x00A807, 0x0004},+{0x00A80B, 0x0010},+{0x00A80C, 0x0004},+{0x00A823, 0x0010},+{0x00A828, 0x0040},+{0x00A82C, 0x0010},+{0x00A82D, 0x0080},+{0x00A830, 0x0002},+{0x00A836, 0x0040},+{0x00A83A, 0x0080},+{0x00A840, 0x0004},+{0x00A874, 0x0020},+{0x00A878, 0x0080},+{0x00A880, 0x0010},+{0x00A882, 0x0004},+{0x00A8B4, 0x0010},+{0x00A8C6, 0x0080},+{0x00A8CE, 0x0020},+{0x00A8D0, 0x0002},+{0x00A8DA, 0x0080},+{0x00A8E0, 0x0010},+{0x00A8F2, 0x0004},+{0x00A8F8, 0x0020},+{0x00A8FB, 0x0004},+{0x00A8FC, 0x0020},+{0x00A8FD, 0x0004},+{0x00A8FF, 0x0010},+{0x00A900, 0x0002},+{0x00A90A, 0x0004},+{0x00A926, 0x0010},+{0x00A92E, 0x0020},+{0x00A930, 0x0004},+{0x00A947, 0x0010},+{0x00A954, 0x0080},+{0x00A95F, 0x0020},+{0x00A960, 0x0004},+{0x00A97D, 0x0080},+{0x00A980, 0x0010},+{0x00A984, 0x0004},+{0x00A9B3, 0x0010},+{0x00A9C1, 0x0020},+{0x00A9CE, 0x0080},+{0x00A9CF, 0x0004},+{0x00A9D0, 0x0002},+{0x00A9DA, 0x0080},+{0x00A9DE, 0x0020},+{0x00A9E0, 0x0004},+{0x00A9E5, 0x0010},+{0x00A9E6, 0x0004},+{0x00A9F0, 0x0002},+{0x00A9FA, 0x0004},+{0x00A9FF, 0x0080},+{0x00AA00, 0x0004},+{0x00AA29, 0x0010},+{0x00AA37, 0x0080},+{0x00AA40, 0x0004},+{0x00AA43, 0x0010},+{0x00AA44, 0x0004},+{0x00AA4C, 0x0010},+{0x00AA4E, 0x0080},+{0x00AA50, 0x0002},+{0x00AA5A, 0x0080},+{0x00AA5C, 0x0020},+{0x00AA60, 0x0004},+{0x00AA77, 0x0040},+{0x00AA7A, 0x0004},+{0x00AA7B, 0x0010},+{0x00AA7E, 0x0004},+{0x00AAB0, 0x0010},+{0x00AAB1, 0x0004},+{0x00AAB2, 0x0010},+{0x00AAB5, 0x0004},+{0x00AAB7, 0x0010},+{0x00AAB9, 0x0004},+{0x00AABE, 0x0010},+{0x00AAC0, 0x0004},+{0x00AAC1, 0x0010},+{0x00AAC2, 0x0004},+{0x00AAC3, 0x0080},+{0x00AADB, 0x0004},+{0x00AADE, 0x0020},+{0x00AAE0, 0x0004},+{0x00AAEB, 0x0010},+{0x00AAF0, 0x0020},+{0x00AAF2, 0x0004},+{0x00AAF5, 0x0010},+{0x00AAF7, 0x0080},+{0x00AB01, 0x0004},+{0x00AB07, 0x0080},+{0x00AB09, 0x0004},+{0x00AB0F, 0x0080},+{0x00AB11, 0x0004},+{0x00AB17, 0x0080},+{0x00AB20, 0x0004},+{0x00AB27, 0x0080},+{0x00AB28, 0x0004},+{0x00AB2F, 0x0080},+{0x00AB30, 0x0004},+{0x00AB5B, 0x0040},+{0x00AB5C, 0x0004},+{0x00AB6A, 0x0040},+{0x00AB6C, 0x0080},+{0x00AB70, 0x0004},+{0x00ABE3, 0x0010},+{0x00ABEB, 0x0020},+{0x00ABEC, 0x0010},+{0x00ABEE, 0x0080},+{0x00ABF0, 0x0002},+{0x00ABFA, 0x0080},+{0x00AC00, 0x0004},+{0x00D7A4, 0x0080},+{0x00D7B0, 0x0004},+{0x00D7C7, 0x0080},+{0x00D7CB, 0x0004},+{0x00D7FC, 0x0080},+{0x00F900, 0x0004},+{0x00FA6E, 0x0080},+{0x00FA70, 0x0004},+{0x00FADA, 0x0080},+{0x00FB00, 0x0004},+{0x00FB07, 0x0080},+{0x00FB13, 0x0004},+{0x00FB18, 0x0080},+{0x00FB1D, 0x0004},+{0x00FB1E, 0x0010},+{0x00FB1F, 0x0004},+{0x00FB29, 0x0040},+{0x00FB2A, 0x0004},+{0x00FB37, 0x0080},+{0x00FB38, 0x0004},+{0x00FB3D, 0x0080},+{0x00FB3E, 0x0004},+{0x00FB3F, 0x0080},+{0x00FB40, 0x0004},+{0x00FB42, 0x0080},+{0x00FB43, 0x0004},+{0x00FB45, 0x0080},+{0x00FB46, 0x0004},+{0x00FBB2, 0x0040},+{0x00FBC3, 0x0080},+{0x00FBD3, 0x0004},+{0x00FD3E, 0x0020},+{0x00FD40, 0x0040},+{0x00FD50, 0x0004},+{0x00FD90, 0x0080},+{0x00FD92, 0x0004},+{0x00FDC8, 0x0080},+{0x00FDCF, 0x0040},+{0x00FDD0, 0x0080},+{0x00FDF0, 0x0004},+{0x00FDFC, 0x0040},+{0x00FE00, 0x0010},+{0x00FE10, 0x0020},+{0x00FE1A, 0x0080},+{0x00FE20, 0x0010},+{0x00FE30, 0x0020},+{0x00FE53, 0x0080},+{0x00FE54, 0x0020},+{0x00FE62, 0x0040},+{0x00FE63, 0x0020},+{0x00FE64, 0x0040},+{0x00FE67, 0x0080},+{0x00FE68, 0x0020},+{0x00FE69, 0x0040},+{0x00FE6A, 0x0020},+{0x00FE6C, 0x0080},+{0x00FE70, 0x0004},+{0x00FE75, 0x0080},+{0x00FE76, 0x0004},+{0x00FEFD, 0x0080},+{0x00FF01, 0x0020},+{0x00FF04, 0x0040},+{0x00FF05, 0x0020},+{0x00FF0B, 0x0040},+{0x00FF0C, 0x0020},+{0x00FF10, 0x0002},+{0x00FF1A, 0x0020},+{0x00FF1C, 0x0040},+{0x00FF1F, 0x0020},+{0x00FF21, 0x0004},+{0x00FF3B, 0x0020},+{0x00FF3E, 0x0040},+{0x00FF3F, 0x0020},+{0x00FF40, 0x0040},+{0x00FF41, 0x0004},+{0x00FF5B, 0x0020},+{0x00FF5C, 0x0040},+{0x00FF5D, 0x0020},+{0x00FF5E, 0x0040},+{0x00FF5F, 0x0020},+{0x00FF66, 0x0004},+{0x00FFBF, 0x0080},+{0x00FFC2, 0x0004},+{0x00FFC8, 0x0080},+{0x00FFCA, 0x0004},+{0x00FFD0, 0x0080},+{0x00FFD2, 0x0004},+{0x00FFD8, 0x0080},+{0x00FFDA, 0x0004},+{0x00FFDD, 0x0080},+{0x00FFE0, 0x0040},+{0x00FFE7, 0x0080},+{0x00FFE8, 0x0040},+{0x00FFEF, 0x0080},+{0x00FFFC, 0x0040},+{0x00FFFE, 0x0080},+{0x010000, 0x0004},+{0x01000C, 0x0080},+{0x01000D, 0x0004},+{0x010027, 0x0080},+{0x010028, 0x0004},+{0x01003B, 0x0080},+{0x01003C, 0x0004},+{0x01003E, 0x0080},+{0x01003F, 0x0004},+{0x01004E, 0x0080},+{0x010050, 0x0004},+{0x01005E, 0x0080},+{0x010080, 0x0004},+{0x0100FB, 0x0080},+{0x010100, 0x0020},+{0x010103, 0x0080},+{0x010107, 0x0002},+{0x010134, 0x0080},+{0x010137, 0x0040},+{0x010140, 0x0002},+{0x010179, 0x0040},+{0x01018A, 0x0002},+{0x01018C, 0x0040},+{0x01018F, 0x0080},+{0x010190, 0x0040},+{0x01019D, 0x0080},+{0x0101A0, 0x0040},+{0x0101A1, 0x0080},+{0x0101D0, 0x0040},+{0x0101FD, 0x0010},+{0x0101FE, 0x0080},+{0x010280, 0x0004},+{0x01029D, 0x0080},+{0x0102A0, 0x0004},+{0x0102D1, 0x0080},+{0x0102E0, 0x0010},+{0x0102E1, 0x0002},+{0x0102FC, 0x0080},+{0x010300, 0x0004},+{0x010320, 0x0002},+{0x010324, 0x0080},+{0x01032D, 0x0004},+{0x010341, 0x0002},+{0x010342, 0x0004},+{0x01034A, 0x0002},+{0x01034B, 0x0080},+{0x010350, 0x0004},+{0x010376, 0x0010},+{0x01037B, 0x0080},+{0x010380, 0x0004},+{0x01039E, 0x0080},+{0x01039F, 0x0020},+{0x0103A0, 0x0004},+{0x0103C4, 0x0080},+{0x0103C8, 0x0004},+{0x0103D0, 0x0020},+{0x0103D1, 0x0002},+{0x0103D6, 0x0080},+{0x010400, 0x0004},+{0x01049E, 0x0080},+{0x0104A0, 0x0002},+{0x0104AA, 0x0080},+{0x0104B0, 0x0004},+{0x0104D4, 0x0080},+{0x0104D8, 0x0004},+{0x0104FC, 0x0080},+{0x010500, 0x0004},+{0x010528, 0x0080},+{0x010530, 0x0004},+{0x010564, 0x0080},+{0x01056F, 0x0020},+{0x010570, 0x0004},+{0x01057B, 0x0080},+{0x01057C, 0x0004},+{0x01058B, 0x0080},+{0x01058C, 0x0004},+{0x010593, 0x0080},+{0x010594, 0x0004},+{0x010596, 0x0080},+{0x010597, 0x0004},+{0x0105A2, 0x0080},+{0x0105A3, 0x0004},+{0x0105B2, 0x0080},+{0x0105B3, 0x0004},+{0x0105BA, 0x0080},+{0x0105BB, 0x0004},+{0x0105BD, 0x0080},+{0x010600, 0x0004},+{0x010737, 0x0080},+{0x010740, 0x0004},+{0x010756, 0x0080},+{0x010760, 0x0004},+{0x010768, 0x0080},+{0x010780, 0x0004},+{0x010786, 0x0080},+{0x010787, 0x0004},+{0x0107B1, 0x0080},+{0x0107B2, 0x0004},+{0x0107BB, 0x0080},+{0x010800, 0x0004},+{0x010806, 0x0080},+{0x010808, 0x0004},+{0x010809, 0x0080},+{0x01080A, 0x0004},+{0x010836, 0x0080},+{0x010837, 0x0004},+{0x010839, 0x0080},+{0x01083C, 0x0004},+{0x01083D, 0x0080},+{0x01083F, 0x0004},+{0x010856, 0x0080},+{0x010857, 0x0020},+{0x010858, 0x0002},+{0x010860, 0x0004},+{0x010877, 0x0040},+{0x010879, 0x0002},+{0x010880, 0x0004},+{0x01089F, 0x0080},+{0x0108A7, 0x0002},+{0x0108B0, 0x0080},+{0x0108E0, 0x0004},+{0x0108F3, 0x0080},+{0x0108F4, 0x0004},+{0x0108F6, 0x0080},+{0x0108FB, 0x0002},+{0x010900, 0x0004},+{0x010916, 0x0002},+{0x01091C, 0x0080},+{0x01091F, 0x0020},+{0x010920, 0x0004},+{0x01093A, 0x0080},+{0x01093F, 0x0020},+{0x010940, 0x0080},+{0x010980, 0x0004},+{0x0109B8, 0x0080},+{0x0109BC, 0x0002},+{0x0109BE, 0x0004},+{0x0109C0, 0x0002},+{0x0109D0, 0x0080},+{0x0109D2, 0x0002},+{0x010A00, 0x0004},+{0x010A01, 0x0010},+{0x010A04, 0x0080},+{0x010A05, 0x0010},+{0x010A07, 0x0080},+{0x010A0C, 0x0010},+{0x010A10, 0x0004},+{0x010A14, 0x0080},+{0x010A15, 0x0004},+{0x010A18, 0x0080},+{0x010A19, 0x0004},+{0x010A36, 0x0080},+{0x010A38, 0x0010},+{0x010A3B, 0x0080},+{0x010A3F, 0x0010},+{0x010A40, 0x0002},+{0x010A49, 0x0080},+{0x010A50, 0x0020},+{0x010A59, 0x0080},+{0x010A60, 0x0004},+{0x010A7D, 0x0002},+{0x010A7F, 0x0020},+{0x010A80, 0x0004},+{0x010A9D, 0x0002},+{0x010AA0, 0x0080},+{0x010AC0, 0x0004},+{0x010AC8, 0x0040},+{0x010AC9, 0x0004},+{0x010AE5, 0x0010},+{0x010AE7, 0x0080},+{0x010AEB, 0x0002},+{0x010AF0, 0x0020},+{0x010AF7, 0x0080},+{0x010B00, 0x0004},+{0x010B36, 0x0080},+{0x010B39, 0x0020},+{0x010B40, 0x0004},+{0x010B56, 0x0080},+{0x010B58, 0x0002},+{0x010B60, 0x0004},+{0x010B73, 0x0080},+{0x010B78, 0x0002},+{0x010B80, 0x0004},+{0x010B92, 0x0080},+{0x010B99, 0x0020},+{0x010B9D, 0x0080},+{0x010BA9, 0x0002},+{0x010BB0, 0x0080},+{0x010C00, 0x0004},+{0x010C49, 0x0080},+{0x010C80, 0x0004},+{0x010CB3, 0x0080},+{0x010CC0, 0x0004},+{0x010CF3, 0x0080},+{0x010CFA, 0x0002},+{0x010D00, 0x0004},+{0x010D24, 0x0010},+{0x010D28, 0x0080},+{0x010D30, 0x0002},+{0x010D3A, 0x0080},+{0x010E60, 0x0002},+{0x010E7F, 0x0080},+{0x010E80, 0x0004},+{0x010EAA, 0x0080},+{0x010EAB, 0x0010},+{0x010EAD, 0x0020},+{0x010EAE, 0x0080},+{0x010EB0, 0x0004},+{0x010EB2, 0x0080},+{0x010EFD, 0x0010},+{0x010F00, 0x0004},+{0x010F1D, 0x0002},+{0x010F27, 0x0004},+{0x010F28, 0x0080},+{0x010F30, 0x0004},+{0x010F46, 0x0010},+{0x010F51, 0x0002},+{0x010F55, 0x0020},+{0x010F5A, 0x0080},+{0x010F70, 0x0004},+{0x010F82, 0x0010},+{0x010F86, 0x0020},+{0x010F8A, 0x0080},+{0x010FB0, 0x0004},+{0x010FC5, 0x0002},+{0x010FCC, 0x0080},+{0x010FE0, 0x0004},+{0x010FF7, 0x0080},+{0x011000, 0x0010},+{0x011003, 0x0004},+{0x011038, 0x0010},+{0x011047, 0x0020},+{0x01104E, 0x0080},+{0x011052, 0x0002},+{0x011070, 0x0010},+{0x011071, 0x0004},+{0x011073, 0x0010},+{0x011075, 0x0004},+{0x011076, 0x0080},+{0x01107F, 0x0010},+{0x011083, 0x0004},+{0x0110B0, 0x0010},+{0x0110BB, 0x0020},+{0x0110BD, 0x0080},+{0x0110BE, 0x0020},+{0x0110C2, 0x0010},+{0x0110C3, 0x0080},+{0x0110D0, 0x0004},+{0x0110E9, 0x0080},+{0x0110F0, 0x0002},+{0x0110FA, 0x0080},+{0x011100, 0x0010},+{0x011103, 0x0004},+{0x011127, 0x0010},+{0x011135, 0x0080},+{0x011136, 0x0002},+{0x011140, 0x0020},+{0x011144, 0x0004},+{0x011145, 0x0010},+{0x011147, 0x0004},+{0x011148, 0x0080},+{0x011150, 0x0004},+{0x011173, 0x0010},+{0x011174, 0x0020},+{0x011176, 0x0004},+{0x011177, 0x0080},+{0x011180, 0x0010},+{0x011183, 0x0004},+{0x0111B3, 0x0010},+{0x0111C1, 0x0004},+{0x0111C5, 0x0020},+{0x0111C9, 0x0010},+{0x0111CD, 0x0020},+{0x0111CE, 0x0010},+{0x0111D0, 0x0002},+{0x0111DA, 0x0004},+{0x0111DB, 0x0020},+{0x0111DC, 0x0004},+{0x0111DD, 0x0020},+{0x0111E0, 0x0080},+{0x0111E1, 0x0002},+{0x0111F5, 0x0080},+{0x011200, 0x0004},+{0x011212, 0x0080},+{0x011213, 0x0004},+{0x01122C, 0x0010},+{0x011238, 0x0020},+{0x01123E, 0x0010},+{0x01123F, 0x0004},+{0x011241, 0x0010},+{0x011242, 0x0080},+{0x011280, 0x0004},+{0x011287, 0x0080},+{0x011288, 0x0004},+{0x011289, 0x0080},+{0x01128A, 0x0004},+{0x01128E, 0x0080},+{0x01128F, 0x0004},+{0x01129E, 0x0080},+{0x01129F, 0x0004},+{0x0112A9, 0x0020},+{0x0112AA, 0x0080},+{0x0112B0, 0x0004},+{0x0112DF, 0x0010},+{0x0112EB, 0x0080},+{0x0112F0, 0x0002},+{0x0112FA, 0x0080},+{0x011300, 0x0010},+{0x011304, 0x0080},+{0x011305, 0x0004},+{0x01130D, 0x0080},+{0x01130F, 0x0004},+{0x011311, 0x0080},+{0x011313, 0x0004},+{0x011329, 0x0080},+{0x01132A, 0x0004},+{0x011331, 0x0080},+{0x011332, 0x0004},+{0x011334, 0x0080},+{0x011335, 0x0004},+{0x01133A, 0x0080},+{0x01133B, 0x0010},+{0x01133D, 0x0004},+{0x01133E, 0x0010},+{0x011345, 0x0080},+{0x011347, 0x0010},+{0x011349, 0x0080},+{0x01134B, 0x0010},+{0x01134E, 0x0080},+{0x011350, 0x0004},+{0x011351, 0x0080},+{0x011357, 0x0010},+{0x011358, 0x0080},+{0x01135D, 0x0004},+{0x011362, 0x0010},+{0x011364, 0x0080},+{0x011366, 0x0010},+{0x01136D, 0x0080},+{0x011370, 0x0010},+{0x011375, 0x0080},+{0x011400, 0x0004},+{0x011435, 0x0010},+{0x011447, 0x0004},+{0x01144B, 0x0020},+{0x011450, 0x0002},+{0x01145A, 0x0020},+{0x01145C, 0x0080},+{0x01145D, 0x0020},+{0x01145E, 0x0010},+{0x01145F, 0x0004},+{0x011462, 0x0080},+{0x011480, 0x0004},+{0x0114B0, 0x0010},+{0x0114C4, 0x0004},+{0x0114C6, 0x0020},+{0x0114C7, 0x0004},+{0x0114C8, 0x0080},+{0x0114D0, 0x0002},+{0x0114DA, 0x0080},+{0x011580, 0x0004},+{0x0115AF, 0x0010},+{0x0115B6, 0x0080},+{0x0115B8, 0x0010},+{0x0115C1, 0x0020},+{0x0115D8, 0x0004},+{0x0115DC, 0x0010},+{0x0115DE, 0x0080},+{0x011600, 0x0004},+{0x011630, 0x0010},+{0x011641, 0x0020},+{0x011644, 0x0004},+{0x011645, 0x0080},+{0x011650, 0x0002},+{0x01165A, 0x0080},+{0x011660, 0x0020},+{0x01166D, 0x0080},+{0x011680, 0x0004},+{0x0116AB, 0x0010},+{0x0116B8, 0x0004},+{0x0116B9, 0x0020},+{0x0116BA, 0x0080},+{0x0116C0, 0x0002},+{0x0116CA, 0x0080},+{0x011700, 0x0004},+{0x01171B, 0x0080},+{0x01171D, 0x0010},+{0x01172C, 0x0080},+{0x011730, 0x0002},+{0x01173C, 0x0020},+{0x01173F, 0x0040},+{0x011740, 0x0004},+{0x011747, 0x0080},+{0x011800, 0x0004},+{0x01182C, 0x0010},+{0x01183B, 0x0020},+{0x01183C, 0x0080},+{0x0118A0, 0x0004},+{0x0118E0, 0x0002},+{0x0118F3, 0x0080},+{0x0118FF, 0x0004},+{0x011907, 0x0080},+{0x011909, 0x0004},+{0x01190A, 0x0080},+{0x01190C, 0x0004},+{0x011914, 0x0080},+{0x011915, 0x0004},+{0x011917, 0x0080},+{0x011918, 0x0004},+{0x011930, 0x0010},+{0x011936, 0x0080},+{0x011937, 0x0010},+{0x011939, 0x0080},+{0x01193B, 0x0010},+{0x01193F, 0x0004},+{0x011940, 0x0010},+{0x011941, 0x0004},+{0x011942, 0x0010},+{0x011944, 0x0020},+{0x011947, 0x0080},+{0x011950, 0x0002},+{0x01195A, 0x0080},+{0x0119A0, 0x0004},+{0x0119A8, 0x0080},+{0x0119AA, 0x0004},+{0x0119D1, 0x0010},+{0x0119D8, 0x0080},+{0x0119DA, 0x0010},+{0x0119E1, 0x0004},+{0x0119E2, 0x0020},+{0x0119E3, 0x0004},+{0x0119E4, 0x0010},+{0x0119E5, 0x0080},+{0x011A00, 0x0004},+{0x011A01, 0x0010},+{0x011A0B, 0x0004},+{0x011A33, 0x0010},+{0x011A3A, 0x0004},+{0x011A3B, 0x0010},+{0x011A3F, 0x0020},+{0x011A47, 0x0010},+{0x011A48, 0x0080},+{0x011A50, 0x0004},+{0x011A51, 0x0010},+{0x011A5C, 0x0004},+{0x011A8A, 0x0010},+{0x011A9A, 0x0020},+{0x011A9D, 0x0004},+{0x011A9E, 0x0020},+{0x011AA3, 0x0080},+{0x011AB0, 0x0004},+{0x011AF9, 0x0080},+{0x011B00, 0x0020},+{0x011B0A, 0x0080},+{0x011C00, 0x0004},+{0x011C09, 0x0080},+{0x011C0A, 0x0004},+{0x011C2F, 0x0010},+{0x011C37, 0x0080},+{0x011C38, 0x0010},+{0x011C40, 0x0004},+{0x011C41, 0x0020},+{0x011C46, 0x0080},+{0x011C50, 0x0002},+{0x011C6D, 0x0080},+{0x011C70, 0x0020},+{0x011C72, 0x0004},+{0x011C90, 0x0080},+{0x011C92, 0x0010},+{0x011CA8, 0x0080},+{0x011CA9, 0x0010},+{0x011CB7, 0x0080},+{0x011D00, 0x0004},+{0x011D07, 0x0080},+{0x011D08, 0x0004},+{0x011D0A, 0x0080},+{0x011D0B, 0x0004},+{0x011D31, 0x0010},+{0x011D37, 0x0080},+{0x011D3A, 0x0010},+{0x011D3B, 0x0080},+{0x011D3C, 0x0010},+{0x011D3E, 0x0080},+{0x011D3F, 0x0010},+{0x011D46, 0x0004},+{0x011D47, 0x0010},+{0x011D48, 0x0080},+{0x011D50, 0x0002},+{0x011D5A, 0x0080},+{0x011D60, 0x0004},+{0x011D66, 0x0080},+{0x011D67, 0x0004},+{0x011D69, 0x0080},+{0x011D6A, 0x0004},+{0x011D8A, 0x0010},+{0x011D8F, 0x0080},+{0x011D90, 0x0010},+{0x011D92, 0x0080},+{0x011D93, 0x0010},+{0x011D98, 0x0004},+{0x011D99, 0x0080},+{0x011DA0, 0x0002},+{0x011DAA, 0x0080},+{0x011EE0, 0x0004},+{0x011EF3, 0x0010},+{0x011EF7, 0x0020},+{0x011EF9, 0x0080},+{0x011F00, 0x0010},+{0x011F02, 0x0004},+{0x011F03, 0x0010},+{0x011F04, 0x0004},+{0x011F11, 0x0080},+{0x011F12, 0x0004},+{0x011F34, 0x0010},+{0x011F3B, 0x0080},+{0x011F3E, 0x0010},+{0x011F43, 0x0020},+{0x011F50, 0x0002},+{0x011F5A, 0x0080},+{0x011FB0, 0x0004},+{0x011FB1, 0x0080},+{0x011FC0, 0x0002},+{0x011FD5, 0x0040},+{0x011FF2, 0x0080},+{0x011FFF, 0x0020},+{0x012000, 0x0004},+{0x01239A, 0x0080},+{0x012400, 0x0002},+{0x01246F, 0x0080},+{0x012470, 0x0020},+{0x012475, 0x0080},+{0x012480, 0x0004},+{0x012544, 0x0080},+{0x012F90, 0x0004},+{0x012FF1, 0x0020},+{0x012FF3, 0x0080},+{0x013000, 0x0004},+{0x013430, 0x0080},+{0x013440, 0x0010},+{0x013441, 0x0004},+{0x013447, 0x0010},+{0x013456, 0x0080},+{0x014400, 0x0004},+{0x014647, 0x0080},+{0x016800, 0x0004},+{0x016A39, 0x0080},+{0x016A40, 0x0004},+{0x016A5F, 0x0080},+{0x016A60, 0x0002},+{0x016A6A, 0x0080},+{0x016A6E, 0x0020},+{0x016A70, 0x0004},+{0x016ABF, 0x0080},+{0x016AC0, 0x0002},+{0x016ACA, 0x0080},+{0x016AD0, 0x0004},+{0x016AEE, 0x0080},+{0x016AF0, 0x0010},+{0x016AF5, 0x0020},+{0x016AF6, 0x0080},+{0x016B00, 0x0004},+{0x016B30, 0x0010},+{0x016B37, 0x0020},+{0x016B3C, 0x0040},+{0x016B40, 0x0004},+{0x016B44, 0x0020},+{0x016B45, 0x0040},+{0x016B46, 0x0080},+{0x016B50, 0x0002},+{0x016B5A, 0x0080},+{0x016B5B, 0x0002},+{0x016B62, 0x0080},+{0x016B63, 0x0004},+{0x016B78, 0x0080},+{0x016B7D, 0x0004},+{0x016B90, 0x0080},+{0x016E40, 0x0004},+{0x016E80, 0x0002},+{0x016E97, 0x0020},+{0x016E9B, 0x0080},+{0x016F00, 0x0004},+{0x016F4B, 0x0080},+{0x016F4F, 0x0010},+{0x016F50, 0x0004},+{0x016F51, 0x0010},+{0x016F88, 0x0080},+{0x016F8F, 0x0010},+{0x016F93, 0x0004},+{0x016FA0, 0x0080},+{0x016FE0, 0x0004},+{0x016FE2, 0x0020},+{0x016FE3, 0x0004},+{0x016FE4, 0x0010},+{0x016FE5, 0x0080},+{0x016FF0, 0x0010},+{0x016FF2, 0x0080},+{0x017000, 0x0004},+{0x0187F8, 0x0080},+{0x018800, 0x0004},+{0x018CD6, 0x0080},+{0x018D00, 0x0004},+{0x018D09, 0x0080},+{0x01AFF0, 0x0004},+{0x01AFF4, 0x0080},+{0x01AFF5, 0x0004},+{0x01AFFC, 0x0080},+{0x01AFFD, 0x0004},+{0x01AFFF, 0x0080},+{0x01B000, 0x0004},+{0x01B123, 0x0080},+{0x01B132, 0x0004},+{0x01B133, 0x0080},+{0x01B150, 0x0004},+{0x01B153, 0x0080},+{0x01B155, 0x0004},+{0x01B156, 0x0080},+{0x01B164, 0x0004},+{0x01B168, 0x0080},+{0x01B170, 0x0004},+{0x01B2FC, 0x0080},+{0x01BC00, 0x0004},+{0x01BC6B, 0x0080},+{0x01BC70, 0x0004},+{0x01BC7D, 0x0080},+{0x01BC80, 0x0004},+{0x01BC89, 0x0080},+{0x01BC90, 0x0004},+{0x01BC9A, 0x0080},+{0x01BC9C, 0x0040},+{0x01BC9D, 0x0010},+{0x01BC9F, 0x0020},+{0x01BCA0, 0x0080},+{0x01CF00, 0x0010},+{0x01CF2E, 0x0080},+{0x01CF30, 0x0010},+{0x01CF47, 0x0080},+{0x01CF50, 0x0040},+{0x01CFC4, 0x0080},+{0x01D000, 0x0040},+{0x01D0F6, 0x0080},+{0x01D100, 0x0040},+{0x01D127, 0x0080},+{0x01D129, 0x0040},+{0x01D165, 0x0010},+{0x01D16A, 0x0040},+{0x01D16D, 0x0010},+{0x01D173, 0x0080},+{0x01D17B, 0x0010},+{0x01D183, 0x0040},+{0x01D185, 0x0010},+{0x01D18C, 0x0040},+{0x01D1AA, 0x0010},+{0x01D1AE, 0x0040},+{0x01D1EB, 0x0080},+{0x01D200, 0x0040},+{0x01D242, 0x0010},+{0x01D245, 0x0040},+{0x01D246, 0x0080},+{0x01D2C0, 0x0002},+{0x01D2D4, 0x0080},+{0x01D2E0, 0x0002},+{0x01D2F4, 0x0080},+{0x01D300, 0x0040},+{0x01D357, 0x0080},+{0x01D360, 0x0002},+{0x01D379, 0x0080},+{0x01D400, 0x0004},+{0x01D455, 0x0080},+{0x01D456, 0x0004},+{0x01D49D, 0x0080},+{0x01D49E, 0x0004},+{0x01D4A0, 0x0080},+{0x01D4A2, 0x0004},+{0x01D4A3, 0x0080},+{0x01D4A5, 0x0004},+{0x01D4A7, 0x0080},+{0x01D4A9, 0x0004},+{0x01D4AD, 0x0080},+{0x01D4AE, 0x0004},+{0x01D4BA, 0x0080},+{0x01D4BB, 0x0004},+{0x01D4BC, 0x0080},+{0x01D4BD, 0x0004},+{0x01D4C4, 0x0080},+{0x01D4C5, 0x0004},+{0x01D506, 0x0080},+{0x01D507, 0x0004},+{0x01D50B, 0x0080},+{0x01D50D, 0x0004},+{0x01D515, 0x0080},+{0x01D516, 0x0004},+{0x01D51D, 0x0080},+{0x01D51E, 0x0004},+{0x01D53A, 0x0080},+{0x01D53B, 0x0004},+{0x01D53F, 0x0080},+{0x01D540, 0x0004},+{0x01D545, 0x0080},+{0x01D546, 0x0004},+{0x01D547, 0x0080},+{0x01D54A, 0x0004},+{0x01D551, 0x0080},+{0x01D552, 0x0004},+{0x01D6A6, 0x0080},+{0x01D6A8, 0x0004},+{0x01D6C1, 0x0040},+{0x01D6C2, 0x0004},+{0x01D6DB, 0x0040},+{0x01D6DC, 0x0004},+{0x01D6FB, 0x0040},+{0x01D6FC, 0x0004},+{0x01D715, 0x0040},+{0x01D716, 0x0004},+{0x01D735, 0x0040},+{0x01D736, 0x0004},+{0x01D74F, 0x0040},+{0x01D750, 0x0004},+{0x01D76F, 0x0040},+{0x01D770, 0x0004},+{0x01D789, 0x0040},+{0x01D78A, 0x0004},+{0x01D7A9, 0x0040},+{0x01D7AA, 0x0004},+{0x01D7C3, 0x0040},+{0x01D7C4, 0x0004},+{0x01D7CC, 0x0080},+{0x01D7CE, 0x0002},+{0x01D800, 0x0040},+{0x01DA00, 0x0010},+{0x01DA37, 0x0040},+{0x01DA3B, 0x0010},+{0x01DA6D, 0x0040},+{0x01DA75, 0x0010},+{0x01DA76, 0x0040},+{0x01DA84, 0x0010},+{0x01DA85, 0x0040},+{0x01DA87, 0x0020},+{0x01DA8C, 0x0080},+{0x01DA9B, 0x0010},+{0x01DAA0, 0x0080},+{0x01DAA1, 0x0010},+{0x01DAB0, 0x0080},+{0x01DF00, 0x0004},+{0x01DF1F, 0x0080},+{0x01DF25, 0x0004},+{0x01DF2B, 0x0080},+{0x01E000, 0x0010},+{0x01E007, 0x0080},+{0x01E008, 0x0010},+{0x01E019, 0x0080},+{0x01E01B, 0x0010},+{0x01E022, 0x0080},+{0x01E023, 0x0010},+{0x01E025, 0x0080},+{0x01E026, 0x0010},+{0x01E02B, 0x0080},+{0x01E030, 0x0004},+{0x01E06E, 0x0080},+{0x01E08F, 0x0010},+{0x01E090, 0x0080},+{0x01E100, 0x0004},+{0x01E12D, 0x0080},+{0x01E130, 0x0010},+{0x01E137, 0x0004},+{0x01E13E, 0x0080},+{0x01E140, 0x0002},+{0x01E14A, 0x0080},+{0x01E14E, 0x0004},+{0x01E14F, 0x0040},+{0x01E150, 0x0080},+{0x01E290, 0x0004},+{0x01E2AE, 0x0010},+{0x01E2AF, 0x0080},+{0x01E2C0, 0x0004},+{0x01E2EC, 0x0010},+{0x01E2F0, 0x0002},+{0x01E2FA, 0x0080},+{0x01E2FF, 0x0040},+{0x01E300, 0x0080},+{0x01E4D0, 0x0004},+{0x01E4EC, 0x0010},+{0x01E4F0, 0x0002},+{0x01E4FA, 0x0080},+{0x01E7E0, 0x0004},+{0x01E7E7, 0x0080},+{0x01E7E8, 0x0004},+{0x01E7EC, 0x0080},+{0x01E7ED, 0x0004},+{0x01E7EF, 0x0080},+{0x01E7F0, 0x0004},+{0x01E7FF, 0x0080},+{0x01E800, 0x0004},+{0x01E8C5, 0x0080},+{0x01E8C7, 0x0002},+{0x01E8D0, 0x0010},+{0x01E8D7, 0x0080},+{0x01E900, 0x0004},+{0x01E944, 0x0010},+{0x01E94B, 0x0004},+{0x01E94C, 0x0080},+{0x01E950, 0x0002},+{0x01E95A, 0x0080},+{0x01E95E, 0x0020},+{0x01E960, 0x0080},+{0x01EC71, 0x0002},+{0x01ECAC, 0x0040},+{0x01ECAD, 0x0002},+{0x01ECB0, 0x0040},+{0x01ECB1, 0x0002},+{0x01ECB5, 0x0080},+{0x01ED01, 0x0002},+{0x01ED2E, 0x0040},+{0x01ED2F, 0x0002},+{0x01ED3E, 0x0080},+{0x01EE00, 0x0004},+{0x01EE04, 0x0080},+{0x01EE05, 0x0004},+{0x01EE20, 0x0080},+{0x01EE21, 0x0004},+{0x01EE23, 0x0080},+{0x01EE24, 0x0004},+{0x01EE25, 0x0080},+{0x01EE27, 0x0004},+{0x01EE28, 0x0080},+{0x01EE29, 0x0004},+{0x01EE33, 0x0080},+{0x01EE34, 0x0004},+{0x01EE38, 0x0080},+{0x01EE39, 0x0004},+{0x01EE3A, 0x0080},+{0x01EE3B, 0x0004},+{0x01EE3C, 0x0080},+{0x01EE42, 0x0004},+{0x01EE43, 0x0080},+{0x01EE47, 0x0004},+{0x01EE48, 0x0080},+{0x01EE49, 0x0004},+{0x01EE4A, 0x0080},+{0x01EE4B, 0x0004},+{0x01EE4C, 0x0080},+{0x01EE4D, 0x0004},+{0x01EE50, 0x0080},+{0x01EE51, 0x0004},+{0x01EE53, 0x0080},+{0x01EE54, 0x0004},+{0x01EE55, 0x0080},+{0x01EE57, 0x0004},+{0x01EE58, 0x0080},+{0x01EE59, 0x0004},+{0x01EE5A, 0x0080},+{0x01EE5B, 0x0004},+{0x01EE5C, 0x0080},+{0x01EE5D, 0x0004},+{0x01EE5E, 0x0080},+{0x01EE5F, 0x0004},+{0x01EE60, 0x0080},+{0x01EE61, 0x0004},+{0x01EE63, 0x0080},+{0x01EE64, 0x0004},+{0x01EE65, 0x0080},+{0x01EE67, 0x0004},+{0x01EE6B, 0x0080},+{0x01EE6C, 0x0004},+{0x01EE73, 0x0080},+{0x01EE74, 0x0004},+{0x01EE78, 0x0080},+{0x01EE79, 0x0004},+{0x01EE7D, 0x0080},+{0x01EE7E, 0x0004},+{0x01EE7F, 0x0080},+{0x01EE80, 0x0004},+{0x01EE8A, 0x0080},+{0x01EE8B, 0x0004},+{0x01EE9C, 0x0080},+{0x01EEA1, 0x0004},+{0x01EEA4, 0x0080},+{0x01EEA5, 0x0004},+{0x01EEAA, 0x0080},+{0x01EEAB, 0x0004},+{0x01EEBC, 0x0080},+{0x01EEF0, 0x0040},+{0x01EEF2, 0x0080},+{0x01F000, 0x0040},+{0x01F02C, 0x0080},+{0x01F030, 0x0040},+{0x01F094, 0x0080},+{0x01F0A0, 0x0040},+{0x01F0AF, 0x0080},+{0x01F0B1, 0x0040},+{0x01F0C0, 0x0080},+{0x01F0C1, 0x0040},+{0x01F0D0, 0x0080},+{0x01F0D1, 0x0040},+{0x01F0F6, 0x0080},+{0x01F100, 0x0002},+{0x01F10D, 0x0040},+{0x01F1AE, 0x0080},+{0x01F1E6, 0x0040},+{0x01F203, 0x0080},+{0x01F210, 0x0040},+{0x01F23C, 0x0080},+{0x01F240, 0x0040},+{0x01F249, 0x0080},+{0x01F250, 0x0040},+{0x01F252, 0x0080},+{0x01F260, 0x0040},+{0x01F266, 0x0080},+{0x01F300, 0x0040},+{0x01F6D8, 0x0080},+{0x01F6DC, 0x0040},+{0x01F6ED, 0x0080},+{0x01F6F0, 0x0040},+{0x01F6FD, 0x0080},+{0x01F700, 0x0040},+{0x01F777, 0x0080},+{0x01F77B, 0x0040},+{0x01F7DA, 0x0080},+{0x01F7E0, 0x0040},+{0x01F7EC, 0x0080},+{0x01F7F0, 0x0040},+{0x01F7F1, 0x0080},+{0x01F800, 0x0040},+{0x01F80C, 0x0080},+{0x01F810, 0x0040},+{0x01F848, 0x0080},+{0x01F850, 0x0040},+{0x01F85A, 0x0080},+{0x01F860, 0x0040},+{0x01F888, 0x0080},+{0x01F890, 0x0040},+{0x01F8AE, 0x0080},+{0x01F8B0, 0x0040},+{0x01F8B2, 0x0080},+{0x01F900, 0x0040},+{0x01FA54, 0x0080},+{0x01FA60, 0x0040},+{0x01FA6E, 0x0080},+{0x01FA70, 0x0040},+{0x01FA7D, 0x0080},+{0x01FA80, 0x0040},+{0x01FA89, 0x0080},+{0x01FA90, 0x0040},+{0x01FABE, 0x0080},+{0x01FABF, 0x0040},+{0x01FAC6, 0x0080},+{0x01FACE, 0x0040},+{0x01FADC, 0x0080},+{0x01FAE0, 0x0040},+{0x01FAE9, 0x0080},+{0x01FAF0, 0x0040},+{0x01FAF9, 0x0080},+{0x01FB00, 0x0040},+{0x01FB93, 0x0080},+{0x01FB94, 0x0040},+{0x01FBCB, 0x0080},+{0x01FBF0, 0x0002},+{0x01FBFA, 0x0080},+{0x020000, 0x0004},+{0x02A6E0, 0x0080},+{0x02A700, 0x0004},+{0x02B73A, 0x0080},+{0x02B740, 0x0004},+{0x02B81E, 0x0080},+{0x02B820, 0x0004},+{0x02CEA2, 0x0080},+{0x02CEB0, 0x0004},+{0x02EBE1, 0x0080},+{0x02EBF0, 0x0004},+{0x02EE5E, 0x0080},+{0x02F800, 0x0004},+{0x02FA1E, 0x0080},+{0x030000, 0x0004},+{0x03134B, 0x0080},+{0x031350, 0x0004},+{0x0323B0, 0x0080},+{0x0E0100, 0x0010},+{0x0E01F0, 0x0080},+{0x110000, 0x0000},};-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation = {-{0x00000021, 0x00000023}, {0x00000025, 0x0000002A}, {0x0000002C, 0x0000002F}, {0x0000003A, 0x0000003B},-{0x0000003F, 0x00000040}, {0x0000005B, 0x0000005D}, {0x0000005F, 0x0000005F}, {0x0000007B, 0x0000007B},-{0x0000007D, 0x0000007D}, {0x000000A1, 0x000000A1}, {0x000000A7, 0x000000A7}, {0x000000AB, 0x000000AB},-{0x000000B6, 0x000000B7}, {0x000000BB, 0x000000BB}, {0x000000BF, 0x000000BF}, {0x0000037E, 0x0000037E},-{0x00000387, 0x00000387}, {0x0000055A, 0x0000055F}, {0x00000589, 0x0000058A}, {0x000005BE, 0x000005BE},-{0x000005C0, 0x000005C0}, {0x000005C3, 0x000005C3}, {0x000005C6, 0x000005C6}, {0x000005F3, 0x000005F4},-{0x00000609, 0x0000060A}, {0x0000060C, 0x0000060D}, {0x0000061B, 0x0000061B}, {0x0000061D, 0x0000061F},-{0x0000066A, 0x0000066D}, {0x000006D4, 0x000006D4}, {0x00000700, 0x0000070D}, {0x000007F7, 0x000007F9},-{0x00000830, 0x0000083E}, {0x0000085E, 0x0000085E}, {0x00000964, 0x00000965}, {0x00000970, 0x00000970},-{0x000009FD, 0x000009FD}, {0x00000A76, 0x00000A76}, {0x00000AF0, 0x00000AF0}, {0x00000C77, 0x00000C77},-{0x00000C84, 0x00000C84}, {0x00000DF4, 0x00000DF4}, {0x00000E4F, 0x00000E4F}, {0x00000E5A, 0x00000E5B},-{0x00000F04, 0x00000F12}, {0x00000F14, 0x00000F14}, {0x00000F3A, 0x00000F3D}, {0x00000F85, 0x00000F85},-{0x00000FD0, 0x00000FD4}, {0x00000FD9, 0x00000FDA}, {0x0000104A, 0x0000104F}, {0x000010FB, 0x000010FB},-{0x00001360, 0x00001368}, {0x00001400, 0x00001400}, {0x0000166E, 0x0000166E}, {0x0000169B, 0x0000169C},-{0x000016EB, 0x000016ED}, {0x00001735, 0x00001736}, {0x000017D4, 0x000017D6}, {0x000017D8, 0x000017DA},-{0x00001800, 0x0000180A}, {0x00001944, 0x00001945}, {0x00001A1E, 0x00001A1F}, {0x00001AA0, 0x00001AA6},-{0x00001AA8, 0x00001AAD}, {0x00001B5A, 0x00001B60}, {0x00001B7D, 0x00001B7E}, {0x00001BFC, 0x00001BFF},-{0x00001C3B, 0x00001C3F}, {0x00001C7E, 0x00001C7F}, {0x00001CC0, 0x00001CC7}, {0x00001CD3, 0x00001CD3},-{0x00002010, 0x00002027}, {0x00002030, 0x00002043}, {0x00002045, 0x00002051}, {0x00002053, 0x0000205E},-{0x0000207D, 0x0000207E}, {0x0000208D, 0x0000208E}, {0x00002308, 0x0000230B}, {0x00002329, 0x0000232A},-{0x00002768, 0x00002775}, {0x000027C5, 0x000027C6}, {0x000027E6, 0x000027EF}, {0x00002983, 0x00002998},-{0x000029D8, 0x000029DB}, {0x000029FC, 0x000029FD}, {0x00002CF9, 0x00002CFC}, {0x00002CFE, 0x00002CFF},-{0x00002D70, 0x00002D70}, {0x00002E00, 0x00002E2E}, {0x00002E30, 0x00002E4F}, {0x00002E52, 0x00002E5D},-{0x00003001, 0x00003003}, {0x00003008, 0x00003011}, {0x00003014, 0x0000301F}, {0x00003030, 0x00003030},-{0x0000303D, 0x0000303D}, {0x000030A0, 0x000030A0}, {0x000030FB, 0x000030FB}, {0x0000A4FE, 0x0000A4FF},-{0x0000A60D, 0x0000A60F}, {0x0000A673, 0x0000A673}, {0x0000A67E, 0x0000A67E}, {0x0000A6F2, 0x0000A6F7},-{0x0000A874, 0x0000A877}, {0x0000A8CE, 0x0000A8CF}, {0x0000A8F8, 0x0000A8FA}, {0x0000A8FC, 0x0000A8FC},-{0x0000A92E, 0x0000A92F}, {0x0000A95F, 0x0000A95F}, {0x0000A9C1, 0x0000A9CD}, {0x0000A9DE, 0x0000A9DF},-{0x0000AA5C, 0x0000AA5F}, {0x0000AADE, 0x0000AADF}, {0x0000AAF0, 0x0000AAF1}, {0x0000ABEB, 0x0000ABEB},-{0x0000FD3E, 0x0000FD3F}, {0x0000FE10, 0x0000FE19}, {0x0000FE30, 0x0000FE52}, {0x0000FE54, 0x0000FE61},-{0x0000FE63, 0x0000FE63}, {0x0000FE68, 0x0000FE68}, {0x0000FE6A, 0x0000FE6B}, {0x0000FF01, 0x0000FF03},-{0x0000FF05, 0x0000FF0A}, {0x0000FF0C, 0x0000FF0F}, {0x0000FF1A, 0x0000FF1B}, {0x0000FF1F, 0x0000FF20},-{0x0000FF3B, 0x0000FF3D}, {0x0000FF3F, 0x0000FF3F}, {0x0000FF5B, 0x0000FF5B}, {0x0000FF5D, 0x0000FF5D},-{0x0000FF5F, 0x0000FF65}, {0x00010100, 0x00010102}, {0x0001039F, 0x0001039F}, {0x000103D0, 0x000103D0},-{0x0001056F, 0x0001056F}, {0x00010857, 0x00010857}, {0x0001091F, 0x0001091F}, {0x0001093F, 0x0001093F},-{0x00010A50, 0x00010A58}, {0x00010A7F, 0x00010A7F}, {0x00010AF0, 0x00010AF6}, {0x00010B39, 0x00010B3F},-{0x00010B99, 0x00010B9C}, {0x00010EAD, 0x00010EAD}, {0x00010F55, 0x00010F59}, {0x00010F86, 0x00010F89},-{0x00011047, 0x0001104D}, {0x000110BB, 0x000110BC}, {0x000110BE, 0x000110C1}, {0x00011140, 0x00011143},-{0x00011174, 0x00011175}, {0x000111C5, 0x000111C8}, {0x000111CD, 0x000111CD}, {0x000111DB, 0x000111DB},-{0x000111DD, 0x000111DF}, {0x00011238, 0x0001123D}, {0x000112A9, 0x000112A9}, {0x0001144B, 0x0001144F},-{0x0001145A, 0x0001145B}, {0x0001145D, 0x0001145D}, {0x000114C6, 0x000114C6}, {0x000115C1, 0x000115D7},-{0x00011641, 0x00011643}, {0x00011660, 0x0001166C}, {0x000116B9, 0x000116B9}, {0x0001173C, 0x0001173E},-{0x0001183B, 0x0001183B}, {0x00011944, 0x00011946}, {0x000119E2, 0x000119E2}, {0x00011A3F, 0x00011A46},-{0x00011A9A, 0x00011A9C}, {0x00011A9E, 0x00011AA2}, {0x00011B00, 0x00011B09}, {0x00011C41, 0x00011C45},-{0x00011C70, 0x00011C71}, {0x00011EF7, 0x00011EF8}, {0x00011F43, 0x00011F4F}, {0x00011FFF, 0x00011FFF},-{0x00012470, 0x00012474}, {0x00012FF1, 0x00012FF2}, {0x00016A6E, 0x00016A6F}, {0x00016AF5, 0x00016AF5},-{0x00016B37, 0x00016B3B}, {0x00016B44, 0x00016B44}, {0x00016E97, 0x00016E9A}, {0x00016FE2, 0x00016FE2},-{0x0001BC9F, 0x0001BC9F}, {0x0001DA87, 0x0001DA8B}, {0x0001E95E, 0x0001E95F},+const std::unordered_set<uint32_t> unicode_set_whitespace = {+0x000009, 0x00000A, 0x00000B, 0x00000C, 0x00000D, 0x000020, 0x000085, 0x0000A0, 0x001680, 0x002000, 0x002001, 0x002002, 0x002003, 0x002004, 0x002005, 0x002006, 0x002007, 0x002008, 0x002009, 0x00200A, 0x002028, 0x002029, 0x00202F, 0x00205F, 0x003000};-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol = {-{0x00000024, 0x00000024}, {0x0000002B, 0x0000002B}, {0x0000003C, 0x0000003E}, {0x0000005E, 0x0000005E},-{0x00000060, 0x00000060}, {0x0000007C, 0x0000007C}, {0x0000007E, 0x0000007E}, {0x000000A2, 0x000000A6},-{0x000000A8, 0x000000A9}, {0x000000AC, 0x000000AC}, {0x000000AE, 0x000000B1}, {0x000000B4, 0x000000B4},-{0x000000B8, 0x000000B8}, {0x000000D7, 0x000000D7}, {0x000000F7, 0x000000F7}, {0x000002C2, 0x000002C5},-{0x000002D2, 0x000002DF}, {0x000002E5, 0x000002EB}, {0x000002ED, 0x000002ED}, {0x000002EF, 0x000002FF},-{0x00000375, 0x00000375}, {0x00000384, 0x00000385}, {0x000003F6, 0x000003F6}, {0x00000482, 0x00000482},-{0x0000058D, 0x0000058F}, {0x00000606, 0x00000608}, {0x0000060B, 0x0000060B}, {0x0000060E, 0x0000060F},-{0x000006DE, 0x000006DE}, {0x000006E9, 0x000006E9}, {0x000006FD, 0x000006FE}, {0x000007F6, 0x000007F6},-{0x000007FE, 0x000007FF}, {0x00000888, 0x00000888}, {0x000009F2, 0x000009F3}, {0x000009FA, 0x000009FB},-{0x00000AF1, 0x00000AF1}, {0x00000B70, 0x00000B70}, {0x00000BF3, 0x00000BFA}, {0x00000C7F, 0x00000C7F},-{0x00000D4F, 0x00000D4F}, {0x00000D79, 0x00000D79}, {0x00000E3F, 0x00000E3F}, {0x00000F01, 0x00000F03},-{0x00000F13, 0x00000F13}, {0x00000F15, 0x00000F17}, {0x00000F1A, 0x00000F1F}, {0x00000F34, 0x00000F34},-{0x00000F36, 0x00000F36}, {0x00000F38, 0x00000F38}, {0x00000FBE, 0x00000FC5}, {0x00000FC7, 0x00000FCC},-{0x00000FCE, 0x00000FCF}, {0x00000FD5, 0x00000FD8}, {0x0000109E, 0x0000109F}, {0x00001390, 0x00001399},-{0x0000166D, 0x0000166D}, {0x000017DB, 0x000017DB}, {0x00001940, 0x00001940}, {0x000019DE, 0x000019FF},-{0x00001B61, 0x00001B6A}, {0x00001B74, 0x00001B7C}, {0x00001FBD, 0x00001FBD}, {0x00001FBF, 0x00001FC1},-{0x00001FCD, 0x00001FCF}, {0x00001FDD, 0x00001FDF}, {0x00001FED, 0x00001FEF}, {0x00001FFD, 0x00001FFE},-{0x00002044, 0x00002044}, {0x00002052, 0x00002052}, {0x0000207A, 0x0000207C}, {0x0000208A, 0x0000208C},-{0x000020A0, 0x000020C0}, {0x00002100, 0x00002101}, {0x00002103, 0x00002106}, {0x00002108, 0x00002109},-{0x00002114, 0x00002114}, {0x00002116, 0x00002118}, {0x0000211E, 0x00002123}, {0x00002125, 0x00002125},-{0x00002127, 0x00002127}, {0x00002129, 0x00002129}, {0x0000212E, 0x0000212E}, {0x0000213A, 0x0000213B},-{0x00002140, 0x00002144}, {0x0000214A, 0x0000214D}, {0x0000214F, 0x0000214F}, {0x0000218A, 0x0000218B},-{0x00002190, 0x00002307}, {0x0000230C, 0x00002328}, {0x0000232B, 0x00002426}, {0x00002440, 0x0000244A},-{0x0000249C, 0x000024E9}, {0x00002500, 0x00002767}, {0x00002794, 0x000027C4}, {0x000027C7, 0x000027E5},-{0x000027F0, 0x00002982}, {0x00002999, 0x000029D7}, {0x000029DC, 0x000029FB}, {0x000029FE, 0x00002B73},-{0x00002B76, 0x00002B95}, {0x00002B97, 0x00002BFF}, {0x00002CE5, 0x00002CEA}, {0x00002E50, 0x00002E51},-{0x00002E80, 0x00002E99}, {0x00002E9B, 0x00002EF3}, {0x00002F00, 0x00002FD5}, {0x00002FF0, 0x00002FFB},-{0x00003004, 0x00003004}, {0x00003012, 0x00003013}, {0x00003020, 0x00003020}, {0x00003036, 0x00003037},-{0x0000303E, 0x0000303F}, {0x0000309B, 0x0000309C}, {0x00003190, 0x00003191}, {0x00003196, 0x0000319F},-{0x000031C0, 0x000031E3}, {0x00003200, 0x0000321E}, {0x0000322A, 0x00003247}, {0x00003250, 0x00003250},-{0x00003260, 0x0000327F}, {0x0000328A, 0x000032B0}, {0x000032C0, 0x000033FF}, {0x00004DC0, 0x00004DFF},-{0x0000A490, 0x0000A4C6}, {0x0000A700, 0x0000A716}, {0x0000A720, 0x0000A721}, {0x0000A789, 0x0000A78A},-{0x0000A828, 0x0000A82B}, {0x0000A836, 0x0000A839}, {0x0000AA77, 0x0000AA79}, {0x0000AB5B, 0x0000AB5B},-{0x0000AB6A, 0x0000AB6B}, {0x0000FB29, 0x0000FB29}, {0x0000FBB2, 0x0000FBC2}, {0x0000FD40, 0x0000FD4F},-{0x0000FDCF, 0x0000FDCF}, {0x0000FDFC, 0x0000FDFF}, {0x0000FE62, 0x0000FE62}, {0x0000FE64, 0x0000FE66},-{0x0000FE69, 0x0000FE69}, {0x0000FF04, 0x0000FF04}, {0x0000FF0B, 0x0000FF0B}, {0x0000FF1C, 0x0000FF1E},-{0x0000FF3E, 0x0000FF3E}, {0x0000FF40, 0x0000FF40}, {0x0000FF5C, 0x0000FF5C}, {0x0000FF5E, 0x0000FF5E},-{0x0000FFE0, 0x0000FFE6}, {0x0000FFE8, 0x0000FFEE}, {0x0000FFFC, 0x0000FFFD}, {0x00010137, 0x0001013F},-{0x00010179, 0x00010189}, {0x0001018C, 0x0001018E}, {0x00010190, 0x0001019C}, {0x000101A0, 0x000101A0},-{0x000101D0, 0x000101FC}, {0x00010877, 0x00010878}, {0x00010AC8, 0x00010AC8}, {0x0001173F, 0x0001173F},-{0x00011FD5, 0x00011FF1}, {0x00016B3C, 0x00016B3F}, {0x00016B45, 0x00016B45}, {0x0001BC9C, 0x0001BC9C},-{0x0001CF50, 0x0001CFC3}, {0x0001D000, 0x0001D0F5}, {0x0001D100, 0x0001D126}, {0x0001D129, 0x0001D164},-{0x0001D16A, 0x0001D16C}, {0x0001D183, 0x0001D184}, {0x0001D18C, 0x0001D1A9}, {0x0001D1AE, 0x0001D1EA},-{0x0001D200, 0x0001D241}, {0x0001D245, 0x0001D245}, {0x0001D300, 0x0001D356}, {0x0001D6C1, 0x0001D6C1},-{0x0001D6DB, 0x0001D6DB}, {0x0001D6FB, 0x0001D6FB}, {0x0001D715, 0x0001D715}, {0x0001D735, 0x0001D735},-{0x0001D74F, 0x0001D74F}, {0x0001D76F, 0x0001D76F}, {0x0001D789, 0x0001D789}, {0x0001D7A9, 0x0001D7A9},-{0x0001D7C3, 0x0001D7C3}, {0x0001D800, 0x0001D9FF}, {0x0001DA37, 0x0001DA3A}, {0x0001DA6D, 0x0001DA74},-{0x0001DA76, 0x0001DA83}, {0x0001DA85, 0x0001DA86}, {0x0001E14F, 0x0001E14F}, {0x0001E2FF, 0x0001E2FF},-{0x0001ECAC, 0x0001ECAC}, {0x0001ECB0, 0x0001ECB0}, {0x0001ED2E, 0x0001ED2E}, {0x0001EEF0, 0x0001EEF1},-{0x0001F000, 0x0001F02B}, {0x0001F030, 0x0001F093}, {0x0001F0A0, 0x0001F0AE}, {0x0001F0B1, 0x0001F0BF},-{0x0001F0C1, 0x0001F0CF}, {0x0001F0D1, 0x0001F0F5}, {0x0001F10D, 0x0001F1AD}, {0x0001F1E6, 0x0001F202},-{0x0001F210, 0x0001F23B}, {0x0001F240, 0x0001F248}, {0x0001F250, 0x0001F251}, {0x0001F260, 0x0001F265},-{0x0001F300, 0x0001F6D7}, {0x0001F6DC, 0x0001F6EC}, {0x0001F6F0, 0x0001F6FC}, {0x0001F700, 0x0001F776},-{0x0001F77B, 0x0001F7D9}, {0x0001F7E0, 0x0001F7EB}, {0x0001F7F0, 0x0001F7F0}, {0x0001F800, 0x0001F80B},-{0x0001F810, 0x0001F847}, {0x0001F850, 0x0001F859}, {0x0001F860, 0x0001F887}, {0x0001F890, 0x0001F8AD},-{0x0001F8B0, 0x0001F8B1}, {0x0001F900, 0x0001FA53}, {0x0001FA60, 0x0001FA6D}, {0x0001FA70, 0x0001FA7C},-{0x0001FA80, 0x0001FA88}, {0x0001FA90, 0x0001FABD}, {0x0001FABF, 0x0001FAC5}, {0x0001FACE, 0x0001FADB},-{0x0001FAE0, 0x0001FAE8}, {0x0001FAF0, 0x0001FAF8}, {0x0001FB00, 0x0001FB92}, {0x0001FB94, 0x0001FBCA},+const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase = {+{0x000041, 0x000061},+{0x000042, 0x000062},+{0x000043, 0x000063},+{0x000044, 0x000064},+{0x000045, 0x000065},+{0x000046, 0x000066},+{0x000047, 0x000067},+{0x000048, 0x000068},+{0x000049, 0x000069},+{0x00004A, 0x00006A},+{0x00004B, 0x00006B},+{0x00004C, 0x00006C},+{0x00004D, 0x00006D},+{0x00004E, 0x00006E},+{0x00004F, 0x00006F},+{0x000050, 0x000070},+{0x000051, 0x000071},+{0x000052, 0x000072},+{0x000053, 0x000073},+{0x000054, 0x000074},+{0x000055, 0x000075},+{0x000056, 0x000076},+{0x000057, 0x000077},+{0x000058, 0x000078},+{0x000059, 0x000079},+{0x00005A, 0x00007A},+{0x0000C0, 0x0000E0},+{0x0000C1, 0x0000E1},+{0x0000C2, 0x0000E2},+{0x0000C3, 0x0000E3},+{0x0000C4, 0x0000E4},+{0x0000C5, 0x0000E5},+{0x0000C6, 0x0000E6},+{0x0000C7, 0x0000E7},+{0x0000C8, 0x0000E8},+{0x0000C9, 0x0000E9},+{0x0000CA, 0x0000EA},+{0x0000CB, 0x0000EB},+{0x0000CC, 0x0000EC},+{0x0000CD, 0x0000ED},+{0x0000CE, 0x0000EE},+{0x0000CF, 0x0000EF},+{0x0000D0, 0x0000F0},+{0x0000D1, 0x0000F1},+{0x0000D2, 0x0000F2},+{0x0000D3, 0x0000F3},+{0x0000D4, 0x0000F4},+{0x0000D5, 0x0000F5},+{0x0000D6, 0x0000F6},+{0x0000D8, 0x0000F8},+{0x0000D9, 0x0000F9},+{0x0000DA, 0x0000FA},+{0x0000DB, 0x0000FB},+{0x0000DC, 0x0000FC},+{0x0000DD, 0x0000FD},+{0x0000DE, 0x0000FE},+{0x000100, 0x000101},+{0x000102, 0x000103},+{0x000104, 0x000105},+{0x000106, 0x000107},+{0x000108, 0x000109},+{0x00010A, 0x00010B},+{0x00010C, 0x00010D},+{0x00010E, 0x00010F},+{0x000110, 0x000111},+{0x000112, 0x000113},+{0x000114, 0x000115},+{0x000116, 0x000117},+{0x000118, 0x000119},+{0x00011A, 0x00011B},+{0x00011C, 0x00011D},+{0x00011E, 0x00011F},+{0x000120, 0x000121},+{0x000122, 0x000123},+{0x000124, 0x000125},+{0x000126, 0x000127},+{0x000128, 0x000129},+{0x00012A, 0x00012B},+{0x00012C, 0x00012D},+{0x00012E, 0x00012F},+{0x000130, 0x000069},+{0x000132, 0x000133},+{0x000134, 0x000135},+{0x000136, 0x000137},+{0x000139, 0x00013A},+{0x00013B, 0x00013C},+{0x00013D, 0x00013E},+{0x00013F, 0x000140},+{0x000141, 0x000142},+{0x000143, 0x000144},+{0x000145, 0x000146},+{0x000147, 0x000148},+{0x00014A, 0x00014B},+{0x00014C, 0x00014D},+{0x00014E, 0x00014F},+{0x000150, 0x000151},+{0x000152, 0x000153},+{0x000154, 0x000155},+{0x000156, 0x000157},+{0x000158, 0x000159},+{0x00015A, 0x00015B},+{0x00015C, 0x00015D},+{0x00015E, 0x00015F},+{0x000160, 0x000161},+{0x000162, 0x000163},+{0x000164, 0x000165},+{0x000166, 0x000167},+{0x000168, 0x000169},+{0x00016A, 0x00016B},+{0x00016C, 0x00016D},+{0x00016E, 0x00016F},+{0x000170, 0x000171},+{0x000172, 0x000173},+{0x000174, 0x000175},+{0x000176, 0x000177},+{0x000178, 0x0000FF},+{0x000179, 0x00017A},+{0x00017B, 0x00017C},+{0x00017D, 0x00017E},+{0x000181, 0x000253},+{0x000182, 0x000183},+{0x000184, 0x000185},+{0x000186, 0x000254},+{0x000187, 0x000188},+{0x000189, 0x000256},+{0x00018A, 0x000257},+{0x00018B, 0x00018C},+{0x00018E, 0x0001DD},+{0x00018F, 0x000259},+{0x000190, 0x00025B},+{0x000191, 0x000192},+{0x000193, 0x000260},+{0x000194, 0x000263},+{0x000196, 0x000269},+{0x000197, 0x000268},+{0x000198, 0x000199},+{0x00019C, 0x00026F},+{0x00019D, 0x000272},+{0x00019F, 0x000275},+{0x0001A0, 0x0001A1},+{0x0001A2, 0x0001A3},+{0x0001A4, 0x0001A5},+{0x0001A6, 0x000280},+{0x0001A7, 0x0001A8},+{0x0001A9, 0x000283},+{0x0001AC, 0x0001AD},+{0x0001AE, 0x000288},+{0x0001AF, 0x0001B0},+{0x0001B1, 0x00028A},+{0x0001B2, 0x00028B},+{0x0001B3, 0x0001B4},+{0x0001B5, 0x0001B6},+{0x0001B7, 0x000292},+{0x0001B8, 0x0001B9},+{0x0001BC, 0x0001BD},+{0x0001C4, 0x0001C6},+{0x0001C5, 0x0001C6},+{0x0001C7, 0x0001C9},+{0x0001C8, 0x0001C9},+{0x0001CA, 0x0001CC},+{0x0001CB, 0x0001CC},+{0x0001CD, 0x0001CE},+{0x0001CF, 0x0001D0},+{0x0001D1, 0x0001D2},+{0x0001D3, 0x0001D4},+{0x0001D5, 0x0001D6},+{0x0001D7, 0x0001D8},+{0x0001D9, 0x0001DA},+{0x0001DB, 0x0001DC},+{0x0001DE, 0x0001DF},+{0x0001E0, 0x0001E1},+{0x0001E2, 0x0001E3},+{0x0001E4, 0x0001E5},+{0x0001E6, 0x0001E7},+{0x0001E8, 0x0001E9},+{0x0001EA, 0x0001EB},+{0x0001EC, 0x0001ED},+{0x0001EE, 0x0001EF},+{0x0001F1, 0x0001F3},+{0x0001F2, 0x0001F3},+{0x0001F4, 0x0001F5},+{0x0001F6, 0x000195},+{0x0001F7, 0x0001BF},+{0x0001F8, 0x0001F9},+{0x0001FA, 0x0001FB},+{0x0001FC, 0x0001FD},+{0x0001FE, 0x0001FF},+{0x000200, 0x000201},+{0x000202, 0x000203},+{0x000204, 0x000205},+{0x000206, 0x000207},+{0x000208, 0x000209},+{0x00020A, 0x00020B},+{0x00020C, 0x00020D},+{0x00020E, 0x00020F},+{0x000210, 0x000211},+{0x000212, 0x000213},+{0x000214, 0x000215},+{0x000216, 0x000217},+{0x000218, 0x000219},+{0x00021A, 0x00021B},+{0x00021C, 0x00021D},+{0x00021E, 0x00021F},+{0x000220, 0x00019E},+{0x000222, 0x000223},+{0x000224, 0x000225},+{0x000226, 0x000227},+{0x000228, 0x000229},+{0x00022A, 0x00022B},+{0x00022C, 0x00022D},+{0x00022E, 0x00022F},+{0x000230, 0x000231},+{0x000232, 0x000233},+{0x00023A, 0x002C65},+{0x00023B, 0x00023C},+{0x00023D, 0x00019A},+{0x00023E, 0x002C66},+{0x000241, 0x000242},+{0x000243, 0x000180},+{0x000244, 0x000289},+{0x000245, 0x00028C},+{0x000246, 0x000247},+{0x000248, 0x000249},+{0x00024A, 0x00024B},+{0x00024C, 0x00024D},+{0x00024E, 0x00024F},+{0x000370, 0x000371},+{0x000372, 0x000373},+{0x000376, 0x000377},+{0x00037F, 0x0003F3},+{0x000386, 0x0003AC},+{0x000388, 0x0003AD},+{0x000389, 0x0003AE},+{0x00038A, 0x0003AF},+{0x00038C, 0x0003CC},+{0x00038E, 0x0003CD},+{0x00038F, 0x0003CE},+{0x000391, 0x0003B1},+{0x000392, 0x0003B2},+{0x000393, 0x0003B3},+{0x000394, 0x0003B4},+{0x000395, 0x0003B5},+{0x000396, 0x0003B6},+{0x000397, 0x0003B7},+{0x000398, 0x0003B8},+{0x000399, 0x0003B9},+{0x00039A, 0x0003BA},+{0x00039B, 0x0003BB},+{0x00039C, 0x0003BC},+{0x00039D, 0x0003BD},+{0x00039E, 0x0003BE},+{0x00039F, 0x0003BF},+{0x0003A0, 0x0003C0},+{0x0003A1, 0x0003C1},+{0x0003A3, 0x0003C3},+{0x0003A4, 0x0003C4},+{0x0003A5, 0x0003C5},+{0x0003A6, 0x0003C6},+{0x0003A7, 0x0003C7},+{0x0003A8, 0x0003C8},+{0x0003A9, 0x0003C9},+{0x0003AA, 0x0003CA},+{0x0003AB, 0x0003CB},+{0x0003CF, 0x0003D7},+{0x0003D8, 0x0003D9},+{0x0003DA, 0x0003DB},+{0x0003DC, 0x0003DD},+{0x0003DE, 0x0003DF},+{0x0003E0, 0x0003E1},+{0x0003E2, 0x0003E3},+{0x0003E4, 0x0003E5},+{0x0003E6, 0x0003E7},+{0x0003E8, 0x0003E9},+{0x0003EA, 0x0003EB},+{0x0003EC, 0x0003ED},+{0x0003EE, 0x0003EF},+{0x0003F4, 0x0003B8},+{0x0003F7, 0x0003F8},+{0x0003F9, 0x0003F2},+{0x0003FA, 0x0003FB},+{0x0003FD, 0x00037B},+{0x0003FE, 0x00037C},+{0x0003FF, 0x00037D},+{0x000400, 0x000450},+{0x000401, 0x000451},+{0x000402, 0x000452},+{0x000403, 0x000453},+{0x000404, 0x000454},+{0x000405, 0x000455},+{0x000406, 0x000456},+{0x000407, 0x000457},+{0x000408, 0x000458},+{0x000409, 0x000459},+{0x00040A, 0x00045A},+{0x00040B, 0x00045B},+{0x00040C, 0x00045C},+{0x00040D, 0x00045D},+{0x00040E, 0x00045E},+{0x00040F, 0x00045F},+{0x000410, 0x000430},+{0x000411, 0x000431},+{0x000412, 0x000432},+{0x000413, 0x000433},+{0x000414, 0x000434},+{0x000415, 0x000435},+{0x000416, 0x000436},+{0x000417, 0x000437},+{0x000418, 0x000438},+{0x000419, 0x000439},+{0x00041A, 0x00043A},+{0x00041B, 0x00043B},+{0x00041C, 0x00043C},+{0x00041D, 0x00043D},+{0x00041E, 0x00043E},+{0x00041F, 0x00043F},+{0x000420, 0x000440},+{0x000421, 0x000441},+{0x000422, 0x000442},+{0x000423, 0x000443},+{0x000424, 0x000444},+{0x000425, 0x000445},+{0x000426, 0x000446},+{0x000427, 0x000447},+{0x000428, 0x000448},+{0x000429, 0x000449},+{0x00042A, 0x00044A},+{0x00042B, 0x00044B},+{0x00042C, 0x00044C},+{0x00042D, 0x00044D},+{0x00042E, 0x00044E},+{0x00042F, 0x00044F},+{0x000460, 0x000461},+{0x000462, 0x000463},+{0x000464, 0x000465},+{0x000466, 0x000467},+{0x000468, 0x000469},+{0x00046A, 0x00046B},+{0x00046C, 0x00046D},+{0x00046E, 0x00046F},+{0x000470, 0x000471},+{0x000472, 0x000473},+{0x000474, 0x000475},+{0x000476, 0x000477},+{0x000478, 0x000479},+{0x00047A, 0x00047B},+{0x00047C, 0x00047D},+{0x00047E, 0x00047F},+{0x000480, 0x000481},+{0x00048A, 0x00048B},+{0x00048C, 0x00048D},+{0x00048E, 0x00048F},+{0x000490, 0x000491},+{0x000492, 0x000493},+{0x000494, 0x000495},+{0x000496, 0x000497},+{0x000498, 0x000499},+{0x00049A, 0x00049B},+{0x00049C, 0x00049D},+{0x00049E, 0x00049F},+{0x0004A0, 0x0004A1},+{0x0004A2, 0x0004A3},+{0x0004A4, 0x0004A5},+{0x0004A6, 0x0004A7},+{0x0004A8, 0x0004A9},+{0x0004AA, 0x0004AB},+{0x0004AC, 0x0004AD},+{0x0004AE, 0x0004AF},+{0x0004B0, 0x0004B1},+{0x0004B2, 0x0004B3},+{0x0004B4, 0x0004B5},+{0x0004B6, 0x0004B7},+{0x0004B8, 0x0004B9},+{0x0004BA, 0x0004BB},+{0x0004BC, 0x0004BD},+{0x0004BE, 0x0004BF},+{0x0004C0, 0x0004CF},+{0x0004C1, 0x0004C2},+{0x0004C3, 0x0004C4},+{0x0004C5, 0x0004C6},+{0x0004C7, 0x0004C8},+{0x0004C9, 0x0004CA},+{0x0004CB, 0x0004CC},+{0x0004CD, 0x0004CE},+{0x0004D0, 0x0004D1},+{0x0004D2, 0x0004D3},+{0x0004D4, 0x0004D5},+{0x0004D6, 0x0004D7},+{0x0004D8, 0x0004D9},+{0x0004DA, 0x0004DB},+{0x0004DC, 0x0004DD},+{0x0004DE, 0x0004DF},+{0x0004E0, 0x0004E1},+{0x0004E2, 0x0004E3},+{0x0004E4, 0x0004E5},+{0x0004E6, 0x0004E7},+{0x0004E8, 0x0004E9},+{0x0004EA, 0x0004EB},+{0x0004EC, 0x0004ED},+{0x0004EE, 0x0004EF},+{0x0004F0, 0x0004F1},+{0x0004F2, 0x0004F3},+{0x0004F4, 0x0004F5},+{0x0004F6, 0x0004F7},+{0x0004F8, 0x0004F9},+{0x0004FA, 0x0004FB},+{0x0004FC, 0x0004FD},+{0x0004FE, 0x0004FF},+{0x000500, 0x000501},+{0x000502, 0x000503},+{0x000504, 0x000505},+{0x000506, 0x000507},+{0x000508, 0x000509},+{0x00050A, 0x00050B},+{0x00050C, 0x00050D},+{0x00050E, 0x00050F},+{0x000510, 0x000511},+{0x000512, 0x000513},+{0x000514, 0x000515},+{0x000516, 0x000517},+{0x000518, 0x000519},+{0x00051A, 0x00051B},+{0x00051C, 0x00051D},+{0x00051E, 0x00051F},+{0x000520, 0x000521},+{0x000522, 0x000523},+{0x000524, 0x000525},+{0x000526, 0x000527},+{0x000528, 0x000529},+{0x00052A, 0x00052B},+{0x00052C, 0x00052D},+{0x00052E, 0x00052F},+{0x000531, 0x000561},+{0x000532, 0x000562},+{0x000533, 0x000563},+{0x000534, 0x000564},+{0x000535, 0x000565},+{0x000536, 0x000566},+{0x000537, 0x000567},+{0x000538, 0x000568},+{0x000539, 0x000569},+{0x00053A, 0x00056A},+{0x00053B, 0x00056B},+{0x00053C, 0x00056C},+{0x00053D, 0x00056D},+{0x00053E, 0x00056E},+{0x00053F, 0x00056F},+{0x000540, 0x000570},+{0x000541, 0x000571},+{0x000542, 0x000572},+{0x000543, 0x000573},+{0x000544, 0x000574},+{0x000545, 0x000575},+{0x000546, 0x000576},+{0x000547, 0x000577},+{0x000548, 0x000578},+{0x000549, 0x000579},+{0x00054A, 0x00057A},+{0x00054B, 0x00057B},+{0x00054C, 0x00057C},+{0x00054D, 0x00057D},+{0x00054E, 0x00057E},+{0x00054F, 0x00057F},+{0x000550, 0x000580},+{0x000551, 0x000581},+{0x000552, 0x000582},+{0x000553, 0x000583},+{0x000554, 0x000584},+{0x000555, 0x000585},+{0x000556, 0x000586},+{0x0010A0, 0x002D00},+{0x0010A1, 0x002D01},+{0x0010A2, 0x002D02},+{0x0010A3, 0x002D03},+{0x0010A4, 0x002D04},+{0x0010A5, 0x002D05},+{0x0010A6, 0x002D06},+{0x0010A7, 0x002D07},+{0x0010A8, 0x002D08},+{0x0010A9, 0x002D09},+{0x0010AA, 0x002D0A},+{0x0010AB, 0x002D0B},+{0x0010AC, 0x002D0C},+{0x0010AD, 0x002D0D},+{0x0010AE, 0x002D0E},+{0x0010AF, 0x002D0F},+{0x0010B0, 0x002D10},+{0x0010B1, 0x002D11},+{0x0010B2, 0x002D12},+{0x0010B3, 0x002D13},+{0x0010B4, 0x002D14},+{0x0010B5, 0x002D15},+{0x0010B6, 0x002D16},+{0x0010B7, 0x002D17},+{0x0010B8, 0x002D18},+{0x0010B9, 0x002D19},+{0x0010BA, 0x002D1A},+{0x0010BB, 0x002D1B},+{0x0010BC, 0x002D1C},+{0x0010BD, 0x002D1D},+{0x0010BE, 0x002D1E},+{0x0010BF, 0x002D1F},+{0x0010C0, 0x002D20},+{0x0010C1, 0x002D21},+{0x0010C2, 0x002D22},+{0x0010C3, 0x002D23},+{0x0010C4, 0x002D24},+{0x0010C5, 0x002D25},+{0x0010C7, 0x002D27},+{0x0010CD, 0x002D2D},+{0x0013A0, 0x00AB70},+{0x0013A1, 0x00AB71},+{0x0013A2, 0x00AB72},+{0x0013A3, 0x00AB73},+{0x0013A4, 0x00AB74},+{0x0013A5, 0x00AB75},+{0x0013A6, 0x00AB76},+{0x0013A7, 0x00AB77},+{0x0013A8, 0x00AB78},+{0x0013A9, 0x00AB79},+{0x0013AA, 0x00AB7A},+{0x0013AB, 0x00AB7B},+{0x0013AC, 0x00AB7C},+{0x0013AD, 0x00AB7D},+{0x0013AE, 0x00AB7E},+{0x0013AF, 0x00AB7F},+{0x0013B0, 0x00AB80},+{0x0013B1, 0x00AB81},+{0x0013B2, 0x00AB82},+{0x0013B3, 0x00AB83},+{0x0013B4, 0x00AB84},+{0x0013B5, 0x00AB85},+{0x0013B6, 0x00AB86},+{0x0013B7, 0x00AB87},+{0x0013B8, 0x00AB88},+{0x0013B9, 0x00AB89},+{0x0013BA, 0x00AB8A},+{0x0013BB, 0x00AB8B},+{0x0013BC, 0x00AB8C},+{0x0013BD, 0x00AB8D},+{0x0013BE, 0x00AB8E},+{0x0013BF, 0x00AB8F},+{0x0013C0, 0x00AB90},+{0x0013C1, 0x00AB91},+{0x0013C2, 0x00AB92},+{0x0013C3, 0x00AB93},+{0x0013C4, 0x00AB94},+{0x0013C5, 0x00AB95},+{0x0013C6, 0x00AB96},+{0x0013C7, 0x00AB97},+{0x0013C8, 0x00AB98},+{0x0013C9, 0x00AB99},+{0x0013CA, 0x00AB9A},+{0x0013CB, 0x00AB9B},+{0x0013CC, 0x00AB9C},+{0x0013CD, 0x00AB9D},+{0x0013CE, 0x00AB9E},+{0x0013CF, 0x00AB9F},+{0x0013D0, 0x00ABA0},+{0x0013D1, 0x00ABA1},+{0x0013D2, 0x00ABA2},+{0x0013D3, 0x00ABA3},+{0x0013D4, 0x00ABA4},+{0x0013D5, 0x00ABA5},+{0x0013D6, 0x00ABA6},+{0x0013D7, 0x00ABA7},+{0x0013D8, 0x00ABA8},+{0x0013D9, 0x00ABA9},+{0x0013DA, 0x00ABAA},+{0x0013DB, 0x00ABAB},+{0x0013DC, 0x00ABAC},+{0x0013DD, 0x00ABAD},+{0x0013DE, 0x00ABAE},+{0x0013DF, 0x00ABAF},+{0x0013E0, 0x00ABB0},+{0x0013E1, 0x00ABB1},+{0x0013E2, 0x00ABB2},+{0x0013E3, 0x00ABB3},+{0x0013E4, 0x00ABB4},+{0x0013E5, 0x00ABB5},+{0x0013E6, 0x00ABB6},+{0x0013E7, 0x00ABB7},+{0x0013E8, 0x00ABB8},+{0x0013E9, 0x00ABB9},+{0x0013EA, 0x00ABBA},+{0x0013EB, 0x00ABBB},+{0x0013EC, 0x00ABBC},+{0x0013ED, 0x00ABBD},+{0x0013EE, 0x00ABBE},+{0x0013EF, 0x00ABBF},+{0x0013F0, 0x0013F8},+{0x0013F1, 0x0013F9},+{0x0013F2, 0x0013FA},+{0x0013F3, 0x0013FB},+{0x0013F4, 0x0013FC},+{0x0013F5, 0x0013FD},+{0x001C90, 0x0010D0},+{0x001C91, 0x0010D1},+{0x001C92, 0x0010D2},+{0x001C93, 0x0010D3},+{0x001C94, 0x0010D4},+{0x001C95, 0x0010D5},+{0x001C96, 0x0010D6},+{0x001C97, 0x0010D7},+{0x001C98, 0x0010D8},+{0x001C99, 0x0010D9},+{0x001C9A, 0x0010DA},+{0x001C9B, 0x0010DB},+{0x001C9C, 0x0010DC},+{0x001C9D, 0x0010DD},+{0x001C9E, 0x0010DE},+{0x001C9F, 0x0010DF},+{0x001CA0, 0x0010E0},+{0x001CA1, 0x0010E1},+{0x001CA2, 0x0010E2},+{0x001CA3, 0x0010E3},+{0x001CA4, 0x0010E4},+{0x001CA5, 0x0010E5},+{0x001CA6, 0x0010E6},+{0x001CA7, 0x0010E7},+{0x001CA8, 0x0010E8},+{0x001CA9, 0x0010E9},+{0x001CAA, 0x0010EA},+{0x001CAB, 0x0010EB},+{0x001CAC, 0x0010EC},+{0x001CAD, 0x0010ED},+{0x001CAE, 0x0010EE},+{0x001CAF, 0x0010EF},+{0x001CB0, 0x0010F0},+{0x001CB1, 0x0010F1},+{0x001CB2, 0x0010F2},+{0x001CB3, 0x0010F3},+{0x001CB4, 0x0010F4},+{0x001CB5, 0x0010F5},+{0x001CB6, 0x0010F6},+{0x001CB7, 0x0010F7},+{0x001CB8, 0x0010F8},+{0x001CB9, 0x0010F9},+{0x001CBA, 0x0010FA},+{0x001CBD, 0x0010FD},+{0x001CBE, 0x0010FE},+{0x001CBF, 0x0010FF},+{0x001E00, 0x001E01},+{0x001E02, 0x001E03},+{0x001E04, 0x001E05},+{0x001E06, 0x001E07},+{0x001E08, 0x001E09},+{0x001E0A, 0x001E0B},+{0x001E0C, 0x001E0D},+{0x001E0E, 0x001E0F},+{0x001E10, 0x001E11},+{0x001E12, 0x001E13},+{0x001E14, 0x001E15},+{0x001E16, 0x001E17},+{0x001E18, 0x001E19},+{0x001E1A, 0x001E1B},+{0x001E1C, 0x001E1D},+{0x001E1E, 0x001E1F},+{0x001E20, 0x001E21},+{0x001E22, 0x001E23},+{0x001E24, 0x001E25},+{0x001E26, 0x001E27},+{0x001E28, 0x001E29},+{0x001E2A, 0x001E2B},+{0x001E2C, 0x001E2D},+{0x001E2E, 0x001E2F},+{0x001E30, 0x001E31},+{0x001E32, 0x001E33},+{0x001E34, 0x001E35},+{0x001E36, 0x001E37},+{0x001E38, 0x001E39},+{0x001E3A, 0x001E3B},+{0x001E3C, 0x001E3D},+{0x001E3E, 0x001E3F},+{0x001E40, 0x001E41},+{0x001E42, 0x001E43},+{0x001E44, 0x001E45},+{0x001E46, 0x001E47},+{0x001E48, 0x001E49},+{0x001E4A, 0x001E4B},+{0x001E4C, 0x001E4D},+{0x001E4E, 0x001E4F},+{0x001E50, 0x001E51},+{0x001E52, 0x001E53},+{0x001E54, 0x001E55},+{0x001E56, 0x001E57},+{0x001E58, 0x001E59},+{0x001E5A, 0x001E5B},+{0x001E5C, 0x001E5D},+{0x001E5E, 0x001E5F},+{0x001E60, 0x001E61},+{0x001E62, 0x001E63},+{0x001E64, 0x001E65},+{0x001E66, 0x001E67},+{0x001E68, 0x001E69},+{0x001E6A, 0x001E6B},+{0x001E6C, 0x001E6D},+{0x001E6E, 0x001E6F},+{0x001E70, 0x001E71},+{0x001E72, 0x001E73},+{0x001E74, 0x001E75},+{0x001E76, 0x001E77},+{0x001E78, 0x001E79},+{0x001E7A, 0x001E7B},+{0x001E7C, 0x001E7D},+{0x001E7E, 0x001E7F},+{0x001E80, 0x001E81},+{0x001E82, 0x001E83},+{0x001E84, 0x001E85},+{0x001E86, 0x001E87},+{0x001E88, 0x001E89},+{0x001E8A, 0x001E8B},+{0x001E8C, 0x001E8D},+{0x001E8E, 0x001E8F},+{0x001E90, 0x001E91},+{0x001E92, 0x001E93},+{0x001E94, 0x001E95},+{0x001E9E, 0x0000DF},+{0x001EA0, 0x001EA1},+{0x001EA2, 0x001EA3},+{0x001EA4, 0x001EA5},+{0x001EA6, 0x001EA7},+{0x001EA8, 0x001EA9},+{0x001EAA, 0x001EAB},+{0x001EAC, 0x001EAD},+{0x001EAE, 0x001EAF},+{0x001EB0, 0x001EB1},+{0x001EB2, 0x001EB3},+{0x001EB4, 0x001EB5},+{0x001EB6, 0x001EB7},+{0x001EB8, 0x001EB9},+{0x001EBA, 0x001EBB},+{0x001EBC, 0x001EBD},+{0x001EBE, 0x001EBF},+{0x001EC0, 0x001EC1},+{0x001EC2, 0x001EC3},+{0x001EC4, 0x001EC5},+{0x001EC6, 0x001EC7},+{0x001EC8, 0x001EC9},+{0x001ECA, 0x001ECB},+{0x001ECC, 0x001ECD},+{0x001ECE, 0x001ECF},+{0x001ED0, 0x001ED1},+{0x001ED2, 0x001ED3},+{0x001ED4, 0x001ED5},+{0x001ED6, 0x001ED7},+{0x001ED8, 0x001ED9},+{0x001EDA, 0x001EDB},+{0x001EDC, 0x001EDD},+{0x001EDE, 0x001EDF},+{0x001EE0, 0x001EE1},+{0x001EE2, 0x001EE3},+{0x001EE4, 0x001EE5},+{0x001EE6, 0x001EE7},+{0x001EE8, 0x001EE9},+{0x001EEA, 0x001EEB},+{0x001EEC, 0x001EED},+{0x001EEE, 0x001EEF},+{0x001EF0, 0x001EF1},+{0x001EF2, 0x001EF3},+{0x001EF4, 0x001EF5},+{0x001EF6, 0x001EF7},+{0x001EF8, 0x001EF9},+{0x001EFA, 0x001EFB},+{0x001EFC, 0x001EFD},+{0x001EFE, 0x001EFF},+{0x001F08, 0x001F00},+{0x001F09, 0x001F01},+{0x001F0A, 0x001F02},+{0x001F0B, 0x001F03},+{0x001F0C, 0x001F04},+{0x001F0D, 0x001F05},+{0x001F0E, 0x001F06},+{0x001F0F, 0x001F07},+{0x001F18, 0x001F10},+{0x001F19, 0x001F11},+{0x001F1A, 0x001F12},+{0x001F1B, 0x001F13},+{0x001F1C, 0x001F14},+{0x001F1D, 0x001F15},+{0x001F28, 0x001F20},+{0x001F29, 0x001F21},+{0x001F2A, 0x001F22},+{0x001F2B, 0x001F23},+{0x001F2C, 0x001F24},+{0x001F2D, 0x001F25},+{0x001F2E, 0x001F26},+{0x001F2F, 0x001F27},+{0x001F38, 0x001F30},+{0x001F39, 0x001F31},+{0x001F3A, 0x001F32},+{0x001F3B, 0x001F33},+{0x001F3C, 0x001F34},+{0x001F3D, 0x001F35},+{0x001F3E, 0x001F36},+{0x001F3F, 0x001F37},+{0x001F48, 0x001F40},+{0x001F49, 0x001F41},+{0x001F4A, 0x001F42},+{0x001F4B, 0x001F43},+{0x001F4C, 0x001F44},+{0x001F4D, 0x001F45},+{0x001F59, 0x001F51},+{0x001F5B, 0x001F53},+{0x001F5D, 0x001F55},+{0x001F5F, 0x001F57},+{0x001F68, 0x001F60},+{0x001F69, 0x001F61},+{0x001F6A, 0x001F62},+{0x001F6B, 0x001F63},+{0x001F6C, 0x001F64},+{0x001F6D, 0x001F65},+{0x001F6E, 0x001F66},+{0x001F6F, 0x001F67},+{0x001F88, 0x001F80},+{0x001F89, 0x001F81},+{0x001F8A, 0x001F82},+{0x001F8B, 0x001F83},+{0x001F8C, 0x001F84},+{0x001F8D, 0x001F85},+{0x001F8E, 0x001F86},+{0x001F8F, 0x001F87},+{0x001F98, 0x001F90},+{0x001F99, 0x001F91},+{0x001F9A, 0x001F92},+{0x001F9B, 0x001F93},+{0x001F9C, 0x001F94},+{0x001F9D, 0x001F95},+{0x001F9E, 0x001F96},+{0x001F9F, 0x001F97},+{0x001FA8, 0x001FA0},+{0x001FA9, 0x001FA1},+{0x001FAA, 0x001FA2},+{0x001FAB, 0x001FA3},+{0x001FAC, 0x001FA4},+{0x001FAD, 0x001FA5},+{0x001FAE, 0x001FA6},+{0x001FAF, 0x001FA7},+{0x001FB8, 0x001FB0},+{0x001FB9, 0x001FB1},+{0x001FBA, 0x001F70},+{0x001FBB, 0x001F71},+{0x001FBC, 0x001FB3},+{0x001FC8, 0x001F72},+{0x001FC9, 0x001F73},+{0x001FCA, 0x001F74},+{0x001FCB, 0x001F75},+{0x001FCC, 0x001FC3},+{0x001FD8, 0x001FD0},+{0x001FD9, 0x001FD1},+{0x001FDA, 0x001F76},+{0x001FDB, 0x001F77},+{0x001FE8, 0x001FE0},+{0x001FE9, 0x001FE1},+{0x001FEA, 0x001F7A},+{0x001FEB, 0x001F7B},+{0x001FEC, 0x001FE5},+{0x001FF8, 0x001F78},+{0x001FF9, 0x001F79},+{0x001FFA, 0x001F7C},+{0x001FFB, 0x001F7D},+{0x001FFC, 0x001FF3},+{0x002126, 0x0003C9},+{0x00212A, 0x00006B},+{0x00212B, 0x0000E5},+{0x002132, 0x00214E},+{0x002160, 0x002170},+{0x002161, 0x002171},+{0x002162, 0x002172},+{0x002163, 0x002173},+{0x002164, 0x002174},+{0x002165, 0x002175},+{0x002166, 0x002176},+{0x002167, 0x002177},+{0x002168, 0x002178},+{0x002169, 0x002179},+{0x00216A, 0x00217A},+{0x00216B, 0x00217B},+{0x00216C, 0x00217C},+{0x00216D, 0x00217D},+{0x00216E, 0x00217E},+{0x00216F, 0x00217F},+{0x002183, 0x002184},+{0x0024B6, 0x0024D0},+{0x0024B7, 0x0024D1},+{0x0024B8, 0x0024D2},+{0x0024B9, 0x0024D3},+{0x0024BA, 0x0024D4},+{0x0024BB, 0x0024D5},+{0x0024BC, 0x0024D6},+{0x0024BD, 0x0024D7},+{0x0024BE, 0x0024D8},+{0x0024BF, 0x0024D9},+{0x0024C0, 0x0024DA},+{0x0024C1, 0x0024DB},+{0x0024C2, 0x0024DC},+{0x0024C3, 0x0024DD},+{0x0024C4, 0x0024DE},+{0x0024C5, 0x0024DF},+{0x0024C6, 0x0024E0},+{0x0024C7, 0x0024E1},+{0x0024C8, 0x0024E2},+{0x0024C9, 0x0024E3},+{0x0024CA, 0x0024E4},+{0x0024CB, 0x0024E5},+{0x0024CC, 0x0024E6},+{0x0024CD, 0x0024E7},+{0x0024CE, 0x0024E8},+{0x0024CF, 0x0024E9},+{0x002C00, 0x002C30},+{0x002C01, 0x002C31},+{0x002C02, 0x002C32},+{0x002C03, 0x002C33},+{0x002C04, 0x002C34},+{0x002C05, 0x002C35},+{0x002C06, 0x002C36},+{0x002C07, 0x002C37},+{0x002C08, 0x002C38},+{0x002C09, 0x002C39},+{0x002C0A, 0x002C3A},+{0x002C0B, 0x002C3B},+{0x002C0C, 0x002C3C},+{0x002C0D, 0x002C3D},+{0x002C0E, 0x002C3E},+{0x002C0F, 0x002C3F},+{0x002C10, 0x002C40},+{0x002C11, 0x002C41},+{0x002C12, 0x002C42},+{0x002C13, 0x002C43},+{0x002C14, 0x002C44},+{0x002C15, 0x002C45},+{0x002C16, 0x002C46},+{0x002C17, 0x002C47},+{0x002C18, 0x002C48},+{0x002C19, 0x002C49},+{0x002C1A, 0x002C4A},+{0x002C1B, 0x002C4B},+{0x002C1C, 0x002C4C},+{0x002C1D, 0x002C4D},+{0x002C1E, 0x002C4E},+{0x002C1F, 0x002C4F},+{0x002C20, 0x002C50},+{0x002C21, 0x002C51},+{0x002C22, 0x002C52},+{0x002C23, 0x002C53},+{0x002C24, 0x002C54},+{0x002C25, 0x002C55},+{0x002C26, 0x002C56},+{0x002C27, 0x002C57},+{0x002C28, 0x002C58},+{0x002C29, 0x002C59},+{0x002C2A, 0x002C5A},+{0x002C2B, 0x002C5B},+{0x002C2C, 0x002C5C},+{0x002C2D, 0x002C5D},+{0x002C2E, 0x002C5E},+{0x002C60, 0x002C61},+{0x002C62, 0x00026B},+{0x002C63, 0x001D7D},+{0x002C64, 0x00027D},+{0x002C67, 0x002C68},+{0x002C69, 0x002C6A},+{0x002C6B, 0x002C6C},+{0x002C6D, 0x000251},+{0x002C6E, 0x000271},+{0x002C6F, 0x000250},+{0x002C70, 0x000252},+{0x002C72, 0x002C73},+{0x002C75, 0x002C76},+{0x002C7E, 0x00023F},+{0x002C7F, 0x000240},+{0x002C80, 0x002C81},+{0x002C82, 0x002C83},+{0x002C84, 0x002C85},+{0x002C86, 0x002C87},+{0x002C88, 0x002C89},+{0x002C8A, 0x002C8B},+{0x002C8C, 0x002C8D},+{0x002C8E, 0x002C8F},+{0x002C90, 0x002C91},+{0x002C92, 0x002C93},+{0x002C94, 0x002C95},+{0x002C96, 0x002C97},+{0x002C98, 0x002C99},+{0x002C9A, 0x002C9B},+{0x002C9C, 0x002C9D},+{0x002C9E, 0x002C9F},+{0x002CA0, 0x002CA1},+{0x002CA2, 0x002CA3},+{0x002CA4, 0x002CA5},+{0x002CA6, 0x002CA7},+{0x002CA8, 0x002CA9},+{0x002CAA, 0x002CAB},+{0x002CAC, 0x002CAD},+{0x002CAE, 0x002CAF},+{0x002CB0, 0x002CB1},+{0x002CB2, 0x002CB3},+{0x002CB4, 0x002CB5},+{0x002CB6, 0x002CB7},+{0x002CB8, 0x002CB9},+{0x002CBA, 0x002CBB},+{0x002CBC, 0x002CBD},+{0x002CBE, 0x002CBF},+{0x002CC0, 0x002CC1},+{0x002CC2, 0x002CC3},+{0x002CC4, 0x002CC5},+{0x002CC6, 0x002CC7},+{0x002CC8, 0x002CC9},+{0x002CCA, 0x002CCB},+{0x002CCC, 0x002CCD},+{0x002CCE, 0x002CCF},+{0x002CD0, 0x002CD1},+{0x002CD2, 0x002CD3},+{0x002CD4, 0x002CD5},+{0x002CD6, 0x002CD7},+{0x002CD8, 0x002CD9},+{0x002CDA, 0x002CDB},+{0x002CDC, 0x002CDD},+{0x002CDE, 0x002CDF},+{0x002CE0, 0x002CE1},+{0x002CE2, 0x002CE3},+{0x002CEB, 0x002CEC},+{0x002CED, 0x002CEE},+{0x002CF2, 0x002CF3},+{0x00A640, 0x00A641},+{0x00A642, 0x00A643},+{0x00A644, 0x00A645},+{0x00A646, 0x00A647},+{0x00A648, 0x00A649},+{0x00A64A, 0x00A64B},+{0x00A64C, 0x00A64D},+{0x00A64E, 0x00A64F},+{0x00A650, 0x00A651},+{0x00A652, 0x00A653},+{0x00A654, 0x00A655},+{0x00A656, 0x00A657},+{0x00A658, 0x00A659},+{0x00A65A, 0x00A65B},+{0x00A65C, 0x00A65D},+{0x00A65E, 0x00A65F},+{0x00A660, 0x00A661},+{0x00A662, 0x00A663},+{0x00A664, 0x00A665},+{0x00A666, 0x00A667},+{0x00A668, 0x00A669},+{0x00A66A, 0x00A66B},+{0x00A66C, 0x00A66D},+{0x00A680, 0x00A681},+{0x00A682, 0x00A683},+{0x00A684, 0x00A685},+{0x00A686, 0x00A687},+{0x00A688, 0x00A689},+{0x00A68A, 0x00A68B},+{0x00A68C, 0x00A68D},+{0x00A68E, 0x00A68F},+{0x00A690, 0x00A691},+{0x00A692, 0x00A693},+{0x00A694, 0x00A695},+{0x00A696, 0x00A697},+{0x00A698, 0x00A699},+{0x00A69A, 0x00A69B},+{0x00A722, 0x00A723},+{0x00A724, 0x00A725},+{0x00A726, 0x00A727},+{0x00A728, 0x00A729},+{0x00A72A, 0x00A72B},+{0x00A72C, 0x00A72D},+{0x00A72E, 0x00A72F},+{0x00A732, 0x00A733},+{0x00A734, 0x00A735},+{0x00A736, 0x00A737},+{0x00A738, 0x00A739},+{0x00A73A, 0x00A73B},+{0x00A73C, 0x00A73D},+{0x00A73E, 0x00A73F},+{0x00A740, 0x00A741},+{0x00A742, 0x00A743},+{0x00A744, 0x00A745},+{0x00A746, 0x00A747},+{0x00A748, 0x00A749},+{0x00A74A, 0x00A74B},+{0x00A74C, 0x00A74D},+{0x00A74E, 0x00A74F},+{0x00A750, 0x00A751},+{0x00A752, 0x00A753},+{0x00A754, 0x00A755},+{0x00A756, 0x00A757},+{0x00A758, 0x00A759},+{0x00A75A, 0x00A75B},+{0x00A75C, 0x00A75D},+{0x00A75E, 0x00A75F},+{0x00A760, 0x00A761},+{0x00A762, 0x00A763},+{0x00A764, 0x00A765},+{0x00A766, 0x00A767},+{0x00A768, 0x00A769},+{0x00A76A, 0x00A76B},+{0x00A76C, 0x00A76D},+{0x00A76E, 0x00A76F},+{0x00A779, 0x00A77A},+{0x00A77B, 0x00A77C},+{0x00A77D, 0x001D79},+{0x00A77E, 0x00A77F},+{0x00A780, 0x00A781},+{0x00A782, 0x00A783},+{0x00A784, 0x00A785},+{0x00A786, 0x00A787},+{0x00A78B, 0x00A78C},+{0x00A78D, 0x000265},+{0x00A790, 0x00A791},+{0x00A792, 0x00A793},+{0x00A796, 0x00A797},+{0x00A798, 0x00A799},+{0x00A79A, 0x00A79B},+{0x00A79C, 0x00A79D},+{0x00A79E, 0x00A79F},+{0x00A7A0, 0x00A7A1},+{0x00A7A2, 0x00A7A3},+{0x00A7A4, 0x00A7A5},+{0x00A7A6, 0x00A7A7},+{0x00A7A8, 0x00A7A9},+{0x00A7AA, 0x000266},+{0x00A7AB, 0x00025C},+{0x00A7AC, 0x000261},+{0x00A7AD, 0x00026C},+{0x00A7AE, 0x00026A},+{0x00A7B0, 0x00029E},+{0x00A7B1, 0x000287},+{0x00A7B2, 0x00029D},+{0x00A7B3, 0x00AB53},+{0x00A7B4, 0x00A7B5},+{0x00A7B6, 0x00A7B7},+{0x00A7B8, 0x00A7B9},+{0x00A7BA, 0x00A7BB},+{0x00A7BC, 0x00A7BD},+{0x00A7BE, 0x00A7BF},+{0x00A7C2, 0x00A7C3},+{0x00A7C4, 0x00A794},+{0x00A7C5, 0x000282},+{0x00A7C6, 0x001D8E},+{0x00A7C7, 0x00A7C8},+{0x00A7C9, 0x00A7CA},+{0x00A7F5, 0x00A7F6},+{0x00FF21, 0x00FF41},+{0x00FF22, 0x00FF42},+{0x00FF23, 0x00FF43},+{0x00FF24, 0x00FF44},+{0x00FF25, 0x00FF45},+{0x00FF26, 0x00FF46},+{0x00FF27, 0x00FF47},+{0x00FF28, 0x00FF48},+{0x00FF29, 0x00FF49},+{0x00FF2A, 0x00FF4A},+{0x00FF2B, 0x00FF4B},+{0x00FF2C, 0x00FF4C},+{0x00FF2D, 0x00FF4D},+{0x00FF2E, 0x00FF4E},+{0x00FF2F, 0x00FF4F},+{0x00FF30, 0x00FF50},+{0x00FF31, 0x00FF51},+{0x00FF32, 0x00FF52},+{0x00FF33, 0x00FF53},+{0x00FF34, 0x00FF54},+{0x00FF35, 0x00FF55},+{0x00FF36, 0x00FF56},+{0x00FF37, 0x00FF57},+{0x00FF38, 0x00FF58},+{0x00FF39, 0x00FF59},+{0x00FF3A, 0x00FF5A},+{0x010400, 0x010428},+{0x010401, 0x010429},+{0x010402, 0x01042A},+{0x010403, 0x01042B},+{0x010404, 0x01042C},+{0x010405, 0x01042D},+{0x010406, 0x01042E},+{0x010407, 0x01042F},+{0x010408, 0x010430},+{0x010409, 0x010431},+{0x01040A, 0x010432},+{0x01040B, 0x010433},+{0x01040C, 0x010434},+{0x01040D, 0x010435},+{0x01040E, 0x010436},+{0x01040F, 0x010437},+{0x010410, 0x010438},+{0x010411, 0x010439},+{0x010412, 0x01043A},+{0x010413, 0x01043B},+{0x010414, 0x01043C},+{0x010415, 0x01043D},+{0x010416, 0x01043E},+{0x010417, 0x01043F},+{0x010418, 0x010440},+{0x010419, 0x010441},+{0x01041A, 0x010442},+{0x01041B, 0x010443},+{0x01041C, 0x010444},+{0x01041D, 0x010445},+{0x01041E, 0x010446},+{0x01041F, 0x010447},+{0x010420, 0x010448},+{0x010421, 0x010449},+{0x010422, 0x01044A},+{0x010423, 0x01044B},+{0x010424, 0x01044C},+{0x010425, 0x01044D},+{0x010426, 0x01044E},+{0x010427, 0x01044F},+{0x0104B0, 0x0104D8},+{0x0104B1, 0x0104D9},+{0x0104B2, 0x0104DA},+{0x0104B3, 0x0104DB},+{0x0104B4, 0x0104DC},+{0x0104B5, 0x0104DD},+{0x0104B6, 0x0104DE},+{0x0104B7, 0x0104DF},+{0x0104B8, 0x0104E0},+{0x0104B9, 0x0104E1},+{0x0104BA, 0x0104E2},+{0x0104BB, 0x0104E3},+{0x0104BC, 0x0104E4},+{0x0104BD, 0x0104E5},+{0x0104BE, 0x0104E6},+{0x0104BF, 0x0104E7},+{0x0104C0, 0x0104E8},+{0x0104C1, 0x0104E9},+{0x0104C2, 0x0104EA},+{0x0104C3, 0x0104EB},+{0x0104C4, 0x0104EC},+{0x0104C5, 0x0104ED},+{0x0104C6, 0x0104EE},+{0x0104C7, 0x0104EF},+{0x0104C8, 0x0104F0},+{0x0104C9, 0x0104F1},+{0x0104CA, 0x0104F2},+{0x0104CB, 0x0104F3},+{0x0104CC, 0x0104F4},+{0x0104CD, 0x0104F5},+{0x0104CE, 0x0104F6},+{0x0104CF, 0x0104F7},+{0x0104D0, 0x0104F8},+{0x0104D1, 0x0104F9},+{0x0104D2, 0x0104FA},+{0x0104D3, 0x0104FB},+{0x010C80, 0x010CC0},+{0x010C81, 0x010CC1},+{0x010C82, 0x010CC2},+{0x010C83, 0x010CC3},+{0x010C84, 0x010CC4},+{0x010C85, 0x010CC5},+{0x010C86, 0x010CC6},+{0x010C87, 0x010CC7},+{0x010C88, 0x010CC8},+{0x010C89, 0x010CC9},+{0x010C8A, 0x010CCA},+{0x010C8B, 0x010CCB},+{0x010C8C, 0x010CCC},+{0x010C8D, 0x010CCD},+{0x010C8E, 0x010CCE},+{0x010C8F, 0x010CCF},+{0x010C90, 0x010CD0},+{0x010C91, 0x010CD1},+{0x010C92, 0x010CD2},+{0x010C93, 0x010CD3},+{0x010C94, 0x010CD4},+{0x010C95, 0x010CD5},+{0x010C96, 0x010CD6},+{0x010C97, 0x010CD7},+{0x010C98, 0x010CD8},+{0x010C99, 0x010CD9},+{0x010C9A, 0x010CDA},+{0x010C9B, 0x010CDB},+{0x010C9C, 0x010CDC},+{0x010C9D, 0x010CDD},+{0x010C9E, 0x010CDE},+{0x010C9F, 0x010CDF},+{0x010CA0, 0x010CE0},+{0x010CA1, 0x010CE1},+{0x010CA2, 0x010CE2},+{0x010CA3, 0x010CE3},+{0x010CA4, 0x010CE4},+{0x010CA5, 0x010CE5},+{0x010CA6, 0x010CE6},+{0x010CA7, 0x010CE7},+{0x010CA8, 0x010CE8},+{0x010CA9, 0x010CE9},+{0x010CAA, 0x010CEA},+{0x010CAB, 0x010CEB},+{0x010CAC, 0x010CEC},+{0x010CAD, 0x010CED},+{0x010CAE, 0x010CEE},+{0x010CAF, 0x010CEF},+{0x010CB0, 0x010CF0},+{0x010CB1, 0x010CF1},+{0x010CB2, 0x010CF2},+{0x0118A0, 0x0118C0},+{0x0118A1, 0x0118C1},+{0x0118A2, 0x0118C2},+{0x0118A3, 0x0118C3},+{0x0118A4, 0x0118C4},+{0x0118A5, 0x0118C5},+{0x0118A6, 0x0118C6},+{0x0118A7, 0x0118C7},+{0x0118A8, 0x0118C8},+{0x0118A9, 0x0118C9},+{0x0118AA, 0x0118CA},+{0x0118AB, 0x0118CB},+{0x0118AC, 0x0118CC},+{0x0118AD, 0x0118CD},+{0x0118AE, 0x0118CE},+{0x0118AF, 0x0118CF},+{0x0118B0, 0x0118D0},+{0x0118B1, 0x0118D1},+{0x0118B2, 0x0118D2},+{0x0118B3, 0x0118D3},+{0x0118B4, 0x0118D4},+{0x0118B5, 0x0118D5},+{0x0118B6, 0x0118D6},+{0x0118B7, 0x0118D7},+{0x0118B8, 0x0118D8},+{0x0118B9, 0x0118D9},+{0x0118BA, 0x0118DA},+{0x0118BB, 0x0118DB},+{0x0118BC, 0x0118DC},+{0x0118BD, 0x0118DD},+{0x0118BE, 0x0118DE},+{0x0118BF, 0x0118DF},+{0x016E40, 0x016E60},+{0x016E41, 0x016E61},+{0x016E42, 0x016E62},+{0x016E43, 0x016E63},+{0x016E44, 0x016E64},+{0x016E45, 0x016E65},+{0x016E46, 0x016E66},+{0x016E47, 0x016E67},+{0x016E48, 0x016E68},+{0x016E49, 0x016E69},+{0x016E4A, 0x016E6A},+{0x016E4B, 0x016E6B},+{0x016E4C, 0x016E6C},+{0x016E4D, 0x016E6D},+{0x016E4E, 0x016E6E},+{0x016E4F, 0x016E6F},+{0x016E50, 0x016E70},+{0x016E51, 0x016E71},+{0x016E52, 0x016E72},+{0x016E53, 0x016E73},+{0x016E54, 0x016E74},+{0x016E55, 0x016E75},+{0x016E56, 0x016E76},+{0x016E57, 0x016E77},+{0x016E58, 0x016E78},+{0x016E59, 0x016E79},+{0x016E5A, 0x016E7A},+{0x016E5B, 0x016E7B},+{0x016E5C, 0x016E7C},+{0x016E5D, 0x016E7D},+{0x016E5E, 0x016E7E},+{0x016E5F, 0x016E7F},+{0x01E900, 0x01E922},+{0x01E901, 0x01E923},+{0x01E902, 0x01E924},+{0x01E903, 0x01E925},+{0x01E904, 0x01E926},+{0x01E905, 0x01E927},+{0x01E906, 0x01E928},+{0x01E907, 0x01E929},+{0x01E908, 0x01E92A},+{0x01E909, 0x01E92B},+{0x01E90A, 0x01E92C},+{0x01E90B, 0x01E92D},+{0x01E90C, 0x01E92E},+{0x01E90D, 0x01E92F},+{0x01E90E, 0x01E930},+{0x01E90F, 0x01E931},+{0x01E910, 0x01E932},+{0x01E911, 0x01E933},+{0x01E912, 0x01E934},+{0x01E913, 0x01E935},+{0x01E914, 0x01E936},+{0x01E915, 0x01E937},+{0x01E916, 0x01E938},+{0x01E917, 0x01E939},+{0x01E918, 0x01E93A},+{0x01E919, 0x01E93B},+{0x01E91A, 0x01E93C},+{0x01E91B, 0x01E93D},+{0x01E91C, 0x01E93E},+{0x01E91D, 0x01E93F},+{0x01E91E, 0x01E940},+{0x01E91F, 0x01E941},+{0x01E920, 0x01E942},+{0x01E921, 0x01E943},};-const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control = {-{0x00000000, 0x0000001F}, {0x0000007F, 0x0000009F}, {0x000000AD, 0x000000AD}, {0x00000378, 0x00000379},-{0x00000380, 0x00000383}, {0x0000038B, 0x0000038B}, {0x0000038D, 0x0000038D}, {0x000003A2, 0x000003A2},-{0x00000530, 0x00000530}, {0x00000557, 0x00000558}, {0x0000058B, 0x0000058C}, {0x00000590, 0x00000590},-{0x000005C8, 0x000005CF}, {0x000005EB, 0x000005EE}, {0x000005F5, 0x00000605}, {0x0000061C, 0x0000061C},-{0x000006DD, 0x000006DD}, {0x0000070E, 0x0000070F}, {0x0000074B, 0x0000074C}, {0x000007B2, 0x000007BF},-{0x000007FB, 0x000007FC}, {0x0000082E, 0x0000082F}, {0x0000083F, 0x0000083F}, {0x0000085C, 0x0000085D},-{0x0000085F, 0x0000085F}, {0x0000086B, 0x0000086F}, {0x0000088F, 0x00000897}, {0x000008E2, 0x000008E2},-{0x00000984, 0x00000984}, {0x0000098D, 0x0000098E}, {0x00000991, 0x00000992}, {0x000009A9, 0x000009A9},-{0x000009B1, 0x000009B1}, {0x000009B3, 0x000009B5}, {0x000009BA, 0x000009BB}, {0x000009C5, 0x000009C6},-{0x000009C9, 0x000009CA}, {0x000009CF, 0x000009D6}, {0x000009D8, 0x000009DB}, {0x000009DE, 0x000009DE},-{0x000009E4, 0x000009E5}, {0x000009FF, 0x00000A00}, {0x00000A04, 0x00000A04}, {0x00000A0B, 0x00000A0E},-{0x00000A11, 0x00000A12}, {0x00000A29, 0x00000A29}, {0x00000A31, 0x00000A31}, {0x00000A34, 0x00000A34},-{0x00000A37, 0x00000A37}, {0x00000A3A, 0x00000A3B}, {0x00000A3D, 0x00000A3D}, {0x00000A43, 0x00000A46},-{0x00000A49, 0x00000A4A}, {0x00000A4E, 0x00000A50}, {0x00000A52, 0x00000A58}, {0x00000A5D, 0x00000A5D},-{0x00000A5F, 0x00000A65}, {0x00000A77, 0x00000A80}, {0x00000A84, 0x00000A84}, {0x00000A8E, 0x00000A8E},-{0x00000A92, 0x00000A92}, {0x00000AA9, 0x00000AA9}, {0x00000AB1, 0x00000AB1}, {0x00000AB4, 0x00000AB4},-{0x00000ABA, 0x00000ABB}, {0x00000AC6, 0x00000AC6}, {0x00000ACA, 0x00000ACA}, {0x00000ACE, 0x00000ACF},-{0x00000AD1, 0x00000ADF}, {0x00000AE4, 0x00000AE5}, {0x00000AF2, 0x00000AF8}, {0x00000B00, 0x00000B00},-{0x00000B04, 0x00000B04}, {0x00000B0D, 0x00000B0E}, {0x00000B11, 0x00000B12}, {0x00000B29, 0x00000B29},-{0x00000B31, 0x00000B31}, {0x00000B34, 0x00000B34}, {0x00000B3A, 0x00000B3B}, {0x00000B45, 0x00000B46},-{0x00000B49, 0x00000B4A}, {0x00000B4E, 0x00000B54}, {0x00000B58, 0x00000B5B}, {0x00000B5E, 0x00000B5E},-{0x00000B64, 0x00000B65}, {0x00000B78, 0x00000B81}, {0x00000B84, 0x00000B84}, {0x00000B8B, 0x00000B8D},-{0x00000B91, 0x00000B91}, {0x00000B96, 0x00000B98}, {0x00000B9B, 0x00000B9B}, {0x00000B9D, 0x00000B9D},-{0x00000BA0, 0x00000BA2}, {0x00000BA5, 0x00000BA7}, {0x00000BAB, 0x00000BAD}, {0x00000BBA, 0x00000BBD},-{0x00000BC3, 0x00000BC5}, {0x00000BC9, 0x00000BC9}, {0x00000BCE, 0x00000BCF}, {0x00000BD1, 0x00000BD6},-{0x00000BD8, 0x00000BE5}, {0x00000BFB, 0x00000BFF}, {0x00000C0D, 0x00000C0D}, {0x00000C11, 0x00000C11},-{0x00000C29, 0x00000C29}, {0x00000C3A, 0x00000C3B}, {0x00000C45, 0x00000C45}, {0x00000C49, 0x00000C49},-{0x00000C4E, 0x00000C54}, {0x00000C57, 0x00000C57}, {0x00000C5B, 0x00000C5C}, {0x00000C5E, 0x00000C5F},-{0x00000C64, 0x00000C65}, {0x00000C70, 0x00000C76}, {0x00000C8D, 0x00000C8D}, {0x00000C91, 0x00000C91},-{0x00000CA9, 0x00000CA9}, {0x00000CB4, 0x00000CB4}, {0x00000CBA, 0x00000CBB}, {0x00000CC5, 0x00000CC5},-{0x00000CC9, 0x00000CC9}, {0x00000CCE, 0x00000CD4}, {0x00000CD7, 0x00000CDC}, {0x00000CDF, 0x00000CDF},-{0x00000CE4, 0x00000CE5}, {0x00000CF0, 0x00000CF0}, {0x00000CF4, 0x00000CFF}, {0x00000D0D, 0x00000D0D},-{0x00000D11, 0x00000D11}, {0x00000D45, 0x00000D45}, {0x00000D49, 0x00000D49}, {0x00000D50, 0x00000D53},-{0x00000D64, 0x00000D65}, {0x00000D80, 0x00000D80}, {0x00000D84, 0x00000D84}, {0x00000D97, 0x00000D99},-{0x00000DB2, 0x00000DB2}, {0x00000DBC, 0x00000DBC}, {0x00000DBE, 0x00000DBF}, {0x00000DC7, 0x00000DC9},-{0x00000DCB, 0x00000DCE}, {0x00000DD5, 0x00000DD5}, {0x00000DD7, 0x00000DD7}, {0x00000DE0, 0x00000DE5},-{0x00000DF0, 0x00000DF1}, {0x00000DF5, 0x00000E00}, {0x00000E3B, 0x00000E3E}, {0x00000E5C, 0x00000E80},-{0x00000E83, 0x00000E83}, {0x00000E85, 0x00000E85}, {0x00000E8B, 0x00000E8B}, {0x00000EA4, 0x00000EA4},-{0x00000EA6, 0x00000EA6}, {0x00000EBE, 0x00000EBF}, {0x00000EC5, 0x00000EC5}, {0x00000EC7, 0x00000EC7},-{0x00000ECF, 0x00000ECF}, {0x00000EDA, 0x00000EDB}, {0x00000EE0, 0x00000EFF}, {0x00000F48, 0x00000F48},-{0x00000F6D, 0x00000F70}, {0x00000F98, 0x00000F98}, {0x00000FBD, 0x00000FBD}, {0x00000FCD, 0x00000FCD},-{0x00000FDB, 0x00000FFF}, {0x000010C6, 0x000010C6}, {0x000010C8, 0x000010CC}, {0x000010CE, 0x000010CF},-{0x00001249, 0x00001249}, {0x0000124E, 0x0000124F}, {0x00001257, 0x00001257}, {0x00001259, 0x00001259},-{0x0000125E, 0x0000125F}, {0x00001289, 0x00001289}, {0x0000128E, 0x0000128F}, {0x000012B1, 0x000012B1},-{0x000012B6, 0x000012B7}, {0x000012BF, 0x000012BF}, {0x000012C1, 0x000012C1}, {0x000012C6, 0x000012C7},-{0x000012D7, 0x000012D7}, {0x00001311, 0x00001311}, {0x00001316, 0x00001317}, {0x0000135B, 0x0000135C},-{0x0000137D, 0x0000137F}, {0x0000139A, 0x0000139F}, {0x000013F6, 0x000013F7}, {0x000013FE, 0x000013FF},-{0x0000169D, 0x0000169F}, {0x000016F9, 0x000016FF}, {0x00001716, 0x0000171E}, {0x00001737, 0x0000173F},-{0x00001754, 0x0000175F}, {0x0000176D, 0x0000176D}, {0x00001771, 0x00001771}, {0x00001774, 0x0000177F},-{0x000017DE, 0x000017DF}, {0x000017EA, 0x000017EF}, {0x000017FA, 0x000017FF}, {0x0000180E, 0x0000180E},-{0x0000181A, 0x0000181F}, {0x00001879, 0x0000187F}, {0x000018AB, 0x000018AF}, {0x000018F6, 0x000018FF},-{0x0000191F, 0x0000191F}, {0x0000192C, 0x0000192F}, {0x0000193C, 0x0000193F}, {0x00001941, 0x00001943},-{0x0000196E, 0x0000196F}, {0x00001975, 0x0000197F}, {0x000019AC, 0x000019AF}, {0x000019CA, 0x000019CF},-{0x000019DB, 0x000019DD}, {0x00001A1C, 0x00001A1D}, {0x00001A5F, 0x00001A5F}, {0x00001A7D, 0x00001A7E},-{0x00001A8A, 0x00001A8F}, {0x00001A9A, 0x00001A9F}, {0x00001AAE, 0x00001AAF}, {0x00001ACF, 0x00001AFF},-{0x00001B4D, 0x00001B4F}, {0x00001B7F, 0x00001B7F}, {0x00001BF4, 0x00001BFB}, {0x00001C38, 0x00001C3A},-{0x00001C4A, 0x00001C4C}, {0x00001C89, 0x00001C8F}, {0x00001CBB, 0x00001CBC}, {0x00001CC8, 0x00001CCF},-{0x00001CFB, 0x00001CFF}, {0x00001F16, 0x00001F17}, {0x00001F1E, 0x00001F1F}, {0x00001F46, 0x00001F47},-{0x00001F4E, 0x00001F4F}, {0x00001F58, 0x00001F58}, {0x00001F5A, 0x00001F5A}, {0x00001F5C, 0x00001F5C},-{0x00001F5E, 0x00001F5E}, {0x00001F7E, 0x00001F7F}, {0x00001FB5, 0x00001FB5}, {0x00001FC5, 0x00001FC5},-{0x00001FD4, 0x00001FD5}, {0x00001FDC, 0x00001FDC}, {0x00001FF0, 0x00001FF1}, {0x00001FF5, 0x00001FF5},-{0x00001FFF, 0x00001FFF}, {0x0000200B, 0x0000200F}, {0x0000202A, 0x0000202E}, {0x00002060, 0x0000206F},-{0x00002072, 0x00002073}, {0x0000208F, 0x0000208F}, {0x0000209D, 0x0000209F}, {0x000020C1, 0x000020CF},-{0x000020F1, 0x000020FF}, {0x0000218C, 0x0000218F}, {0x00002427, 0x0000243F}, {0x0000244B, 0x0000245F},-{0x00002B74, 0x00002B75}, {0x00002B96, 0x00002B96}, {0x00002CF4, 0x00002CF8}, {0x00002D26, 0x00002D26},-{0x00002D28, 0x00002D2C}, {0x00002D2E, 0x00002D2F}, {0x00002D68, 0x00002D6E}, {0x00002D71, 0x00002D7E},-{0x00002D97, 0x00002D9F}, {0x00002DA7, 0x00002DA7}, {0x00002DAF, 0x00002DAF}, {0x00002DB7, 0x00002DB7},-{0x00002DBF, 0x00002DBF}, {0x00002DC7, 0x00002DC7}, {0x00002DCF, 0x00002DCF}, {0x00002DD7, 0x00002DD7},-{0x00002DDF, 0x00002DDF}, {0x00002E5E, 0x00002E7F}, {0x00002E9A, 0x00002E9A}, {0x00002EF4, 0x00002EFF},-{0x00002FD6, 0x00002FEF}, {0x00002FFC, 0x00002FFF}, {0x00003040, 0x00003040}, {0x00003097, 0x00003098},-{0x00003100, 0x00003104}, {0x00003130, 0x00003130}, {0x0000318F, 0x0000318F}, {0x000031E4, 0x000031EF},-{0x0000321F, 0x0000321F}, {0x0000A48D, 0x0000A48F}, {0x0000A4C7, 0x0000A4CF}, {0x0000A62C, 0x0000A63F},-{0x0000A6F8, 0x0000A6FF}, {0x0000A7CB, 0x0000A7CF}, {0x0000A7D2, 0x0000A7D2}, {0x0000A7D4, 0x0000A7D4},-{0x0000A7DA, 0x0000A7F1}, {0x0000A82D, 0x0000A82F}, {0x0000A83A, 0x0000A83F}, {0x0000A878, 0x0000A87F},-{0x0000A8C6, 0x0000A8CD}, {0x0000A8DA, 0x0000A8DF}, {0x0000A954, 0x0000A95E}, {0x0000A97D, 0x0000A97F},-{0x0000A9CE, 0x0000A9CE}, {0x0000A9DA, 0x0000A9DD}, {0x0000A9FF, 0x0000A9FF}, {0x0000AA37, 0x0000AA3F},-{0x0000AA4E, 0x0000AA4F}, {0x0000AA5A, 0x0000AA5B}, {0x0000AAC3, 0x0000AADA}, {0x0000AAF7, 0x0000AB00},-{0x0000AB07, 0x0000AB08}, {0x0000AB0F, 0x0000AB10}, {0x0000AB17, 0x0000AB1F}, {0x0000AB27, 0x0000AB27},-{0x0000AB2F, 0x0000AB2F}, {0x0000AB6C, 0x0000AB6F}, {0x0000ABEE, 0x0000ABEF}, {0x0000ABFA, 0x0000ABFF},-{0x0000D7A4, 0x0000D7AF}, {0x0000D7C7, 0x0000D7CA}, {0x0000D7FC, 0x0000D7FF}, {0x0000E000, 0x0000F8FF},-{0x0000FA6E, 0x0000FA6F}, {0x0000FADA, 0x0000FAFF}, {0x0000FB07, 0x0000FB12}, {0x0000FB18, 0x0000FB1C},-{0x0000FB37, 0x0000FB37}, {0x0000FB3D, 0x0000FB3D}, {0x0000FB3F, 0x0000FB3F}, {0x0000FB42, 0x0000FB42},-{0x0000FB45, 0x0000FB45}, {0x0000FBC3, 0x0000FBD2}, {0x0000FD90, 0x0000FD91}, {0x0000FDC8, 0x0000FDCE},-{0x0000FDD0, 0x0000FDEF}, {0x0000FE1A, 0x0000FE1F}, {0x0000FE53, 0x0000FE53}, {0x0000FE67, 0x0000FE67},-{0x0000FE6C, 0x0000FE6F}, {0x0000FE75, 0x0000FE75}, {0x0000FEFD, 0x0000FEFE}, {0x0000FF00, 0x0000FF00},-{0x0000FFBF, 0x0000FFC1}, {0x0000FFC8, 0x0000FFC9}, {0x0000FFD0, 0x0000FFD1}, {0x0000FFD8, 0x0000FFD9},-{0x0000FFDD, 0x0000FFDF}, {0x0000FFE7, 0x0000FFE7}, {0x0000FFEF, 0x0000FFFB}, {0x0000FFFE, 0x0000FFFF},-{0x0001000C, 0x0001000C}, {0x00010027, 0x00010027}, {0x0001003B, 0x0001003B}, {0x0001003E, 0x0001003E},-{0x0001004E, 0x0001004F}, {0x0001005E, 0x0001007F}, {0x000100FB, 0x000100FF}, {0x00010103, 0x00010106},-{0x00010134, 0x00010136}, {0x0001018F, 0x0001018F}, {0x0001019D, 0x0001019F}, {0x000101A1, 0x000101CF},-{0x000101FE, 0x0001027F}, {0x0001029D, 0x0001029F}, {0x000102D1, 0x000102DF}, {0x000102FC, 0x000102FF},-{0x00010324, 0x0001032C}, {0x0001034B, 0x0001034F}, {0x0001037B, 0x0001037F}, {0x0001039E, 0x0001039E},-{0x000103C4, 0x000103C7}, {0x000103D6, 0x000103FF}, {0x0001049E, 0x0001049F}, {0x000104AA, 0x000104AF},-{0x000104D4, 0x000104D7}, {0x000104FC, 0x000104FF}, {0x00010528, 0x0001052F}, {0x00010564, 0x0001056E},-{0x0001057B, 0x0001057B}, {0x0001058B, 0x0001058B}, {0x00010593, 0x00010593}, {0x00010596, 0x00010596},-{0x000105A2, 0x000105A2}, {0x000105B2, 0x000105B2}, {0x000105BA, 0x000105BA}, {0x000105BD, 0x000105FF},-{0x00010737, 0x0001073F}, {0x00010756, 0x0001075F}, {0x00010768, 0x0001077F}, {0x00010786, 0x00010786},-{0x000107B1, 0x000107B1}, {0x000107BB, 0x000107FF}, {0x00010806, 0x00010807}, {0x00010809, 0x00010809},-{0x00010836, 0x00010836}, {0x00010839, 0x0001083B}, {0x0001083D, 0x0001083E}, {0x00010856, 0x00010856},-{0x0001089F, 0x000108A6}, {0x000108B0, 0x000108DF}, {0x000108F3, 0x000108F3}, {0x000108F6, 0x000108FA},-{0x0001091C, 0x0001091E}, {0x0001093A, 0x0001093E}, {0x00010940, 0x0001097F}, {0x000109B8, 0x000109BB},-{0x000109D0, 0x000109D1}, {0x00010A04, 0x00010A04}, {0x00010A07, 0x00010A0B}, {0x00010A14, 0x00010A14},-{0x00010A18, 0x00010A18}, {0x00010A36, 0x00010A37}, {0x00010A3B, 0x00010A3E}, {0x00010A49, 0x00010A4F},-{0x00010A59, 0x00010A5F}, {0x00010AA0, 0x00010ABF}, {0x00010AE7, 0x00010AEA}, {0x00010AF7, 0x00010AFF},-{0x00010B36, 0x00010B38}, {0x00010B56, 0x00010B57}, {0x00010B73, 0x00010B77}, {0x00010B92, 0x00010B98},-{0x00010B9D, 0x00010BA8}, {0x00010BB0, 0x00010BFF}, {0x00010C49, 0x00010C7F}, {0x00010CB3, 0x00010CBF},-{0x00010CF3, 0x00010CF9}, {0x00010D28, 0x00010D2F}, {0x00010D3A, 0x00010E5F}, {0x00010E7F, 0x00010E7F},-{0x00010EAA, 0x00010EAA}, {0x00010EAE, 0x00010EAF}, {0x00010EB2, 0x00010EFC}, {0x00010F28, 0x00010F2F},-{0x00010F5A, 0x00010F6F}, {0x00010F8A, 0x00010FAF}, {0x00010FCC, 0x00010FDF}, {0x00010FF7, 0x00010FFF},-{0x0001104E, 0x00011051}, {0x00011076, 0x0001107E}, {0x000110BD, 0x000110BD}, {0x000110C3, 0x000110CF},-{0x000110E9, 0x000110EF}, {0x000110FA, 0x000110FF}, {0x00011135, 0x00011135}, {0x00011148, 0x0001114F},-{0x00011177, 0x0001117F}, {0x000111E0, 0x000111E0}, {0x000111F5, 0x000111FF}, {0x00011212, 0x00011212},-{0x00011242, 0x0001127F}, {0x00011287, 0x00011287}, {0x00011289, 0x00011289}, {0x0001128E, 0x0001128E},-{0x0001129E, 0x0001129E}, {0x000112AA, 0x000112AF}, {0x000112EB, 0x000112EF}, {0x000112FA, 0x000112FF},-{0x00011304, 0x00011304}, {0x0001130D, 0x0001130E}, {0x00011311, 0x00011312}, {0x00011329, 0x00011329},-{0x00011331, 0x00011331}, {0x00011334, 0x00011334}, {0x0001133A, 0x0001133A}, {0x00011345, 0x00011346},-{0x00011349, 0x0001134A}, {0x0001134E, 0x0001134F}, {0x00011351, 0x00011356}, {0x00011358, 0x0001135C},-{0x00011364, 0x00011365}, {0x0001136D, 0x0001136F}, {0x00011375, 0x000113FF}, {0x0001145C, 0x0001145C},-{0x00011462, 0x0001147F}, {0x000114C8, 0x000114CF}, {0x000114DA, 0x0001157F}, {0x000115B6, 0x000115B7},-{0x000115DE, 0x000115FF}, {0x00011645, 0x0001164F}, {0x0001165A, 0x0001165F}, {0x0001166D, 0x0001167F},-{0x000116BA, 0x000116BF}, {0x000116CA, 0x000116FF}, {0x0001171B, 0x0001171C}, {0x0001172C, 0x0001172F},-{0x00011747, 0x000117FF}, {0x0001183C, 0x0001189F}, {0x000118F3, 0x000118FE}, {0x00011907, 0x00011908},-{0x0001190A, 0x0001190B}, {0x00011914, 0x00011914}, {0x00011917, 0x00011917}, {0x00011936, 0x00011936},-{0x00011939, 0x0001193A}, {0x00011947, 0x0001194F}, {0x0001195A, 0x0001199F}, {0x000119A8, 0x000119A9},-{0x000119D8, 0x000119D9}, {0x000119E5, 0x000119FF}, {0x00011A48, 0x00011A4F}, {0x00011AA3, 0x00011AAF},-{0x00011AF9, 0x00011AFF}, {0x00011B0A, 0x00011BFF}, {0x00011C09, 0x00011C09}, {0x00011C37, 0x00011C37},-{0x00011C46, 0x00011C4F}, {0x00011C6D, 0x00011C6F}, {0x00011C90, 0x00011C91}, {0x00011CA8, 0x00011CA8},-{0x00011CB7, 0x00011CFF}, {0x00011D07, 0x00011D07}, {0x00011D0A, 0x00011D0A}, {0x00011D37, 0x00011D39},-{0x00011D3B, 0x00011D3B}, {0x00011D3E, 0x00011D3E}, {0x00011D48, 0x00011D4F}, {0x00011D5A, 0x00011D5F},-{0x00011D66, 0x00011D66}, {0x00011D69, 0x00011D69}, {0x00011D8F, 0x00011D8F}, {0x00011D92, 0x00011D92},-{0x00011D99, 0x00011D9F}, {0x00011DAA, 0x00011EDF}, {0x00011EF9, 0x00011EFF}, {0x00011F11, 0x00011F11},-{0x00011F3B, 0x00011F3D}, {0x00011F5A, 0x00011FAF}, {0x00011FB1, 0x00011FBF}, {0x00011FF2, 0x00011FFE},-{0x0001239A, 0x000123FF}, {0x0001246F, 0x0001246F}, {0x00012475, 0x0001247F}, {0x00012544, 0x00012F8F},-{0x00012FF3, 0x00012FFF}, {0x00013430, 0x0001343F}, {0x00013456, 0x000143FF}, {0x00014647, 0x000167FF},-{0x00016A39, 0x00016A3F}, {0x00016A5F, 0x00016A5F}, {0x00016A6A, 0x00016A6D}, {0x00016ABF, 0x00016ABF},-{0x00016ACA, 0x00016ACF}, {0x00016AEE, 0x00016AEF}, {0x00016AF6, 0x00016AFF}, {0x00016B46, 0x00016B4F},-{0x00016B5A, 0x00016B5A}, {0x00016B62, 0x00016B62}, {0x00016B78, 0x00016B7C}, {0x00016B90, 0x00016E3F},-{0x00016E9B, 0x00016EFF}, {0x00016F4B, 0x00016F4E}, {0x00016F88, 0x00016F8E}, {0x00016FA0, 0x00016FDF},-{0x00016FE5, 0x00016FEF}, {0x00016FF2, 0x00016FFF}, {0x000187F8, 0x000187FF}, {0x00018CD6, 0x00018CFF},-{0x00018D09, 0x0001AFEF}, {0x0001AFF4, 0x0001AFF4}, {0x0001AFFC, 0x0001AFFC}, {0x0001AFFF, 0x0001AFFF},-{0x0001B123, 0x0001B131}, {0x0001B133, 0x0001B14F}, {0x0001B153, 0x0001B154}, {0x0001B156, 0x0001B163},-{0x0001B168, 0x0001B16F}, {0x0001B2FC, 0x0001BBFF}, {0x0001BC6B, 0x0001BC6F}, {0x0001BC7D, 0x0001BC7F},-{0x0001BC89, 0x0001BC8F}, {0x0001BC9A, 0x0001BC9B}, {0x0001BCA0, 0x0001CEFF}, {0x0001CF2E, 0x0001CF2F},-{0x0001CF47, 0x0001CF4F}, {0x0001CFC4, 0x0001CFFF}, {0x0001D0F6, 0x0001D0FF}, {0x0001D127, 0x0001D128},-{0x0001D173, 0x0001D17A}, {0x0001D1EB, 0x0001D1FF}, {0x0001D246, 0x0001D2BF}, {0x0001D2D4, 0x0001D2DF},-{0x0001D2F4, 0x0001D2FF}, {0x0001D357, 0x0001D35F}, {0x0001D379, 0x0001D3FF}, {0x0001D455, 0x0001D455},-{0x0001D49D, 0x0001D49D}, {0x0001D4A0, 0x0001D4A1}, {0x0001D4A3, 0x0001D4A4}, {0x0001D4A7, 0x0001D4A8},-{0x0001D4AD, 0x0001D4AD}, {0x0001D4BA, 0x0001D4BA}, {0x0001D4BC, 0x0001D4BC}, {0x0001D4C4, 0x0001D4C4},-{0x0001D506, 0x0001D506}, {0x0001D50B, 0x0001D50C}, {0x0001D515, 0x0001D515}, {0x0001D51D, 0x0001D51D},-{0x0001D53A, 0x0001D53A}, {0x0001D53F, 0x0001D53F}, {0x0001D545, 0x0001D545}, {0x0001D547, 0x0001D549},-{0x0001D551, 0x0001D551}, {0x0001D6A6, 0x0001D6A7}, {0x0001D7CC, 0x0001D7CD}, {0x0001DA8C, 0x0001DA9A},-{0x0001DAA0, 0x0001DAA0}, {0x0001DAB0, 0x0001DEFF}, {0x0001DF1F, 0x0001DF24}, {0x0001DF2B, 0x0001DFFF},-{0x0001E007, 0x0001E007}, {0x0001E019, 0x0001E01A}, {0x0001E022, 0x0001E022}, {0x0001E025, 0x0001E025},-{0x0001E02B, 0x0001E02F}, {0x0001E06E, 0x0001E08E}, {0x0001E090, 0x0001E0FF}, {0x0001E12D, 0x0001E12F},-{0x0001E13E, 0x0001E13F}, {0x0001E14A, 0x0001E14D}, {0x0001E150, 0x0001E28F}, {0x0001E2AF, 0x0001E2BF},-{0x0001E2FA, 0x0001E2FE}, {0x0001E300, 0x0001E4CF}, {0x0001E4FA, 0x0001E7DF}, {0x0001E7E7, 0x0001E7E7},-{0x0001E7EC, 0x0001E7EC}, {0x0001E7EF, 0x0001E7EF}, {0x0001E7FF, 0x0001E7FF}, {0x0001E8C5, 0x0001E8C6},-{0x0001E8D7, 0x0001E8FF}, {0x0001E94C, 0x0001E94F}, {0x0001E95A, 0x0001E95D}, {0x0001E960, 0x0001EC70},-{0x0001ECB5, 0x0001ED00}, {0x0001ED3E, 0x0001EDFF}, {0x0001EE04, 0x0001EE04}, {0x0001EE20, 0x0001EE20},-{0x0001EE23, 0x0001EE23}, {0x0001EE25, 0x0001EE26}, {0x0001EE28, 0x0001EE28}, {0x0001EE33, 0x0001EE33},-{0x0001EE38, 0x0001EE38}, {0x0001EE3A, 0x0001EE3A}, {0x0001EE3C, 0x0001EE41}, {0x0001EE43, 0x0001EE46},-{0x0001EE48, 0x0001EE48}, {0x0001EE4A, 0x0001EE4A}, {0x0001EE4C, 0x0001EE4C}, {0x0001EE50, 0x0001EE50},-{0x0001EE53, 0x0001EE53}, {0x0001EE55, 0x0001EE56}, {0x0001EE58, 0x0001EE58}, {0x0001EE5A, 0x0001EE5A},-{0x0001EE5C, 0x0001EE5C}, {0x0001EE5E, 0x0001EE5E}, {0x0001EE60, 0x0001EE60}, {0x0001EE63, 0x0001EE63},-{0x0001EE65, 0x0001EE66}, {0x0001EE6B, 0x0001EE6B}, {0x0001EE73, 0x0001EE73}, {0x0001EE78, 0x0001EE78},-{0x0001EE7D, 0x0001EE7D}, {0x0001EE7F, 0x0001EE7F}, {0x0001EE8A, 0x0001EE8A}, {0x0001EE9C, 0x0001EEA0},-{0x0001EEA4, 0x0001EEA4}, {0x0001EEAA, 0x0001EEAA}, {0x0001EEBC, 0x0001EEEF}, {0x0001EEF2, 0x0001EFFF},-{0x0001F02C, 0x0001F02F}, {0x0001F094, 0x0001F09F}, {0x0001F0AF, 0x0001F0B0}, {0x0001F0C0, 0x0001F0C0},-{0x0001F0D0, 0x0001F0D0}, {0x0001F0F6, 0x0001F0FF}, {0x0001F1AE, 0x0001F1E5}, {0x0001F203, 0x0001F20F},-{0x0001F23C, 0x0001F23F}, {0x0001F249, 0x0001F24F}, {0x0001F252, 0x0001F25F}, {0x0001F266, 0x0001F2FF},-{0x0001F6D8, 0x0001F6DB}, {0x0001F6ED, 0x0001F6EF}, {0x0001F6FD, 0x0001F6FF}, {0x0001F777, 0x0001F77A},-{0x0001F7DA, 0x0001F7DF}, {0x0001F7EC, 0x0001F7EF}, {0x0001F7F1, 0x0001F7FF}, {0x0001F80C, 0x0001F80F},-{0x0001F848, 0x0001F84F}, {0x0001F85A, 0x0001F85F}, {0x0001F888, 0x0001F88F}, {0x0001F8AE, 0x0001F8AF},-{0x0001F8B2, 0x0001F8FF}, {0x0001FA54, 0x0001FA5F}, {0x0001FA6E, 0x0001FA6F}, {0x0001FA7D, 0x0001FA7F},-{0x0001FA89, 0x0001FA8F}, {0x0001FABE, 0x0001FABE}, {0x0001FAC6, 0x0001FACD}, {0x0001FADC, 0x0001FADF},-{0x0001FAE9, 0x0001FAEF}, {0x0001FAF9, 0x0001FAFF}, {0x0001FB93, 0x0001FB93}, {0x0001FBCB, 0x0001FBEF},-{0x0001FBFA, 0x0001FFFF}, {0x0002A6E0, 0x0002A6FF}, {0x0002B73A, 0x0002B73F}, {0x0002B81E, 0x0002B81F},-{0x0002CEA2, 0x0002CEAF}, {0x0002EBE1, 0x0002F7FF}, {0x0002FA1E, 0x0002FFFF}, {0x0003134B, 0x0003134F},-{0x000323B0, 0x000E00FF}, {0x000E01F0, 0x0010FFFF},+const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase = {+{0x000061, 0x000041},+{0x000062, 0x000042},+{0x000063, 0x000043},+{0x000064, 0x000044},+{0x000065, 0x000045},+{0x000066, 0x000046},+{0x000067, 0x000047},+{0x000068, 0x000048},+{0x000069, 0x000049},+{0x00006A, 0x00004A},+{0x00006B, 0x00004B},+{0x00006C, 0x00004C},+{0x00006D, 0x00004D},+{0x00006E, 0x00004E},+{0x00006F, 0x00004F},+{0x000070, 0x000050},+{0x000071, 0x000051},+{0x000072, 0x000052},+{0x000073, 0x000053},+{0x000074, 0x000054},+{0x000075, 0x000055},+{0x000076, 0x000056},+{0x000077, 0x000057},+{0x000078, 0x000058},+{0x000079, 0x000059},+{0x00007A, 0x00005A},+{0x0000B5, 0x00039C},+{0x0000DF, 0x000053},+{0x0000E0, 0x0000C0},+{0x0000E1, 0x0000C1},+{0x0000E2, 0x0000C2},+{0x0000E3, 0x0000C3},+{0x0000E4, 0x0000C4},+{0x0000E5, 0x0000C5},+{0x0000E6, 0x0000C6},+{0x0000E7, 0x0000C7},+{0x0000E8, 0x0000C8},+{0x0000E9, 0x0000C9},+{0x0000EA, 0x0000CA},+{0x0000EB, 0x0000CB},+{0x0000EC, 0x0000CC},+{0x0000ED, 0x0000CD},+{0x0000EE, 0x0000CE},+{0x0000EF, 0x0000CF},+{0x0000F0, 0x0000D0},+{0x0000F1, 0x0000D1},+{0x0000F2, 0x0000D2},+{0x0000F3, 0x0000D3},+{0x0000F4, 0x0000D4},+{0x0000F5, 0x0000D5},+{0x0000F6, 0x0000D6},+{0x0000F8, 0x0000D8},+{0x0000F9, 0x0000D9},+{0x0000FA, 0x0000DA},+{0x0000FB, 0x0000DB},+{0x0000FC, 0x0000DC},+{0x0000FD, 0x0000DD},+{0x0000FE, 0x0000DE},+{0x0000FF, 0x000178},+{0x000101, 0x000100},+{0x000103, 0x000102},+{0x000105, 0x000104},+{0x000107, 0x000106},+{0x000109, 0x000108},+{0x00010B, 0x00010A},+{0x00010D, 0x00010C},+{0x00010F, 0x00010E},+{0x000111, 0x000110},+{0x000113, 0x000112},+{0x000115, 0x000114},+{0x000117, 0x000116},+{0x000119, 0x000118},+{0x00011B, 0x00011A},+{0x00011D, 0x00011C},+{0x00011F, 0x00011E},+{0x000121, 0x000120},+{0x000123, 0x000122},+{0x000125, 0x000124},+{0x000127, 0x000126},+{0x000129, 0x000128},+{0x00012B, 0x00012A},+{0x00012D, 0x00012C},+{0x00012F, 0x00012E},+{0x000131, 0x000049},+{0x000133, 0x000132},+{0x000135, 0x000134},+{0x000137, 0x000136},+{0x00013A, 0x000139},+{0x00013C, 0x00013B},+{0x00013E, 0x00013D},+{0x000140, 0x00013F},+{0x000142, 0x000141},+{0x000144, 0x000143},+{0x000146, 0x000145},+{0x000148, 0x000147},+{0x000149, 0x0002BC},+{0x00014B, 0x00014A},+{0x00014D, 0x00014C},+{0x00014F, 0x00014E},+{0x000151, 0x000150},+{0x000153, 0x000152},+{0x000155, 0x000154},+{0x000157, 0x000156},+{0x000159, 0x000158},+{0x00015B, 0x00015A},+{0x00015D, 0x00015C},+{0x00015F, 0x00015E},+{0x000161, 0x000160},+{0x000163, 0x000162},+{0x000165, 0x000164},+{0x000167, 0x000166},+{0x000169, 0x000168},+{0x00016B, 0x00016A},+{0x00016D, 0x00016C},+{0x00016F, 0x00016E},+{0x000171, 0x000170},+{0x000173, 0x000172},+{0x000175, 0x000174},+{0x000177, 0x000176},+{0x00017A, 0x000179},+{0x00017C, 0x00017B},+{0x00017E, 0x00017D},+{0x00017F, 0x000053},+{0x000180, 0x000243},+{0x000183, 0x000182},+{0x000185, 0x000184},+{0x000188, 0x000187},+{0x00018C, 0x00018B},+{0x000192, 0x000191},+{0x000195, 0x0001F6},+{0x000199, 0x000198},+{0x00019A, 0x00023D},+{0x00019E, 0x000220},+{0x0001A1, 0x0001A0},+{0x0001A3, 0x0001A2},+{0x0001A5, 0x0001A4},+{0x0001A8, 0x0001A7},+{0x0001AD, 0x0001AC},+{0x0001B0, 0x0001AF},+{0x0001B4, 0x0001B3},+{0x0001B6, 0x0001B5},+{0x0001B9, 0x0001B8},+{0x0001BD, 0x0001BC},+{0x0001BF, 0x0001F7},+{0x0001C5, 0x0001C4},+{0x0001C6, 0x0001C4},+{0x0001C8, 0x0001C7},+{0x0001C9, 0x0001C7},+{0x0001CB, 0x0001CA},+{0x0001CC, 0x0001CA},+{0x0001CE, 0x0001CD},+{0x0001D0, 0x0001CF},+{0x0001D2, 0x0001D1},+{0x0001D4, 0x0001D3},+{0x0001D6, 0x0001D5},+{0x0001D8, 0x0001D7},+{0x0001DA, 0x0001D9},+{0x0001DC, 0x0001DB},+{0x0001DD, 0x00018E},+{0x0001DF, 0x0001DE},+{0x0001E1, 0x0001E0},+{0x0001E3, 0x0001E2},+{0x0001E5, 0x0001E4},+{0x0001E7, 0x0001E6},+{0x0001E9, 0x0001E8},+{0x0001EB, 0x0001EA},+{0x0001ED, 0x0001EC},+{0x0001EF, 0x0001EE},+{0x0001F0, 0x00004A},+{0x0001F2, 0x0001F1},+{0x0001F3, 0x0001F1},+{0x0001F5, 0x0001F4},+{0x0001F9, 0x0001F8},+{0x0001FB, 0x0001FA},+{0x0001FD, 0x0001FC},+{0x0001FF, 0x0001FE},+{0x000201, 0x000200},+{0x000203, 0x000202},+{0x000205, 0x000204},+{0x000207, 0x000206},+{0x000209, 0x000208},+{0x00020B, 0x00020A},+{0x00020D, 0x00020C},+{0x00020F, 0x00020E},+{0x000211, 0x000210},+{0x000213, 0x000212},+{0x000215, 0x000214},+{0x000217, 0x000216},+{0x000219, 0x000218},+{0x00021B, 0x00021A},+{0x00021D, 0x00021C},+{0x00021F, 0x00021E},+{0x000223, 0x000222},+{0x000225, 0x000224},+{0x000227, 0x000226},+{0x000229, 0x000228},+{0x00022B, 0x00022A},+{0x00022D, 0x00022C},+{0x00022F, 0x00022E},+{0x000231, 0x000230},+{0x000233, 0x000232},+{0x00023C, 0x00023B},+{0x00023F, 0x002C7E},+{0x000240, 0x002C7F},+{0x000242, 0x000241},+{0x000247, 0x000246},+{0x000249, 0x000248},+{0x00024B, 0x00024A},+{0x00024D, 0x00024C},+{0x00024F, 0x00024E},+{0x000250, 0x002C6F},+{0x000251, 0x002C6D},+{0x000252, 0x002C70},+{0x000253, 0x000181},+{0x000254, 0x000186},+{0x000256, 0x000189},+{0x000257, 0x00018A},+{0x000259, 0x00018F},+{0x00025B, 0x000190},+{0x00025C, 0x00A7AB},+{0x000260, 0x000193},+{0x000261, 0x00A7AC},+{0x000263, 0x000194},+{0x000265, 0x00A78D},+{0x000266, 0x00A7AA},+{0x000268, 0x000197},+{0x000269, 0x000196},+{0x00026A, 0x00A7AE},+{0x00026B, 0x002C62},+{0x00026C, 0x00A7AD},+{0x00026F, 0x00019C},+{0x000271, 0x002C6E},+{0x000272, 0x00019D},+{0x000275, 0x00019F},+{0x00027D, 0x002C64},+{0x000280, 0x0001A6},+{0x000282, 0x00A7C5},+{0x000283, 0x0001A9},+{0x000287, 0x00A7B1},+{0x000288, 0x0001AE},+{0x000289, 0x000244},+{0x00028A, 0x0001B1},+{0x00028B, 0x0001B2},+{0x00028C, 0x000245},+{0x000292, 0x0001B7},+{0x00029D, 0x00A7B2},+{0x00029E, 0x00A7B0},+{0x000345, 0x000399},+{0x000371, 0x000370},+{0x000373, 0x000372},+{0x000377, 0x000376},+{0x00037B, 0x0003FD},+{0x00037C, 0x0003FE},+{0x00037D, 0x0003FF},+{0x000390, 0x000399},+{0x0003AC, 0x000386},+{0x0003AD, 0x000388},+{0x0003AE, 0x000389},+{0x0003AF, 0x00038A},+{0x0003B0, 0x0003A5},+{0x0003B1, 0x000391},+{0x0003B2, 0x000392},+{0x0003B3, 0x000393},+{0x0003B4, 0x000394},+{0x0003B5, 0x000395},+{0x0003B6, 0x000396},+{0x0003B7, 0x000397},+{0x0003B8, 0x000398},+{0x0003B9, 0x000399},+{0x0003BA, 0x00039A},+{0x0003BB, 0x00039B},+{0x0003BC, 0x00039C},+{0x0003BD, 0x00039D},+{0x0003BE, 0x00039E},+{0x0003BF, 0x00039F},+{0x0003C0, 0x0003A0},+{0x0003C1, 0x0003A1},+{0x0003C2, 0x0003A3},+{0x0003C3, 0x0003A3},+{0x0003C4, 0x0003A4},+{0x0003C5, 0x0003A5},+{0x0003C6, 0x0003A6},+{0x0003C7, 0x0003A7},+{0x0003C8, 0x0003A8},+{0x0003C9, 0x0003A9},+{0x0003CA, 0x0003AA},+{0x0003CB, 0x0003AB},+{0x0003CC, 0x00038C},+{0x0003CD, 0x00038E},+{0x0003CE, 0x00038F},+{0x0003D0, 0x000392},+{0x0003D1, 0x000398},+{0x0003D5, 0x0003A6},+{0x0003D6, 0x0003A0},+{0x0003D7, 0x0003CF},+{0x0003D9, 0x0003D8},+{0x0003DB, 0x0003DA},+{0x0003DD, 0x0003DC},+{0x0003DF, 0x0003DE},+{0x0003E1, 0x0003E0},+{0x0003E3, 0x0003E2},+{0x0003E5, 0x0003E4},+{0x0003E7, 0x0003E6},+{0x0003E9, 0x0003E8},+{0x0003EB, 0x0003EA},+{0x0003ED, 0x0003EC},+{0x0003EF, 0x0003EE},+{0x0003F0, 0x00039A},+{0x0003F1, 0x0003A1},+{0x0003F2, 0x0003F9},+{0x0003F3, 0x00037F},+{0x0003F5, 0x000395},+{0x0003F8, 0x0003F7},+{0x0003FB, 0x0003FA},+{0x000430, 0x000410},+{0x000431, 0x000411},+{0x000432, 0x000412},+{0x000433, 0x000413},+{0x000434, 0x000414},+{0x000435, 0x000415},+{0x000436, 0x000416},+{0x000437, 0x000417},+{0x000438, 0x000418},+{0x000439, 0x000419},+{0x00043A, 0x00041A},+{0x00043B, 0x00041B},+{0x00043C, 0x00041C},+{0x00043D, 0x00041D},+{0x00043E, 0x00041E},+{0x00043F, 0x00041F},+{0x000440, 0x000420},+{0x000441, 0x000421},+{0x000442, 0x000422},+{0x000443, 0x000423},+{0x000444, 0x000424},+{0x000445, 0x000425},+{0x000446, 0x000426},+{0x000447, 0x000427},+{0x000448, 0x000428},+{0x000449, 0x000429},+{0x00044A, 0x00042A},+{0x00044B, 0x00042B},+{0x00044C, 0x00042C},+{0x00044D, 0x00042D},+{0x00044E, 0x00042E},+{0x00044F, 0x00042F},+{0x000450, 0x000400},+{0x000451, 0x000401},+{0x000452, 0x000402},+{0x000453, 0x000403},+{0x000454, 0x000404},+{0x000455, 0x000405},+{0x000456, 0x000406},+{0x000457, 0x000407},+{0x000458, 0x000408},+{0x000459, 0x000409},+{0x00045A, 0x00040A},+{0x00045B, 0x00040B},+{0x00045C, 0x00040C},+{0x00045D, 0x00040D},+{0x00045E, 0x00040E},+{0x00045F, 0x00040F},+{0x000461, 0x000460},+{0x000463, 0x000462},+{0x000465, 0x000464},+{0x000467, 0x000466},+{0x000469, 0x000468},+{0x00046B, 0x00046A},+{0x00046D, 0x00046C},+{0x00046F, 0x00046E},+{0x000471, 0x000470},+{0x000473, 0x000472},+{0x000475, 0x000474},+{0x000477, 0x000476},+{0x000479, 0x000478},+{0x00047B, 0x00047A},+{0x00047D, 0x00047C},+{0x00047F, 0x00047E},+{0x000481, 0x000480},+{0x00048B, 0x00048A},+{0x00048D, 0x00048C},+{0x00048F, 0x00048E},+{0x000491, 0x000490},+{0x000493, 0x000492},+{0x000495, 0x000494},+{0x000497, 0x000496},+{0x000499, 0x000498},+{0x00049B, 0x00049A},+{0x00049D, 0x00049C},+{0x00049F, 0x00049E},+{0x0004A1, 0x0004A0},+{0x0004A3, 0x0004A2},+{0x0004A5, 0x0004A4},+{0x0004A7, 0x0004A6},+{0x0004A9, 0x0004A8},+{0x0004AB, 0x0004AA},+{0x0004AD, 0x0004AC},+{0x0004AF, 0x0004AE},+{0x0004B1, 0x0004B0},+{0x0004B3, 0x0004B2},+{0x0004B5, 0x0004B4},+{0x0004B7, 0x0004B6},+{0x0004B9, 0x0004B8},+{0x0004BB, 0x0004BA},+{0x0004BD, 0x0004BC},+{0x0004BF, 0x0004BE},+{0x0004C2, 0x0004C1},+{0x0004C4, 0x0004C3},+{0x0004C6, 0x0004C5},+{0x0004C8, 0x0004C7},+{0x0004CA, 0x0004C9},+{0x0004CC, 0x0004CB},+{0x0004CE, 0x0004CD},+{0x0004CF, 0x0004C0},+{0x0004D1, 0x0004D0},+{0x0004D3, 0x0004D2},+{0x0004D5, 0x0004D4},+{0x0004D7, 0x0004D6},+{0x0004D9, 0x0004D8},+{0x0004DB, 0x0004DA},+{0x0004DD, 0x0004DC},+{0x0004DF, 0x0004DE},+{0x0004E1, 0x0004E0},+{0x0004E3, 0x0004E2},+{0x0004E5, 0x0004E4},+{0x0004E7, 0x0004E6},+{0x0004E9, 0x0004E8},+{0x0004EB, 0x0004EA},+{0x0004ED, 0x0004EC},+{0x0004EF, 0x0004EE},+{0x0004F1, 0x0004F0},+{0x0004F3, 0x0004F2},+{0x0004F5, 0x0004F4},+{0x0004F7, 0x0004F6},+{0x0004F9, 0x0004F8},+{0x0004FB, 0x0004FA},+{0x0004FD, 0x0004FC},+{0x0004FF, 0x0004FE},+{0x000501, 0x000500},+{0x000503, 0x000502},+{0x000505, 0x000504},+{0x000507, 0x000506},+{0x000509, 0x000508},+{0x00050B, 0x00050A},+{0x00050D, 0x00050C},+{0x00050F, 0x00050E},+{0x000511, 0x000510},+{0x000513, 0x000512},+{0x000515, 0x000514},+{0x000517, 0x000516},+{0x000519, 0x000518},+{0x00051B, 0x00051A},+{0x00051D, 0x00051C},+{0x00051F, 0x00051E},+{0x000521, 0x000520},+{0x000523, 0x000522},+{0x000525, 0x000524},+{0x000527, 0x000526},+{0x000529, 0x000528},+{0x00052B, 0x00052A},+{0x00052D, 0x00052C},+{0x00052F, 0x00052E},+{0x000561, 0x000531},+{0x000562, 0x000532},+{0x000563, 0x000533},+{0x000564, 0x000534},+{0x000565, 0x000535},+{0x000566, 0x000536},+{0x000567, 0x000537},+{0x000568, 0x000538},+{0x000569, 0x000539},+{0x00056A, 0x00053A},+{0x00056B, 0x00053B},+{0x00056C, 0x00053C},+{0x00056D, 0x00053D},+{0x00056E, 0x00053E},+{0x00056F, 0x00053F},+{0x000570, 0x000540},+{0x000571, 0x000541},+{0x000572, 0x000542},+{0x000573, 0x000543},+{0x000574, 0x000544},+{0x000575, 0x000545},+{0x000576, 0x000546},+{0x000577, 0x000547},+{0x000578, 0x000548},+{0x000579, 0x000549},+{0x00057A, 0x00054A},+{0x00057B, 0x00054B},+{0x00057C, 0x00054C},+{0x00057D, 0x00054D},+{0x00057E, 0x00054E},+{0x00057F, 0x00054F},+{0x000580, 0x000550},+{0x000581, 0x000551},+{0x000582, 0x000552},+{0x000583, 0x000553},+{0x000584, 0x000554},+{0x000585, 0x000555},+{0x000586, 0x000556},+{0x000587, 0x000535},+{0x0010D0, 0x001C90},+{0x0010D1, 0x001C91},+{0x0010D2, 0x001C92},+{0x0010D3, 0x001C93},+{0x0010D4, 0x001C94},+{0x0010D5, 0x001C95},+{0x0010D6, 0x001C96},+{0x0010D7, 0x001C97},+{0x0010D8, 0x001C98},+{0x0010D9, 0x001C99},+{0x0010DA, 0x001C9A},+{0x0010DB, 0x001C9B},+{0x0010DC, 0x001C9C},+{0x0010DD, 0x001C9D},+{0x0010DE, 0x001C9E},+{0x0010DF, 0x001C9F},+{0x0010E0, 0x001CA0},+{0x0010E1, 0x001CA1},+{0x0010E2, 0x001CA2},+{0x0010E3, 0x001CA3},+{0x0010E4, 0x001CA4},+{0x0010E5, 0x001CA5},+{0x0010E6, 0x001CA6},+{0x0010E7, 0x001CA7},+{0x0010E8, 0x001CA8},+{0x0010E9, 0x001CA9},+{0x0010EA, 0x001CAA},+{0x0010EB, 0x001CAB},+{0x0010EC, 0x001CAC},+{0x0010ED, 0x001CAD},+{0x0010EE, 0x001CAE},+{0x0010EF, 0x001CAF},+{0x0010F0, 0x001CB0},+{0x0010F1, 0x001CB1},+{0x0010F2, 0x001CB2},+{0x0010F3, 0x001CB3},+{0x0010F4, 0x001CB4},+{0x0010F5, 0x001CB5},+{0x0010F6, 0x001CB6},+{0x0010F7, 0x001CB7},+{0x0010F8, 0x001CB8},+{0x0010F9, 0x001CB9},+{0x0010FA, 0x001CBA},+{0x0010FD, 0x001CBD},+{0x0010FE, 0x001CBE},+{0x0010FF, 0x001CBF},+{0x0013F8, 0x0013F0},+{0x0013F9, 0x0013F1},+{0x0013FA, 0x0013F2},+{0x0013FB, 0x0013F3},+{0x0013FC, 0x0013F4},+{0x0013FD, 0x0013F5},+{0x001C80, 0x000412},+{0x001C81, 0x000414},+{0x001C82, 0x00041E},+{0x001C83, 0x000421},+{0x001C84, 0x000422},+{0x001C85, 0x000422},+{0x001C86, 0x00042A},+{0x001C87, 0x000462},+{0x001C88, 0x00A64A},+{0x001D79, 0x00A77D},+{0x001D7D, 0x002C63},+{0x001D8E, 0x00A7C6},+{0x001E01, 0x001E00},+{0x001E03, 0x001E02},+{0x001E05, 0x001E04},+{0x001E07, 0x001E06},+{0x001E09, 0x001E08},+{0x001E0B, 0x001E0A},+{0x001E0D, 0x001E0C},+{0x001E0F, 0x001E0E},+{0x001E11, 0x001E10},+{0x001E13, 0x001E12},+{0x001E15, 0x001E14},+{0x001E17, 0x001E16},+{0x001E19, 0x001E18},+{0x001E1B, 0x001E1A},+{0x001E1D, 0x001E1C},+{0x001E1F, 0x001E1E},+{0x001E21, 0x001E20},+{0x001E23, 0x001E22},+{0x001E25, 0x001E24},+{0x001E27, 0x001E26},+{0x001E29, 0x001E28},+{0x001E2B, 0x001E2A},+{0x001E2D, 0x001E2C},+{0x001E2F, 0x001E2E},+{0x001E31, 0x001E30},+{0x001E33, 0x001E32},+{0x001E35, 0x001E34},+{0x001E37, 0x001E36},+{0x001E39, 0x001E38},+{0x001E3B, 0x001E3A},+{0x001E3D, 0x001E3C},+{0x001E3F, 0x001E3E},+{0x001E41, 0x001E40},+{0x001E43, 0x001E42},+{0x001E45, 0x001E44},+{0x001E47, 0x001E46},+{0x001E49, 0x001E48},+{0x001E4B, 0x001E4A},+{0x001E4D, 0x001E4C},+{0x001E4F, 0x001E4E},+{0x001E51, 0x001E50},+{0x001E53, 0x001E52},+{0x001E55, 0x001E54},+{0x001E57, 0x001E56},+{0x001E59, 0x001E58},+{0x001E5B, 0x001E5A},+{0x001E5D, 0x001E5C},+{0x001E5F, 0x001E5E},+{0x001E61, 0x001E60},+{0x001E63, 0x001E62},+{0x001E65, 0x001E64},+{0x001E67, 0x001E66},+{0x001E69, 0x001E68},+{0x001E6B, 0x001E6A},+{0x001E6D, 0x001E6C},+{0x001E6F, 0x001E6E},+{0x001E71, 0x001E70},+{0x001E73, 0x001E72},+{0x001E75, 0x001E74},+{0x001E77, 0x001E76},+{0x001E79, 0x001E78},+{0x001E7B, 0x001E7A},+{0x001E7D, 0x001E7C},+{0x001E7F, 0x001E7E},+{0x001E81, 0x001E80},+{0x001E83, 0x001E82},+{0x001E85, 0x001E84},+{0x001E87, 0x001E86},+{0x001E89, 0x001E88},+{0x001E8B, 0x001E8A},+{0x001E8D, 0x001E8C},+{0x001E8F, 0x001E8E},+{0x001E91, 0x001E90},+{0x001E93, 0x001E92},+{0x001E95, 0x001E94},+{0x001E96, 0x000048},+{0x001E97, 0x000054},+{0x001E98, 0x000057},+{0x001E99, 0x000059},+{0x001E9A, 0x000041},+{0x001E9B, 0x001E60},+{0x001EA1, 0x001EA0},+{0x001EA3, 0x001EA2},+{0x001EA5, 0x001EA4},+{0x001EA7, 0x001EA6},+{0x001EA9, 0x001EA8},+{0x001EAB, 0x001EAA},+{0x001EAD, 0x001EAC},+{0x001EAF, 0x001EAE},+{0x001EB1, 0x001EB0},+{0x001EB3, 0x001EB2},+{0x001EB5, 0x001EB4},+{0x001EB7, 0x001EB6},+{0x001EB9, 0x001EB8},+{0x001EBB, 0x001EBA},+{0x001EBD, 0x001EBC},+{0x001EBF, 0x001EBE},+{0x001EC1, 0x001EC0},+{0x001EC3, 0x001EC2},+{0x001EC5, 0x001EC4},+{0x001EC7, 0x001EC6},+{0x001EC9, 0x001EC8},+{0x001ECB, 0x001ECA},+{0x001ECD, 0x001ECC},+{0x001ECF, 0x001ECE},+{0x001ED1, 0x001ED0},+{0x001ED3, 0x001ED2},+{0x001ED5, 0x001ED4},+{0x001ED7, 0x001ED6},+{0x001ED9, 0x001ED8},+{0x001EDB, 0x001EDA},+{0x001EDD, 0x001EDC},+{0x001EDF, 0x001EDE},+{0x001EE1, 0x001EE0},+{0x001EE3, 0x001EE2},+{0x001EE5, 0x001EE4},+{0x001EE7, 0x001EE6},+{0x001EE9, 0x001EE8},+{0x001EEB, 0x001EEA},+{0x001EED, 0x001EEC},+{0x001EEF, 0x001EEE},+{0x001EF1, 0x001EF0},+{0x001EF3, 0x001EF2},+{0x001EF5, 0x001EF4},+{0x001EF7, 0x001EF6},+{0x001EF9, 0x001EF8},+{0x001EFB, 0x001EFA},+{0x001EFD, 0x001EFC},+{0x001EFF, 0x001EFE},+{0x001F00, 0x001F08},+{0x001F01, 0x001F09},+{0x001F02, 0x001F0A},+{0x001F03, 0x001F0B},+{0x001F04, 0x001F0C},+{0x001F05, 0x001F0D},+{0x001F06, 0x001F0E},+{0x001F07, 0x001F0F},+{0x001F10, 0x001F18},+{0x001F11, 0x001F19},+{0x001F12, 0x001F1A},+{0x001F13, 0x001F1B},+{0x001F14, 0x001F1C},+{0x001F15, 0x001F1D},+{0x001F20, 0x001F28},+{0x001F21, 0x001F29},+{0x001F22, 0x001F2A},+{0x001F23, 0x001F2B},+{0x001F24, 0x001F2C},+{0x001F25, 0x001F2D},+{0x001F26, 0x001F2E},+{0x001F27, 0x001F2F},+{0x001F30, 0x001F38},+{0x001F31, 0x001F39},+{0x001F32, 0x001F3A},+{0x001F33, 0x001F3B},+{0x001F34, 0x001F3C},+{0x001F35, 0x001F3D},+{0x001F36, 0x001F3E},+{0x001F37, 0x001F3F},+{0x001F40, 0x001F48},+{0x001F41, 0x001F49},+{0x001F42, 0x001F4A},+{0x001F43, 0x001F4B},+{0x001F44, 0x001F4C},+{0x001F45, 0x001F4D},+{0x001F50, 0x0003A5},+{0x001F51, 0x001F59},+{0x001F52, 0x0003A5},+{0x001F53, 0x001F5B},+{0x001F54, 0x0003A5},+{0x001F55, 0x001F5D},+{0x001F56, 0x0003A5},+{0x001F57, 0x001F5F},+{0x001F60, 0x001F68},+{0x001F61, 0x001F69},+{0x001F62, 0x001F6A},+{0x001F63, 0x001F6B},+{0x001F64, 0x001F6C},+{0x001F65, 0x001F6D},+{0x001F66, 0x001F6E},+{0x001F67, 0x001F6F},+{0x001F70, 0x001FBA},+{0x001F71, 0x001FBB},+{0x001F72, 0x001FC8},+{0x001F73, 0x001FC9},+{0x001F74, 0x001FCA},+{0x001F75, 0x001FCB},+{0x001F76, 0x001FDA},+{0x001F77, 0x001FDB},+{0x001F78, 0x001FF8},+{0x001F79, 0x001FF9},+{0x001F7A, 0x001FEA},+{0x001F7B, 0x001FEB},+{0x001F7C, 0x001FFA},+{0x001F7D, 0x001FFB},+{0x001F80, 0x001F08},+{0x001F81, 0x001F09},+{0x001F82, 0x001F0A},+{0x001F83, 0x001F0B},+{0x001F84, 0x001F0C},+{0x001F85, 0x001F0D},+{0x001F86, 0x001F0E},+{0x001F87, 0x001F0F},+{0x001F88, 0x001F08},+{0x001F89, 0x001F09},+{0x001F8A, 0x001F0A},+{0x001F8B, 0x001F0B},+{0x001F8C, 0x001F0C},+{0x001F8D, 0x001F0D},+{0x001F8E, 0x001F0E},+{0x001F8F, 0x001F0F},+{0x001F90, 0x001F28},+{0x001F91, 0x001F29},+{0x001F92, 0x001F2A},+{0x001F93, 0x001F2B},+{0x001F94, 0x001F2C},+{0x001F95, 0x001F2D},+{0x001F96, 0x001F2E},+{0x001F97, 0x001F2F},+{0x001F98, 0x001F28},+{0x001F99, 0x001F29},+{0x001F9A, 0x001F2A},+{0x001F9B, 0x001F2B},+{0x001F9C, 0x001F2C},+{0x001F9D, 0x001F2D},+{0x001F9E, 0x001F2E},+{0x001F9F, 0x001F2F},+{0x001FA0, 0x001F68},+{0x001FA1, 0x001F69},+{0x001FA2, 0x001F6A},+{0x001FA3, 0x001F6B},+{0x001FA4, 0x001F6C},+{0x001FA5, 0x001F6D},+{0x001FA6, 0x001F6E},+{0x001FA7, 0x001F6F},+{0x001FA8, 0x001F68},+{0x001FA9, 0x001F69},+{0x001FAA, 0x001F6A},+{0x001FAB, 0x001F6B},+{0x001FAC, 0x001F6C},+{0x001FAD, 0x001F6D},+{0x001FAE, 0x001F6E},+{0x001FAF, 0x001F6F},+{0x001FB0, 0x001FB8},+{0x001FB1, 0x001FB9},+{0x001FB2, 0x001FBA},+{0x001FB3, 0x000391},+{0x001FB4, 0x000386},+{0x001FB6, 0x000391},+{0x001FB7, 0x000391},+{0x001FBC, 0x000391},+{0x001FBE, 0x000399},+{0x001FC2, 0x001FCA},+{0x001FC3, 0x000397},+{0x001FC4, 0x000389},+{0x001FC6, 0x000397},+{0x001FC7, 0x000397},+{0x001FCC, 0x000397},+{0x001FD0, 0x001FD8},+{0x001FD1, 0x001FD9},+{0x001FD2, 0x000399},+{0x001FD3, 0x000399},+{0x001FD6, 0x000399},+{0x001FD7, 0x000399},+{0x001FE0, 0x001FE8},+{0x001FE1, 0x001FE9},+{0x001FE2, 0x0003A5},+{0x001FE3, 0x0003A5},+{0x001FE4, 0x0003A1},+{0x001FE5, 0x001FEC},+{0x001FE6, 0x0003A5},+{0x001FE7, 0x0003A5},+{0x001FF2, 0x001FFA},+{0x001FF3, 0x0003A9},+{0x001FF4, 0x00038F},+{0x001FF6, 0x0003A9},+{0x001FF7, 0x0003A9},+{0x001FFC, 0x0003A9},+{0x00214E, 0x002132},+{0x002170, 0x002160},+{0x002171, 0x002161},+{0x002172, 0x002162},+{0x002173, 0x002163},+{0x002174, 0x002164},+{0x002175, 0x002165},+{0x002176, 0x002166},+{0x002177, 0x002167},+{0x002178, 0x002168},+{0x002179, 0x002169},+{0x00217A, 0x00216A},+{0x00217B, 0x00216B},+{0x00217C, 0x00216C},+{0x00217D, 0x00216D},+{0x00217E, 0x00216E},+{0x00217F, 0x00216F},+{0x002184, 0x002183},+{0x0024D0, 0x0024B6},+{0x0024D1, 0x0024B7},+{0x0024D2, 0x0024B8},+{0x0024D3, 0x0024B9},+{0x0024D4, 0x0024BA},+{0x0024D5, 0x0024BB},+{0x0024D6, 0x0024BC},+{0x0024D7, 0x0024BD},+{0x0024D8, 0x0024BE},+{0x0024D9, 0x0024BF},+{0x0024DA, 0x0024C0},+{0x0024DB, 0x0024C1},+{0x0024DC, 0x0024C2},+{0x0024DD, 0x0024C3},+{0x0024DE, 0x0024C4},+{0x0024DF, 0x0024C5},+{0x0024E0, 0x0024C6},+{0x0024E1, 0x0024C7},+{0x0024E2, 0x0024C8},+{0x0024E3, 0x0024C9},+{0x0024E4, 0x0024CA},+{0x0024E5, 0x0024CB},+{0x0024E6, 0x0024CC},+{0x0024E7, 0x0024CD},+{0x0024E8, 0x0024CE},+{0x0024E9, 0x0024CF},+{0x002C30, 0x002C00},+{0x002C31, 0x002C01},+{0x002C32, 0x002C02},+{0x002C33, 0x002C03},+{0x002C34, 0x002C04},+{0x002C35, 0x002C05},+{0x002C36, 0x002C06},+{0x002C37, 0x002C07},+{0x002C38, 0x002C08},+{0x002C39, 0x002C09},+{0x002C3A, 0x002C0A},+{0x002C3B, 0x002C0B},+{0x002C3C, 0x002C0C},+{0x002C3D, 0x002C0D},+{0x002C3E, 0x002C0E},+{0x002C3F, 0x002C0F},+{0x002C40, 0x002C10},+{0x002C41, 0x002C11},+{0x002C42, 0x002C12},+{0x002C43, 0x002C13},+{0x002C44, 0x002C14},+{0x002C45, 0x002C15},+{0x002C46, 0x002C16},+{0x002C47, 0x002C17},+{0x002C48, 0x002C18},+{0x002C49, 0x002C19},+{0x002C4A, 0x002C1A},+{0x002C4B, 0x002C1B},+{0x002C4C, 0x002C1C},+{0x002C4D, 0x002C1D},+{0x002C4E, 0x002C1E},+{0x002C4F, 0x002C1F},+{0x002C50, 0x002C20},+{0x002C51, 0x002C21},+{0x002C52, 0x002C22},+{0x002C53, 0x002C23},+{0x002C54, 0x002C24},+{0x002C55, 0x002C25},+{0x002C56, 0x002C26},+{0x002C57, 0x002C27},+{0x002C58, 0x002C28},+{0x002C59, 0x002C29},+{0x002C5A, 0x002C2A},+{0x002C5B, 0x002C2B},+{0x002C5C, 0x002C2C},+{0x002C5D, 0x002C2D},+{0x002C5E, 0x002C2E},+{0x002C61, 0x002C60},+{0x002C65, 0x00023A},+{0x002C66, 0x00023E},+{0x002C68, 0x002C67},+{0x002C6A, 0x002C69},+{0x002C6C, 0x002C6B},+{0x002C73, 0x002C72},+{0x002C76, 0x002C75},+{0x002C81, 0x002C80},+{0x002C83, 0x002C82},+{0x002C85, 0x002C84},+{0x002C87, 0x002C86},+{0x002C89, 0x002C88},+{0x002C8B, 0x002C8A},+{0x002C8D, 0x002C8C},+{0x002C8F, 0x002C8E},+{0x002C91, 0x002C90},+{0x002C93, 0x002C92},+{0x002C95, 0x002C94},+{0x002C97, 0x002C96},+{0x002C99, 0x002C98},+{0x002C9B, 0x002C9A},+{0x002C9D, 0x002C9C},+{0x002C9F, 0x002C9E},+{0x002CA1, 0x002CA0},+{0x002CA3, 0x002CA2},+{0x002CA5, 0x002CA4},+{0x002CA7, 0x002CA6},+{0x002CA9, 0x002CA8},+{0x002CAB, 0x002CAA},+{0x002CAD, 0x002CAC},+{0x002CAF, 0x002CAE},+{0x002CB1, 0x002CB0},+{0x002CB3, 0x002CB2},+{0x002CB5, 0x002CB4},+{0x002CB7, 0x002CB6},+{0x002CB9, 0x002CB8},+{0x002CBB, 0x002CBA},+{0x002CBD, 0x002CBC},+{0x002CBF, 0x002CBE},+{0x002CC1, 0x002CC0},+{0x002CC3, 0x002CC2},+{0x002CC5, 0x002CC4},+{0x002CC7, 0x002CC6},+{0x002CC9, 0x002CC8},+{0x002CCB, 0x002CCA},+{0x002CCD, 0x002CCC},+{0x002CCF, 0x002CCE},+{0x002CD1, 0x002CD0},+{0x002CD3, 0x002CD2},+{0x002CD5, 0x002CD4},+{0x002CD7, 0x002CD6},+{0x002CD9, 0x002CD8},+{0x002CDB, 0x002CDA},+{0x002CDD, 0x002CDC},+{0x002CDF, 0x002CDE},+{0x002CE1, 0x002CE0},+{0x002CE3, 0x002CE2},+{0x002CEC, 0x002CEB},+{0x002CEE, 0x002CED},+{0x002CF3, 0x002CF2},+{0x002D00, 0x0010A0},+{0x002D01, 0x0010A1},+{0x002D02, 0x0010A2},+{0x002D03, 0x0010A3},+{0x002D04, 0x0010A4},+{0x002D05, 0x0010A5},+{0x002D06, 0x0010A6},+{0x002D07, 0x0010A7},+{0x002D08, 0x0010A8},+{0x002D09, 0x0010A9},+{0x002D0A, 0x0010AA},+{0x002D0B, 0x0010AB},+{0x002D0C, 0x0010AC},+{0x002D0D, 0x0010AD},+{0x002D0E, 0x0010AE},+{0x002D0F, 0x0010AF},+{0x002D10, 0x0010B0},+{0x002D11, 0x0010B1},+{0x002D12, 0x0010B2},+{0x002D13, 0x0010B3},+{0x002D14, 0x0010B4},+{0x002D15, 0x0010B5},+{0x002D16, 0x0010B6},+{0x002D17, 0x0010B7},+{0x002D18, 0x0010B8},+{0x002D19, 0x0010B9},+{0x002D1A, 0x0010BA},+{0x002D1B, 0x0010BB},+{0x002D1C, 0x0010BC},+{0x002D1D, 0x0010BD},+{0x002D1E, 0x0010BE},+{0x002D1F, 0x0010BF},+{0x002D20, 0x0010C0},+{0x002D21, 0x0010C1},+{0x002D22, 0x0010C2},+{0x002D23, 0x0010C3},+{0x002D24, 0x0010C4},+{0x002D25, 0x0010C5},+{0x002D27, 0x0010C7},+{0x002D2D, 0x0010CD},+{0x00A641, 0x00A640},+{0x00A643, 0x00A642},+{0x00A645, 0x00A644},+{0x00A647, 0x00A646},+{0x00A649, 0x00A648},+{0x00A64B, 0x00A64A},+{0x00A64D, 0x00A64C},+{0x00A64F, 0x00A64E},+{0x00A651, 0x00A650},+{0x00A653, 0x00A652},+{0x00A655, 0x00A654},+{0x00A657, 0x00A656},+{0x00A659, 0x00A658},+{0x00A65B, 0x00A65A},+{0x00A65D, 0x00A65C},+{0x00A65F, 0x00A65E},+{0x00A661, 0x00A660},+{0x00A663, 0x00A662},+{0x00A665, 0x00A664},+{0x00A667, 0x00A666},+{0x00A669, 0x00A668},+{0x00A66B, 0x00A66A},+{0x00A66D, 0x00A66C},+{0x00A681, 0x00A680},+{0x00A683, 0x00A682},+{0x00A685, 0x00A684},+{0x00A687, 0x00A686},+{0x00A689, 0x00A688},+{0x00A68B, 0x00A68A},+{0x00A68D, 0x00A68C},+{0x00A68F, 0x00A68E},+{0x00A691, 0x00A690},+{0x00A693, 0x00A692},+{0x00A695, 0x00A694},+{0x00A697, 0x00A696},+{0x00A699, 0x00A698},+{0x00A69B, 0x00A69A},+{0x00A723, 0x00A722},+{0x00A725, 0x00A724},+{0x00A727, 0x00A726},+{0x00A729, 0x00A728},+{0x00A72B, 0x00A72A},+{0x00A72D, 0x00A72C},+{0x00A72F, 0x00A72E},+{0x00A733, 0x00A732},+{0x00A735, 0x00A734},+{0x00A737, 0x00A736},+{0x00A739, 0x00A738},+{0x00A73B, 0x00A73A},+{0x00A73D, 0x00A73C},+{0x00A73F, 0x00A73E},+{0x00A741, 0x00A740},+{0x00A743, 0x00A742},+{0x00A745, 0x00A744},+{0x00A747, 0x00A746},+{0x00A749, 0x00A748},+{0x00A74B, 0x00A74A},+{0x00A74D, 0x00A74C},+{0x00A74F, 0x00A74E},+{0x00A751, 0x00A750},+{0x00A753, 0x00A752},+{0x00A755, 0x00A754},+{0x00A757, 0x00A756},+{0x00A759, 0x00A758},+{0x00A75B, 0x00A75A},+{0x00A75D, 0x00A75C},+{0x00A75F, 0x00A75E},+{0x00A761, 0x00A760},+{0x00A763, 0x00A762},+{0x00A765, 0x00A764},+{0x00A767, 0x00A766},+{0x00A769, 0x00A768},+{0x00A76B, 0x00A76A},+{0x00A76D, 0x00A76C},+{0x00A76F, 0x00A76E},+{0x00A77A, 0x00A779},+{0x00A77C, 0x00A77B},+{0x00A77F, 0x00A77E},+{0x00A781, 0x00A780},+{0x00A783, 0x00A782},+{0x00A785, 0x00A784},+{0x00A787, 0x00A786},+{0x00A78C, 0x00A78B},+{0x00A791, 0x00A790},+{0x00A793, 0x00A792},+{0x00A794, 0x00A7C4},+{0x00A797, 0x00A796},+{0x00A799, 0x00A798},+{0x00A79B, 0x00A79A},+{0x00A79D, 0x00A79C},+{0x00A79F, 0x00A79E},+{0x00A7A1, 0x00A7A0},+{0x00A7A3, 0x00A7A2},+{0x00A7A5, 0x00A7A4},+{0x00A7A7, 0x00A7A6},+{0x00A7A9, 0x00A7A8},+{0x00A7B5, 0x00A7B4},+{0x00A7B7, 0x00A7B6},+{0x00A7B9, 0x00A7B8},+{0x00A7BB, 0x00A7BA},+{0x00A7BD, 0x00A7BC},+{0x00A7BF, 0x00A7BE},+{0x00A7C3, 0x00A7C2},+{0x00A7C8, 0x00A7C7},+{0x00A7CA, 0x00A7C9},+{0x00A7F6, 0x00A7F5},+{0x00AB53, 0x00A7B3},+{0x00AB70, 0x0013A0},+{0x00AB71, 0x0013A1},+{0x00AB72, 0x0013A2},+{0x00AB73, 0x0013A3},+{0x00AB74, 0x0013A4},+{0x00AB75, 0x0013A5},+{0x00AB76, 0x0013A6},+{0x00AB77, 0x0013A7},+{0x00AB78, 0x0013A8},+{0x00AB79, 0x0013A9},+{0x00AB7A, 0x0013AA},+{0x00AB7B, 0x0013AB},+{0x00AB7C, 0x0013AC},+{0x00AB7D, 0x0013AD},+{0x00AB7E, 0x0013AE},+{0x00AB7F, 0x0013AF},+{0x00AB80, 0x0013B0},+{0x00AB81, 0x0013B1},+{0x00AB82, 0x0013B2},+{0x00AB83, 0x0013B3},+{0x00AB84, 0x0013B4},+{0x00AB85, 0x0013B5},+{0x00AB86, 0x0013B6},+{0x00AB87, 0x0013B7},+{0x00AB88, 0x0013B8},+{0x00AB89, 0x0013B9},+{0x00AB8A, 0x0013BA},+{0x00AB8B, 0x0013BB},+{0x00AB8C, 0x0013BC},+{0x00AB8D, 0x0013BD},+{0x00AB8E, 0x0013BE},+{0x00AB8F, 0x0013BF},+{0x00AB90, 0x0013C0},+{0x00AB91, 0x0013C1},+{0x00AB92, 0x0013C2},+{0x00AB93, 0x0013C3},+{0x00AB94, 0x0013C4},+{0x00AB95, 0x0013C5},+{0x00AB96, 0x0013C6},+{0x00AB97, 0x0013C7},+{0x00AB98, 0x0013C8},+{0x00AB99, 0x0013C9},+{0x00AB9A, 0x0013CA},+{0x00AB9B, 0x0013CB},+{0x00AB9C, 0x0013CC},+{0x00AB9D, 0x0013CD},+{0x00AB9E, 0x0013CE},+{0x00AB9F, 0x0013CF},+{0x00ABA0, 0x0013D0},+{0x00ABA1, 0x0013D1},+{0x00ABA2, 0x0013D2},+{0x00ABA3, 0x0013D3},+{0x00ABA4, 0x0013D4},+{0x00ABA5, 0x0013D5},+{0x00ABA6, 0x0013D6},+{0x00ABA7, 0x0013D7},+{0x00ABA8, 0x0013D8},+{0x00ABA9, 0x0013D9},+{0x00ABAA, 0x0013DA},+{0x00ABAB, 0x0013DB},+{0x00ABAC, 0x0013DC},+{0x00ABAD, 0x0013DD},+{0x00ABAE, 0x0013DE},+{0x00ABAF, 0x0013DF},+{0x00ABB0, 0x0013E0},+{0x00ABB1, 0x0013E1},+{0x00ABB2, 0x0013E2},+{0x00ABB3, 0x0013E3},+{0x00ABB4, 0x0013E4},+{0x00ABB5, 0x0013E5},+{0x00ABB6, 0x0013E6},+{0x00ABB7, 0x0013E7},+{0x00ABB8, 0x0013E8},+{0x00ABB9, 0x0013E9},+{0x00ABBA, 0x0013EA},+{0x00ABBB, 0x0013EB},+{0x00ABBC, 0x0013EC},+{0x00ABBD, 0x0013ED},+{0x00ABBE, 0x0013EE},+{0x00ABBF, 0x0013EF},+{0x00FB00, 0x000046},+{0x00FB01, 0x000046},+{0x00FB02, 0x000046},+{0x00FB03, 0x000046},+{0x00FB04, 0x000046},+{0x00FB05, 0x000053},+{0x00FB06, 0x000053},+{0x00FB13, 0x000544},+{0x00FB14, 0x000544},+{0x00FB15, 0x000544},+{0x00FB16, 0x00054E},+{0x00FB17, 0x000544},+{0x00FF41, 0x00FF21},+{0x00FF42, 0x00FF22},+{0x00FF43, 0x00FF23},+{0x00FF44, 0x00FF24},+{0x00FF45, 0x00FF25},+{0x00FF46, 0x00FF26},+{0x00FF47, 0x00FF27},+{0x00FF48, 0x00FF28},+{0x00FF49, 0x00FF29},+{0x00FF4A, 0x00FF2A},+{0x00FF4B, 0x00FF2B},+{0x00FF4C, 0x00FF2C},+{0x00FF4D, 0x00FF2D},+{0x00FF4E, 0x00FF2E},+{0x00FF4F, 0x00FF2F},+{0x00FF50, 0x00FF30},+{0x00FF51, 0x00FF31},+{0x00FF52, 0x00FF32},+{0x00FF53, 0x00FF33},+{0x00FF54, 0x00FF34},+{0x00FF55, 0x00FF35},+{0x00FF56, 0x00FF36},+{0x00FF57, 0x00FF37},+{0x00FF58, 0x00FF38},+{0x00FF59, 0x00FF39},+{0x00FF5A, 0x00FF3A},+{0x010428, 0x010400},+{0x010429, 0x010401},+{0x01042A, 0x010402},+{0x01042B, 0x010403},+{0x01042C, 0x010404},+{0x01042D, 0x010405},+{0x01042E, 0x010406},+{0x01042F, 0x010407},+{0x010430, 0x010408},+{0x010431, 0x010409},+{0x010432, 0x01040A},+{0x010433, 0x01040B},+{0x010434, 0x01040C},+{0x010435, 0x01040D},+{0x010436, 0x01040E},+{0x010437, 0x01040F},+{0x010438, 0x010410},+{0x010439, 0x010411},+{0x01043A, 0x010412},+{0x01043B, 0x010413},+{0x01043C, 0x010414},+{0x01043D, 0x010415},+{0x01043E, 0x010416},+{0x01043F, 0x010417},+{0x010440, 0x010418},+{0x010441, 0x010419},+{0x010442, 0x01041A},+{0x010443, 0x01041B},+{0x010444, 0x01041C},+{0x010445, 0x01041D},+{0x010446, 0x01041E},+{0x010447, 0x01041F},+{0x010448, 0x010420},+{0x010449, 0x010421},+{0x01044A, 0x010422},+{0x01044B, 0x010423},+{0x01044C, 0x010424},+{0x01044D, 0x010425},+{0x01044E, 0x010426},+{0x01044F, 0x010427},+{0x0104D8, 0x0104B0},+{0x0104D9, 0x0104B1},+{0x0104DA, 0x0104B2},+{0x0104DB, 0x0104B3},+{0x0104DC, 0x0104B4},+{0x0104DD, 0x0104B5},+{0x0104DE, 0x0104B6},+{0x0104DF, 0x0104B7},+{0x0104E0, 0x0104B8},+{0x0104E1, 0x0104B9},+{0x0104E2, 0x0104BA},+{0x0104E3, 0x0104BB},+{0x0104E4, 0x0104BC},+{0x0104E5, 0x0104BD},+{0x0104E6, 0x0104BE},+{0x0104E7, 0x0104BF},+{0x0104E8, 0x0104C0},+{0x0104E9, 0x0104C1},+{0x0104EA, 0x0104C2},+{0x0104EB, 0x0104C3},+{0x0104EC, 0x0104C4},+{0x0104ED, 0x0104C5},+{0x0104EE, 0x0104C6},+{0x0104EF, 0x0104C7},+{0x0104F0, 0x0104C8},+{0x0104F1, 0x0104C9},+{0x0104F2, 0x0104CA},+{0x0104F3, 0x0104CB},+{0x0104F4, 0x0104CC},+{0x0104F5, 0x0104CD},+{0x0104F6, 0x0104CE},+{0x0104F7, 0x0104CF},+{0x0104F8, 0x0104D0},+{0x0104F9, 0x0104D1},+{0x0104FA, 0x0104D2},+{0x0104FB, 0x0104D3},+{0x010CC0, 0x010C80},+{0x010CC1, 0x010C81},+{0x010CC2, 0x010C82},+{0x010CC3, 0x010C83},+{0x010CC4, 0x010C84},+{0x010CC5, 0x010C85},+{0x010CC6, 0x010C86},+{0x010CC7, 0x010C87},+{0x010CC8, 0x010C88},+{0x010CC9, 0x010C89},+{0x010CCA, 0x010C8A},+{0x010CCB, 0x010C8B},+{0x010CCC, 0x010C8C},+{0x010CCD, 0x010C8D},+{0x010CCE, 0x010C8E},+{0x010CCF, 0x010C8F},+{0x010CD0, 0x010C90},+{0x010CD1, 0x010C91},+{0x010CD2, 0x010C92},+{0x010CD3, 0x010C93},+{0x010CD4, 0x010C94},+{0x010CD5, 0x010C95},+{0x010CD6, 0x010C96},+{0x010CD7, 0x010C97},+{0x010CD8, 0x010C98},+{0x010CD9, 0x010C99},+{0x010CDA, 0x010C9A},+{0x010CDB, 0x010C9B},+{0x010CDC, 0x010C9C},+{0x010CDD, 0x010C9D},+{0x010CDE, 0x010C9E},+{0x010CDF, 0x010C9F},+{0x010CE0, 0x010CA0},+{0x010CE1, 0x010CA1},+{0x010CE2, 0x010CA2},+{0x010CE3, 0x010CA3},+{0x010CE4, 0x010CA4},+{0x010CE5, 0x010CA5},+{0x010CE6, 0x010CA6},+{0x010CE7, 0x010CA7},+{0x010CE8, 0x010CA8},+{0x010CE9, 0x010CA9},+{0x010CEA, 0x010CAA},+{0x010CEB, 0x010CAB},+{0x010CEC, 0x010CAC},+{0x010CED, 0x010CAD},+{0x010CEE, 0x010CAE},+{0x010CEF, 0x010CAF},+{0x010CF0, 0x010CB0},+{0x010CF1, 0x010CB1},+{0x010CF2, 0x010CB2},+{0x0118C0, 0x0118A0},+{0x0118C1, 0x0118A1},+{0x0118C2, 0x0118A2},+{0x0118C3, 0x0118A3},+{0x0118C4, 0x0118A4},+{0x0118C5, 0x0118A5},+{0x0118C6, 0x0118A6},+{0x0118C7, 0x0118A7},+{0x0118C8, 0x0118A8},+{0x0118C9, 0x0118A9},+{0x0118CA, 0x0118AA},+{0x0118CB, 0x0118AB},+{0x0118CC, 0x0118AC},+{0x0118CD, 0x0118AD},+{0x0118CE, 0x0118AE},+{0x0118CF, 0x0118AF},+{0x0118D0, 0x0118B0},+{0x0118D1, 0x0118B1},+{0x0118D2, 0x0118B2},+{0x0118D3, 0x0118B3},+{0x0118D4, 0x0118B4},+{0x0118D5, 0x0118B5},+{0x0118D6, 0x0118B6},+{0x0118D7, 0x0118B7},+{0x0118D8, 0x0118B8},+{0x0118D9, 0x0118B9},+{0x0118DA, 0x0118BA},+{0x0118DB, 0x0118BB},+{0x0118DC, 0x0118BC},+{0x0118DD, 0x0118BD},+{0x0118DE, 0x0118BE},+{0x0118DF, 0x0118BF},+{0x016E60, 0x016E40},+{0x016E61, 0x016E41},+{0x016E62, 0x016E42},+{0x016E63, 0x016E43},+{0x016E64, 0x016E44},+{0x016E65, 0x016E45},+{0x016E66, 0x016E46},+{0x016E67, 0x016E47},+{0x016E68, 0x016E48},+{0x016E69, 0x016E49},+{0x016E6A, 0x016E4A},+{0x016E6B, 0x016E4B},+{0x016E6C, 0x016E4C},+{0x016E6D, 0x016E4D},+{0x016E6E, 0x016E4E},+{0x016E6F, 0x016E4F},+{0x016E70, 0x016E50},+{0x016E71, 0x016E51},+{0x016E72, 0x016E52},+{0x016E73, 0x016E53},+{0x016E74, 0x016E54},+{0x016E75, 0x016E55},+{0x016E76, 0x016E56},+{0x016E77, 0x016E57},+{0x016E78, 0x016E58},+{0x016E79, 0x016E59},+{0x016E7A, 0x016E5A},+{0x016E7B, 0x016E5B},+{0x016E7C, 0x016E5C},+{0x016E7D, 0x016E5D},+{0x016E7E, 0x016E5E},+{0x016E7F, 0x016E5F},+{0x01E922, 0x01E900},+{0x01E923, 0x01E901},+{0x01E924, 0x01E902},+{0x01E925, 0x01E903},+{0x01E926, 0x01E904},+{0x01E927, 0x01E905},+{0x01E928, 0x01E906},+{0x01E929, 0x01E907},+{0x01E92A, 0x01E908},+{0x01E92B, 0x01E909},+{0x01E92C, 0x01E90A},+{0x01E92D, 0x01E90B},+{0x01E92E, 0x01E90C},+{0x01E92F, 0x01E90D},+{0x01E930, 0x01E90E},+{0x01E931, 0x01E90F},+{0x01E932, 0x01E910},+{0x01E933, 0x01E911},+{0x01E934, 0x01E912},+{0x01E935, 0x01E913},+{0x01E936, 0x01E914},+{0x01E937, 0x01E915},+{0x01E938, 0x01E916},+{0x01E939, 0x01E917},+{0x01E93A, 0x01E918},+{0x01E93B, 0x01E919},+{0x01E93C, 0x01E91A},+{0x01E93D, 0x01E91B},+{0x01E93E, 0x01E91C},+{0x01E93F, 0x01E91D},+{0x01E940, 0x01E91E},+{0x01E941, 0x01E91F},+{0x01E942, 0x01E920},+{0x01E943, 0x01E921},};-const std::multimap<uint32_t, uint32_t> unicode_map_nfd = {-{0x000000C0, 0x00000041}, {0x000000C0, 0x00000300}, {0x000000C1, 0x00000041}, {0x000000C1, 0x00000301},-{0x000000C2, 0x00000041}, {0x000000C2, 0x00000302}, {0x000000C3, 0x00000041}, {0x000000C3, 0x00000303},-{0x000000C4, 0x00000041}, {0x000000C4, 0x00000308}, {0x000000C5, 0x00000041}, {0x000000C5, 0x0000030A},-{0x000000C7, 0x00000043}, {0x000000C7, 0x00000327}, {0x000000C8, 0x00000045}, {0x000000C8, 0x00000300},-{0x000000C9, 0x00000045}, {0x000000C9, 0x00000301}, {0x000000CA, 0x00000045}, {0x000000CA, 0x00000302},-{0x000000CB, 0x00000045}, {0x000000CB, 0x00000308}, {0x000000CC, 0x00000049}, {0x000000CC, 0x00000300},-{0x000000CD, 0x00000049}, {0x000000CD, 0x00000301}, {0x000000CE, 0x00000049}, {0x000000CE, 0x00000302},-{0x000000CF, 0x00000049}, {0x000000CF, 0x00000308}, {0x000000D1, 0x0000004E}, {0x000000D1, 0x00000303},-{0x000000D2, 0x0000004F}, {0x000000D2, 0x00000300}, {0x000000D3, 0x0000004F}, {0x000000D3, 0x00000301},-{0x000000D4, 0x0000004F}, {0x000000D4, 0x00000302}, {0x000000D5, 0x0000004F}, {0x000000D5, 0x00000303},-{0x000000D6, 0x0000004F}, {0x000000D6, 0x00000308}, {0x000000D9, 0x00000055}, {0x000000D9, 0x00000300},-{0x000000DA, 0x00000055}, {0x000000DA, 0x00000301}, {0x000000DB, 0x00000055}, {0x000000DB, 0x00000302},-{0x000000DC, 0x00000055}, {0x000000DC, 0x00000308}, {0x000000DD, 0x00000059}, {0x000000DD, 0x00000301},-{0x000000E0, 0x00000061}, {0x000000E0, 0x00000300}, {0x000000E1, 0x00000061}, {0x000000E1, 0x00000301},-{0x000000E2, 0x00000061}, {0x000000E2, 0x00000302}, {0x000000E3, 0x00000061}, {0x000000E3, 0x00000303},-{0x000000E4, 0x00000061}, {0x000000E4, 0x00000308}, {0x000000E5, 0x00000061}, {0x000000E5, 0x0000030A},-{0x000000E7, 0x00000063}, {0x000000E7, 0x00000327}, {0x000000E8, 0x00000065}, {0x000000E8, 0x00000300},-{0x000000E9, 0x00000065}, {0x000000E9, 0x00000301}, {0x000000EA, 0x00000065}, {0x000000EA, 0x00000302},-{0x000000EB, 0x00000065}, {0x000000EB, 0x00000308}, {0x000000EC, 0x00000069}, {0x000000EC, 0x00000300},-{0x000000ED, 0x00000069}, {0x000000ED, 0x00000301}, {0x000000EE, 0x00000069}, {0x000000EE, 0x00000302},-{0x000000EF, 0x00000069}, {0x000000EF, 0x00000308}, {0x000000F1, 0x0000006E}, {0x000000F1, 0x00000303},-{0x000000F2, 0x0000006F}, {0x000000F2, 0x00000300}, {0x000000F3, 0x0000006F}, {0x000000F3, 0x00000301},-{0x000000F4, 0x0000006F}, {0x000000F4, 0x00000302}, {0x000000F5, 0x0000006F}, {0x000000F5, 0x00000303},-{0x000000F6, 0x0000006F}, {0x000000F6, 0x00000308}, {0x000000F9, 0x00000075}, {0x000000F9, 0x00000300},-{0x000000FA, 0x00000075}, {0x000000FA, 0x00000301}, {0x000000FB, 0x00000075}, {0x000000FB, 0x00000302},-{0x000000FC, 0x00000075}, {0x000000FC, 0x00000308}, {0x000000FD, 0x00000079}, {0x000000FD, 0x00000301},-{0x000000FF, 0x00000079}, {0x000000FF, 0x00000308}, {0x00000100, 0x00000041}, {0x00000100, 0x00000304},-{0x00000101, 0x00000061}, {0x00000101, 0x00000304}, {0x00000102, 0x00000041}, {0x00000102, 0x00000306},-{0x00000103, 0x00000061}, {0x00000103, 0x00000306}, {0x00000104, 0x00000041}, {0x00000104, 0x00000328},-{0x00000105, 0x00000061}, {0x00000105, 0x00000328}, {0x00000106, 0x00000043}, {0x00000106, 0x00000301},-{0x00000107, 0x00000063}, {0x00000107, 0x00000301}, {0x00000108, 0x00000043}, {0x00000108, 0x00000302},-{0x00000109, 0x00000063}, {0x00000109, 0x00000302}, {0x0000010A, 0x00000043}, {0x0000010A, 0x00000307},-{0x0000010B, 0x00000063}, {0x0000010B, 0x00000307}, {0x0000010C, 0x00000043}, {0x0000010C, 0x0000030C},-{0x0000010D, 0x00000063}, {0x0000010D, 0x0000030C}, {0x0000010E, 0x00000044}, {0x0000010E, 0x0000030C},-{0x0000010F, 0x00000064}, {0x0000010F, 0x0000030C}, {0x00000112, 0x00000045}, {0x00000112, 0x00000304},-{0x00000113, 0x00000065}, {0x00000113, 0x00000304}, {0x00000114, 0x00000045}, {0x00000114, 0x00000306},-{0x00000115, 0x00000065}, {0x00000115, 0x00000306}, {0x00000116, 0x00000045}, {0x00000116, 0x00000307},-{0x00000117, 0x00000065}, {0x00000117, 0x00000307}, {0x00000118, 0x00000045}, {0x00000118, 0x00000328},-{0x00000119, 0x00000065}, {0x00000119, 0x00000328}, {0x0000011A, 0x00000045}, {0x0000011A, 0x0000030C},-{0x0000011B, 0x00000065}, {0x0000011B, 0x0000030C}, {0x0000011C, 0x00000047}, {0x0000011C, 0x00000302},-{0x0000011D, 0x00000067}, {0x0000011D, 0x00000302}, {0x0000011E, 0x00000047}, {0x0000011E, 0x00000306},-{0x0000011F, 0x00000067}, {0x0000011F, 0x00000306}, {0x00000120, 0x00000047}, {0x00000120, 0x00000307},-{0x00000121, 0x00000067}, {0x00000121, 0x00000307}, {0x00000122, 0x00000047}, {0x00000122, 0x00000327},-{0x00000123, 0x00000067}, {0x00000123, 0x00000327}, {0x00000124, 0x00000048}, {0x00000124, 0x00000302},-{0x00000125, 0x00000068}, {0x00000125, 0x00000302}, {0x00000128, 0x00000049}, {0x00000128, 0x00000303},-{0x00000129, 0x00000069}, {0x00000129, 0x00000303}, {0x0000012A, 0x00000049}, {0x0000012A, 0x00000304},-{0x0000012B, 0x00000069}, {0x0000012B, 0x00000304}, {0x0000012C, 0x00000049}, {0x0000012C, 0x00000306},-{0x0000012D, 0x00000069}, {0x0000012D, 0x00000306}, {0x0000012E, 0x00000049}, {0x0000012E, 0x00000328},-{0x0000012F, 0x00000069}, {0x0000012F, 0x00000328}, {0x00000130, 0x00000049}, {0x00000130, 0x00000307},-{0x00000134, 0x0000004A}, {0x00000134, 0x00000302}, {0x00000135, 0x0000006A}, {0x00000135, 0x00000302},-{0x00000136, 0x0000004B}, {0x00000136, 0x00000327}, {0x00000137, 0x0000006B}, {0x00000137, 0x00000327},-{0x00000139, 0x0000004C}, {0x00000139, 0x00000301}, {0x0000013A, 0x0000006C}, {0x0000013A, 0x00000301},-{0x0000013B, 0x0000004C}, {0x0000013B, 0x00000327}, {0x0000013C, 0x0000006C}, {0x0000013C, 0x00000327},-{0x0000013D, 0x0000004C}, {0x0000013D, 0x0000030C}, {0x0000013E, 0x0000006C}, {0x0000013E, 0x0000030C},-{0x00000143, 0x0000004E}, {0x00000143, 0x00000301}, {0x00000144, 0x0000006E}, {0x00000144, 0x00000301},-{0x00000145, 0x0000004E}, {0x00000145, 0x00000327}, {0x00000146, 0x0000006E}, {0x00000146, 0x00000327},-{0x00000147, 0x0000004E}, {0x00000147, 0x0000030C}, {0x00000148, 0x0000006E}, {0x00000148, 0x0000030C},-{0x0000014C, 0x0000004F}, {0x0000014C, 0x00000304}, {0x0000014D, 0x0000006F}, {0x0000014D, 0x00000304},-{0x0000014E, 0x0000004F}, {0x0000014E, 0x00000306}, {0x0000014F, 0x0000006F}, {0x0000014F, 0x00000306},-{0x00000150, 0x0000004F}, {0x00000150, 0x0000030B}, {0x00000151, 0x0000006F}, {0x00000151, 0x0000030B},-{0x00000154, 0x00000052}, {0x00000154, 0x00000301}, {0x00000155, 0x00000072}, {0x00000155, 0x00000301},-{0x00000156, 0x00000052}, {0x00000156, 0x00000327}, {0x00000157, 0x00000072}, {0x00000157, 0x00000327},-{0x00000158, 0x00000052}, {0x00000158, 0x0000030C}, {0x00000159, 0x00000072}, {0x00000159, 0x0000030C},-{0x0000015A, 0x00000053}, {0x0000015A, 0x00000301}, {0x0000015B, 0x00000073}, {0x0000015B, 0x00000301},-{0x0000015C, 0x00000053}, {0x0000015C, 0x00000302}, {0x0000015D, 0x00000073}, {0x0000015D, 0x00000302},-{0x0000015E, 0x00000053}, {0x0000015E, 0x00000327}, {0x0000015F, 0x00000073}, {0x0000015F, 0x00000327},-{0x00000160, 0x00000053}, {0x00000160, 0x0000030C}, {0x00000161, 0x00000073}, {0x00000161, 0x0000030C},-{0x00000162, 0x00000054}, {0x00000162, 0x00000327}, {0x00000163, 0x00000074}, {0x00000163, 0x00000327},-{0x00000164, 0x00000054}, {0x00000164, 0x0000030C}, {0x00000165, 0x00000074}, {0x00000165, 0x0000030C},-{0x00000168, 0x00000055}, {0x00000168, 0x00000303}, {0x00000169, 0x00000075}, {0x00000169, 0x00000303},-{0x0000016A, 0x00000055}, {0x0000016A, 0x00000304}, {0x0000016B, 0x00000075}, {0x0000016B, 0x00000304},-{0x0000016C, 0x00000055}, {0x0000016C, 0x00000306}, {0x0000016D, 0x00000075}, {0x0000016D, 0x00000306},-{0x0000016E, 0x00000055}, {0x0000016E, 0x0000030A}, {0x0000016F, 0x00000075}, {0x0000016F, 0x0000030A},-{0x00000170, 0x00000055}, {0x00000170, 0x0000030B}, {0x00000171, 0x00000075}, {0x00000171, 0x0000030B},-{0x00000172, 0x00000055}, {0x00000172, 0x00000328}, {0x00000173, 0x00000075}, {0x00000173, 0x00000328},-{0x00000174, 0x00000057}, {0x00000174, 0x00000302}, {0x00000175, 0x00000077}, {0x00000175, 0x00000302},-{0x00000176, 0x00000059}, {0x00000176, 0x00000302}, {0x00000177, 0x00000079}, {0x00000177, 0x00000302},-{0x00000178, 0x00000059}, {0x00000178, 0x00000308}, {0x00000179, 0x0000005A}, {0x00000179, 0x00000301},-{0x0000017A, 0x0000007A}, {0x0000017A, 0x00000301}, {0x0000017B, 0x0000005A}, {0x0000017B, 0x00000307},-{0x0000017C, 0x0000007A}, {0x0000017C, 0x00000307}, {0x0000017D, 0x0000005A}, {0x0000017D, 0x0000030C},-{0x0000017E, 0x0000007A}, {0x0000017E, 0x0000030C}, {0x000001A0, 0x0000004F}, {0x000001A0, 0x0000031B},-{0x000001A1, 0x0000006F}, {0x000001A1, 0x0000031B}, {0x000001AF, 0x00000055}, {0x000001AF, 0x0000031B},-{0x000001B0, 0x00000075}, {0x000001B0, 0x0000031B}, {0x000001CD, 0x00000041}, {0x000001CD, 0x0000030C},-{0x000001CE, 0x00000061}, {0x000001CE, 0x0000030C}, {0x000001CF, 0x00000049}, {0x000001CF, 0x0000030C},-{0x000001D0, 0x00000069}, {0x000001D0, 0x0000030C}, {0x000001D1, 0x0000004F}, {0x000001D1, 0x0000030C},-{0x000001D2, 0x0000006F}, {0x000001D2, 0x0000030C}, {0x000001D3, 0x00000055}, {0x000001D3, 0x0000030C},-{0x000001D4, 0x00000075}, {0x000001D4, 0x0000030C}, {0x000001D5, 0x00000055}, {0x000001D5, 0x00000308},-{0x000001D5, 0x00000304}, {0x000001D6, 0x00000075}, {0x000001D6, 0x00000308}, {0x000001D6, 0x00000304},-{0x000001D7, 0x00000055}, {0x000001D7, 0x00000308}, {0x000001D7, 0x00000301}, {0x000001D8, 0x00000075},-{0x000001D8, 0x00000308}, {0x000001D8, 0x00000301}, {0x000001D9, 0x00000055}, {0x000001D9, 0x00000308},-{0x000001D9, 0x0000030C}, {0x000001DA, 0x00000075}, {0x000001DA, 0x00000308}, {0x000001DA, 0x0000030C},-{0x000001DB, 0x00000055}, {0x000001DB, 0x00000308}, {0x000001DB, 0x00000300}, {0x000001DC, 0x00000075},-{0x000001DC, 0x00000308}, {0x000001DC, 0x00000300}, {0x000001DE, 0x00000041}, {0x000001DE, 0x00000308},-{0x000001DE, 0x00000304}, {0x000001DF, 0x00000061}, {0x000001DF, 0x00000308}, {0x000001DF, 0x00000304},-{0x000001E0, 0x00000041}, {0x000001E0, 0x00000307}, {0x000001E0, 0x00000304}, {0x000001E1, 0x00000061},-{0x000001E1, 0x00000307}, {0x000001E1, 0x00000304}, {0x000001E2, 0x000000C6}, {0x000001E2, 0x00000304},-{0x000001E3, 0x000000E6}, {0x000001E3, 0x00000304}, {0x000001E6, 0x00000047}, {0x000001E6, 0x0000030C},-{0x000001E7, 0x00000067}, {0x000001E7, 0x0000030C}, {0x000001E8, 0x0000004B}, {0x000001E8, 0x0000030C},-{0x000001E9, 0x0000006B}, {0x000001E9, 0x0000030C}, {0x000001EA, 0x0000004F}, {0x000001EA, 0x00000328},-{0x000001EB, 0x0000006F}, {0x000001EB, 0x00000328}, {0x000001EC, 0x0000004F}, {0x000001EC, 0x00000328},-{0x000001EC, 0x00000304}, {0x000001ED, 0x0000006F}, {0x000001ED, 0x00000328}, {0x000001ED, 0x00000304},-{0x000001EE, 0x000001B7}, {0x000001EE, 0x0000030C}, {0x000001EF, 0x00000292}, {0x000001EF, 0x0000030C},-{0x000001F0, 0x0000006A}, {0x000001F0, 0x0000030C}, {0x000001F4, 0x00000047}, {0x000001F4, 0x00000301},-{0x000001F5, 0x00000067}, {0x000001F5, 0x00000301}, {0x000001F8, 0x0000004E}, {0x000001F8, 0x00000300},-{0x000001F9, 0x0000006E}, {0x000001F9, 0x00000300}, {0x000001FA, 0x00000041}, {0x000001FA, 0x0000030A},-{0x000001FA, 0x00000301}, {0x000001FB, 0x00000061}, {0x000001FB, 0x0000030A}, {0x000001FB, 0x00000301},-{0x000001FC, 0x000000C6}, {0x000001FC, 0x00000301}, {0x000001FD, 0x000000E6}, {0x000001FD, 0x00000301},-{0x000001FE, 0x000000D8}, {0x000001FE, 0x00000301}, {0x000001FF, 0x000000F8}, {0x000001FF, 0x00000301},-{0x00000200, 0x00000041}, {0x00000200, 0x0000030F}, {0x00000201, 0x00000061}, {0x00000201, 0x0000030F},-{0x00000202, 0x00000041}, {0x00000202, 0x00000311}, {0x00000203, 0x00000061}, {0x00000203, 0x00000311},-{0x00000204, 0x00000045}, {0x00000204, 0x0000030F}, {0x00000205, 0x00000065}, {0x00000205, 0x0000030F},-{0x00000206, 0x00000045}, {0x00000206, 0x00000311}, {0x00000207, 0x00000065}, {0x00000207, 0x00000311},-{0x00000208, 0x00000049}, {0x00000208, 0x0000030F}, {0x00000209, 0x00000069}, {0x00000209, 0x0000030F},-{0x0000020A, 0x00000049}, {0x0000020A, 0x00000311}, {0x0000020B, 0x00000069}, {0x0000020B, 0x00000311},-{0x0000020C, 0x0000004F}, {0x0000020C, 0x0000030F}, {0x0000020D, 0x0000006F}, {0x0000020D, 0x0000030F},-{0x0000020E, 0x0000004F}, {0x0000020E, 0x00000311}, {0x0000020F, 0x0000006F}, {0x0000020F, 0x00000311},-{0x00000210, 0x00000052}, {0x00000210, 0x0000030F}, {0x00000211, 0x00000072}, {0x00000211, 0x0000030F},-{0x00000212, 0x00000052}, {0x00000212, 0x00000311}, {0x00000213, 0x00000072}, {0x00000213, 0x00000311},-{0x00000214, 0x00000055}, {0x00000214, 0x0000030F}, {0x00000215, 0x00000075}, {0x00000215, 0x0000030F},-{0x00000216, 0x00000055}, {0x00000216, 0x00000311}, {0x00000217, 0x00000075}, {0x00000217, 0x00000311},-{0x00000218, 0x00000053}, {0x00000218, 0x00000326}, {0x00000219, 0x00000073}, {0x00000219, 0x00000326},-{0x0000021A, 0x00000054}, {0x0000021A, 0x00000326}, {0x0000021B, 0x00000074}, {0x0000021B, 0x00000326},-{0x0000021E, 0x00000048}, {0x0000021E, 0x0000030C}, {0x0000021F, 0x00000068}, {0x0000021F, 0x0000030C},-{0x00000226, 0x00000041}, {0x00000226, 0x00000307}, {0x00000227, 0x00000061}, {0x00000227, 0x00000307},-{0x00000228, 0x00000045}, {0x00000228, 0x00000327}, {0x00000229, 0x00000065}, {0x00000229, 0x00000327},-{0x0000022A, 0x0000004F}, {0x0000022A, 0x00000308}, {0x0000022A, 0x00000304}, {0x0000022B, 0x0000006F},-{0x0000022B, 0x00000308}, {0x0000022B, 0x00000304}, {0x0000022C, 0x0000004F}, {0x0000022C, 0x00000303},-{0x0000022C, 0x00000304}, {0x0000022D, 0x0000006F}, {0x0000022D, 0x00000303}, {0x0000022D, 0x00000304},-{0x0000022E, 0x0000004F}, {0x0000022E, 0x00000307}, {0x0000022F, 0x0000006F}, {0x0000022F, 0x00000307},-{0x00000230, 0x0000004F}, {0x00000230, 0x00000307}, {0x00000230, 0x00000304}, {0x00000231, 0x0000006F},-{0x00000231, 0x00000307}, {0x00000231, 0x00000304}, {0x00000232, 0x00000059}, {0x00000232, 0x00000304},-{0x00000233, 0x00000079}, {0x00000233, 0x00000304}, {0x00000340, 0x00000300}, {0x00000341, 0x00000301},-{0x00000343, 0x00000313}, {0x00000344, 0x00000308}, {0x00000344, 0x00000301}, {0x00000374, 0x000002B9},-{0x0000037E, 0x0000003B}, {0x00000385, 0x000000A8}, {0x00000385, 0x00000301}, {0x00000386, 0x00000391},-{0x00000386, 0x00000301}, {0x00000387, 0x000000B7}, {0x00000388, 0x00000395}, {0x00000388, 0x00000301},-{0x00000389, 0x00000397}, {0x00000389, 0x00000301}, {0x0000038A, 0x00000399}, {0x0000038A, 0x00000301},-{0x0000038C, 0x0000039F}, {0x0000038C, 0x00000301}, {0x0000038E, 0x000003A5}, {0x0000038E, 0x00000301},-{0x0000038F, 0x000003A9}, {0x0000038F, 0x00000301}, {0x00000390, 0x000003B9}, {0x00000390, 0x00000308},-{0x00000390, 0x00000301}, {0x000003AA, 0x00000399}, {0x000003AA, 0x00000308}, {0x000003AB, 0x000003A5},-{0x000003AB, 0x00000308}, {0x000003AC, 0x000003B1}, {0x000003AC, 0x00000301}, {0x000003AD, 0x000003B5},-{0x000003AD, 0x00000301}, {0x000003AE, 0x000003B7}, {0x000003AE, 0x00000301}, {0x000003AF, 0x000003B9},-{0x000003AF, 0x00000301}, {0x000003B0, 0x000003C5}, {0x000003B0, 0x00000308}, {0x000003B0, 0x00000301},-{0x000003CA, 0x000003B9}, {0x000003CA, 0x00000308}, {0x000003CB, 0x000003C5}, {0x000003CB, 0x00000308},-{0x000003CC, 0x000003BF}, {0x000003CC, 0x00000301}, {0x000003CD, 0x000003C5}, {0x000003CD, 0x00000301},-{0x000003CE, 0x000003C9}, {0x000003CE, 0x00000301}, {0x000003D3, 0x000003D2}, {0x000003D3, 0x00000301},-{0x000003D4, 0x000003D2}, {0x000003D4, 0x00000308}, {0x00000400, 0x00000415}, {0x00000400, 0x00000300},-{0x00000401, 0x00000415}, {0x00000401, 0x00000308}, {0x00000403, 0x00000413}, {0x00000403, 0x00000301},-{0x00000407, 0x00000406}, {0x00000407, 0x00000308}, {0x0000040C, 0x0000041A}, {0x0000040C, 0x00000301},-{0x0000040D, 0x00000418}, {0x0000040D, 0x00000300}, {0x0000040E, 0x00000423}, {0x0000040E, 0x00000306},-{0x00000419, 0x00000418}, {0x00000419, 0x00000306}, {0x00000439, 0x00000438}, {0x00000439, 0x00000306},-{0x00000450, 0x00000435}, {0x00000450, 0x00000300}, {0x00000451, 0x00000435}, {0x00000451, 0x00000308},-{0x00000453, 0x00000433}, {0x00000453, 0x00000301}, {0x00000457, 0x00000456}, {0x00000457, 0x00000308},-{0x0000045C, 0x0000043A}, {0x0000045C, 0x00000301}, {0x0000045D, 0x00000438}, {0x0000045D, 0x00000300},-{0x0000045E, 0x00000443}, {0x0000045E, 0x00000306}, {0x00000476, 0x00000474}, {0x00000476, 0x0000030F},-{0x00000477, 0x00000475}, {0x00000477, 0x0000030F}, {0x000004C1, 0x00000416}, {0x000004C1, 0x00000306},-{0x000004C2, 0x00000436}, {0x000004C2, 0x00000306}, {0x000004D0, 0x00000410}, {0x000004D0, 0x00000306},-{0x000004D1, 0x00000430}, {0x000004D1, 0x00000306}, {0x000004D2, 0x00000410}, {0x000004D2, 0x00000308},-{0x000004D3, 0x00000430}, {0x000004D3, 0x00000308}, {0x000004D6, 0x00000415}, {0x000004D6, 0x00000306},-{0x000004D7, 0x00000435}, {0x000004D7, 0x00000306}, {0x000004DA, 0x000004D8}, {0x000004DA, 0x00000308},-{0x000004DB, 0x000004D9}, {0x000004DB, 0x00000308}, {0x000004DC, 0x00000416}, {0x000004DC, 0x00000308},-{0x000004DD, 0x00000436}, {0x000004DD, 0x00000308}, {0x000004DE, 0x00000417}, {0x000004DE, 0x00000308},-{0x000004DF, 0x00000437}, {0x000004DF, 0x00000308}, {0x000004E2, 0x00000418}, {0x000004E2, 0x00000304},-{0x000004E3, 0x00000438}, {0x000004E3, 0x00000304}, {0x000004E4, 0x00000418}, {0x000004E4, 0x00000308},-{0x000004E5, 0x00000438}, {0x000004E5, 0x00000308}, {0x000004E6, 0x0000041E}, {0x000004E6, 0x00000308},-{0x000004E7, 0x0000043E}, {0x000004E7, 0x00000308}, {0x000004EA, 0x000004E8}, {0x000004EA, 0x00000308},-{0x000004EB, 0x000004E9}, {0x000004EB, 0x00000308}, {0x000004EC, 0x0000042D}, {0x000004EC, 0x00000308},-{0x000004ED, 0x0000044D}, {0x000004ED, 0x00000308}, {0x000004EE, 0x00000423}, {0x000004EE, 0x00000304},-{0x000004EF, 0x00000443}, {0x000004EF, 0x00000304}, {0x000004F0, 0x00000423}, {0x000004F0, 0x00000308},-{0x000004F1, 0x00000443}, {0x000004F1, 0x00000308}, {0x000004F2, 0x00000423}, {0x000004F2, 0x0000030B},-{0x000004F3, 0x00000443}, {0x000004F3, 0x0000030B}, {0x000004F4, 0x00000427}, {0x000004F4, 0x00000308},-{0x000004F5, 0x00000447}, {0x000004F5, 0x00000308}, {0x000004F8, 0x0000042B}, {0x000004F8, 0x00000308},-{0x000004F9, 0x0000044B}, {0x000004F9, 0x00000308}, {0x00000622, 0x00000627}, {0x00000622, 0x00000653},-{0x00000623, 0x00000627}, {0x00000623, 0x00000654}, {0x00000624, 0x00000648}, {0x00000624, 0x00000654},-{0x00000625, 0x00000627}, {0x00000625, 0x00000655}, {0x00000626, 0x0000064A}, {0x00000626, 0x00000654},-{0x000006C0, 0x000006D5}, {0x000006C0, 0x00000654}, {0x000006C2, 0x000006C1}, {0x000006C2, 0x00000654},-{0x000006D3, 0x000006D2}, {0x000006D3, 0x00000654}, {0x00000929, 0x00000928}, {0x00000929, 0x0000093C},-{0x00000931, 0x00000930}, {0x00000931, 0x0000093C}, {0x00000934, 0x00000933}, {0x00000934, 0x0000093C},-{0x00000958, 0x00000915}, {0x00000958, 0x0000093C}, {0x00000959, 0x00000916}, {0x00000959, 0x0000093C},-{0x0000095A, 0x00000917}, {0x0000095A, 0x0000093C}, {0x0000095B, 0x0000091C}, {0x0000095B, 0x0000093C},-{0x0000095C, 0x00000921}, {0x0000095C, 0x0000093C}, {0x0000095D, 0x00000922}, {0x0000095D, 0x0000093C},-{0x0000095E, 0x0000092B}, {0x0000095E, 0x0000093C}, {0x0000095F, 0x0000092F}, {0x0000095F, 0x0000093C},-{0x000009CB, 0x000009C7}, {0x000009CB, 0x000009BE}, {0x000009CC, 0x000009C7}, {0x000009CC, 0x000009D7},-{0x000009DC, 0x000009A1}, {0x000009DC, 0x000009BC}, {0x000009DD, 0x000009A2}, {0x000009DD, 0x000009BC},-{0x000009DF, 0x000009AF}, {0x000009DF, 0x000009BC}, {0x00000A33, 0x00000A32}, {0x00000A33, 0x00000A3C},-{0x00000A36, 0x00000A38}, {0x00000A36, 0x00000A3C}, {0x00000A59, 0x00000A16}, {0x00000A59, 0x00000A3C},-{0x00000A5A, 0x00000A17}, {0x00000A5A, 0x00000A3C}, {0x00000A5B, 0x00000A1C}, {0x00000A5B, 0x00000A3C},-{0x00000A5E, 0x00000A2B}, {0x00000A5E, 0x00000A3C}, {0x00000B48, 0x00000B47}, {0x00000B48, 0x00000B56},-{0x00000B4B, 0x00000B47}, {0x00000B4B, 0x00000B3E}, {0x00000B4C, 0x00000B47}, {0x00000B4C, 0x00000B57},-{0x00000B5C, 0x00000B21}, {0x00000B5C, 0x00000B3C}, {0x00000B5D, 0x00000B22}, {0x00000B5D, 0x00000B3C},-{0x00000B94, 0x00000B92}, {0x00000B94, 0x00000BD7}, {0x00000BCA, 0x00000BC6}, {0x00000BCA, 0x00000BBE},-{0x00000BCB, 0x00000BC7}, {0x00000BCB, 0x00000BBE}, {0x00000BCC, 0x00000BC6}, {0x00000BCC, 0x00000BD7},-{0x00000C48, 0x00000C46}, {0x00000C48, 0x00000C56}, {0x00000CC0, 0x00000CBF}, {0x00000CC0, 0x00000CD5},-{0x00000CC7, 0x00000CC6}, {0x00000CC7, 0x00000CD5}, {0x00000CC8, 0x00000CC6}, {0x00000CC8, 0x00000CD6},-{0x00000CCA, 0x00000CC6}, {0x00000CCA, 0x00000CC2}, {0x00000CCB, 0x00000CC6}, {0x00000CCB, 0x00000CC2},-{0x00000CCB, 0x00000CD5}, {0x00000D4A, 0x00000D46}, {0x00000D4A, 0x00000D3E}, {0x00000D4B, 0x00000D47},-{0x00000D4B, 0x00000D3E}, {0x00000D4C, 0x00000D46}, {0x00000D4C, 0x00000D57}, {0x00000DDA, 0x00000DD9},-{0x00000DDA, 0x00000DCA}, {0x00000DDC, 0x00000DD9}, {0x00000DDC, 0x00000DCF}, {0x00000DDD, 0x00000DD9},-{0x00000DDD, 0x00000DCF}, {0x00000DDD, 0x00000DCA}, {0x00000DDE, 0x00000DD9}, {0x00000DDE, 0x00000DDF},-{0x00000F43, 0x00000F42}, {0x00000F43, 0x00000FB7}, {0x00000F4D, 0x00000F4C}, {0x00000F4D, 0x00000FB7},-{0x00000F52, 0x00000F51}, {0x00000F52, 0x00000FB7}, {0x00000F57, 0x00000F56}, {0x00000F57, 0x00000FB7},-{0x00000F5C, 0x00000F5B}, {0x00000F5C, 0x00000FB7}, {0x00000F69, 0x00000F40}, {0x00000F69, 0x00000FB5},-{0x00000F73, 0x00000F71}, {0x00000F73, 0x00000F72}, {0x00000F75, 0x00000F71}, {0x00000F75, 0x00000F74},-{0x00000F76, 0x00000FB2}, {0x00000F76, 0x00000F80}, {0x00000F78, 0x00000FB3}, {0x00000F78, 0x00000F80},-{0x00000F81, 0x00000F71}, {0x00000F81, 0x00000F80}, {0x00000F93, 0x00000F92}, {0x00000F93, 0x00000FB7},-{0x00000F9D, 0x00000F9C}, {0x00000F9D, 0x00000FB7}, {0x00000FA2, 0x00000FA1}, {0x00000FA2, 0x00000FB7},-{0x00000FA7, 0x00000FA6}, {0x00000FA7, 0x00000FB7}, {0x00000FAC, 0x00000FAB}, {0x00000FAC, 0x00000FB7},-{0x00000FB9, 0x00000F90}, {0x00000FB9, 0x00000FB5}, {0x00001026, 0x00001025}, {0x00001026, 0x0000102E},-{0x00001B06, 0x00001B05}, {0x00001B06, 0x00001B35}, {0x00001B08, 0x00001B07}, {0x00001B08, 0x00001B35},-{0x00001B0A, 0x00001B09}, {0x00001B0A, 0x00001B35}, {0x00001B0C, 0x00001B0B}, {0x00001B0C, 0x00001B35},-{0x00001B0E, 0x00001B0D}, {0x00001B0E, 0x00001B35}, {0x00001B12, 0x00001B11}, {0x00001B12, 0x00001B35},-{0x00001B3B, 0x00001B3A}, {0x00001B3B, 0x00001B35}, {0x00001B3D, 0x00001B3C}, {0x00001B3D, 0x00001B35},-{0x00001B40, 0x00001B3E}, {0x00001B40, 0x00001B35}, {0x00001B41, 0x00001B3F}, {0x00001B41, 0x00001B35},-{0x00001B43, 0x00001B42}, {0x00001B43, 0x00001B35}, {0x00001E00, 0x00000041}, {0x00001E00, 0x00000325},-{0x00001E01, 0x00000061}, {0x00001E01, 0x00000325}, {0x00001E02, 0x00000042}, {0x00001E02, 0x00000307},-{0x00001E03, 0x00000062}, {0x00001E03, 0x00000307}, {0x00001E04, 0x00000042}, {0x00001E04, 0x00000323},-{0x00001E05, 0x00000062}, {0x00001E05, 0x00000323}, {0x00001E06, 0x00000042}, {0x00001E06, 0x00000331},-{0x00001E07, 0x00000062}, {0x00001E07, 0x00000331}, {0x00001E08, 0x00000043}, {0x00001E08, 0x00000327},-{0x00001E08, 0x00000301}, {0x00001E09, 0x00000063}, {0x00001E09, 0x00000327}, {0x00001E09, 0x00000301},-{0x00001E0A, 0x00000044}, {0x00001E0A, 0x00000307}, {0x00001E0B, 0x00000064}, {0x00001E0B, 0x00000307},-{0x00001E0C, 0x00000044}, {0x00001E0C, 0x00000323}, {0x00001E0D, 0x00000064}, {0x00001E0D, 0x00000323},-{0x00001E0E, 0x00000044}, {0x00001E0E, 0x00000331}, {0x00001E0F, 0x00000064}, {0x00001E0F, 0x00000331},-{0x00001E10, 0x00000044}, {0x00001E10, 0x00000327}, {0x00001E11, 0x00000064}, {0x00001E11, 0x00000327},-{0x00001E12, 0x00000044}, {0x00001E12, 0x0000032D}, {0x00001E13, 0x00000064}, {0x00001E13, 0x0000032D},-{0x00001E14, 0x00000045}, {0x00001E14, 0x00000304}, {0x00001E14, 0x00000300}, {0x00001E15, 0x00000065},-{0x00001E15, 0x00000304}, {0x00001E15, 0x00000300}, {0x00001E16, 0x00000045}, {0x00001E16, 0x00000304},-{0x00001E16, 0x00000301}, {0x00001E17, 0x00000065}, {0x00001E17, 0x00000304}, {0x00001E17, 0x00000301},-{0x00001E18, 0x00000045}, {0x00001E18, 0x0000032D}, {0x00001E19, 0x00000065}, {0x00001E19, 0x0000032D},-{0x00001E1A, 0x00000045}, {0x00001E1A, 0x00000330}, {0x00001E1B, 0x00000065}, {0x00001E1B, 0x00000330},-{0x00001E1C, 0x00000045}, {0x00001E1C, 0x00000327}, {0x00001E1C, 0x00000306}, {0x00001E1D, 0x00000065},-{0x00001E1D, 0x00000327}, {0x00001E1D, 0x00000306}, {0x00001E1E, 0x00000046}, {0x00001E1E, 0x00000307},-{0x00001E1F, 0x00000066}, {0x00001E1F, 0x00000307}, {0x00001E20, 0x00000047}, {0x00001E20, 0x00000304},-{0x00001E21, 0x00000067}, {0x00001E21, 0x00000304}, {0x00001E22, 0x00000048}, {0x00001E22, 0x00000307},-{0x00001E23, 0x00000068}, {0x00001E23, 0x00000307}, {0x00001E24, 0x00000048}, {0x00001E24, 0x00000323},-{0x00001E25, 0x00000068}, {0x00001E25, 0x00000323}, {0x00001E26, 0x00000048}, {0x00001E26, 0x00000308},-{0x00001E27, 0x00000068}, {0x00001E27, 0x00000308}, {0x00001E28, 0x00000048}, {0x00001E28, 0x00000327},-{0x00001E29, 0x00000068}, {0x00001E29, 0x00000327}, {0x00001E2A, 0x00000048}, {0x00001E2A, 0x0000032E},-{0x00001E2B, 0x00000068}, {0x00001E2B, 0x0000032E}, {0x00001E2C, 0x00000049}, {0x00001E2C, 0x00000330},-{0x00001E2D, 0x00000069}, {0x00001E2D, 0x00000330}, {0x00001E2E, 0x00000049}, {0x00001E2E, 0x00000308},-{0x00001E2E, 0x00000301}, {0x00001E2F, 0x00000069}, {0x00001E2F, 0x00000308}, {0x00001E2F, 0x00000301},-{0x00001E30, 0x0000004B}, {0x00001E30, 0x00000301}, {0x00001E31, 0x0000006B}, {0x00001E31, 0x00000301},-{0x00001E32, 0x0000004B}, {0x00001E32, 0x00000323}, {0x00001E33, 0x0000006B}, {0x00001E33, 0x00000323},-{0x00001E34, 0x0000004B}, {0x00001E34, 0x00000331}, {0x00001E35, 0x0000006B}, {0x00001E35, 0x00000331},-{0x00001E36, 0x0000004C}, {0x00001E36, 0x00000323}, {0x00001E37, 0x0000006C}, {0x00001E37, 0x00000323},-{0x00001E38, 0x0000004C}, {0x00001E38, 0x00000323}, {0x00001E38, 0x00000304}, {0x00001E39, 0x0000006C},-{0x00001E39, 0x00000323}, {0x00001E39, 0x00000304}, {0x00001E3A, 0x0000004C}, {0x00001E3A, 0x00000331},-{0x00001E3B, 0x0000006C}, {0x00001E3B, 0x00000331}, {0x00001E3C, 0x0000004C}, {0x00001E3C, 0x0000032D},-{0x00001E3D, 0x0000006C}, {0x00001E3D, 0x0000032D}, {0x00001E3E, 0x0000004D}, {0x00001E3E, 0x00000301},-{0x00001E3F, 0x0000006D}, {0x00001E3F, 0x00000301}, {0x00001E40, 0x0000004D}, {0x00001E40, 0x00000307},-{0x00001E41, 0x0000006D}, {0x00001E41, 0x00000307}, {0x00001E42, 0x0000004D}, {0x00001E42, 0x00000323},-{0x00001E43, 0x0000006D}, {0x00001E43, 0x00000323}, {0x00001E44, 0x0000004E}, {0x00001E44, 0x00000307},-{0x00001E45, 0x0000006E}, {0x00001E45, 0x00000307}, {0x00001E46, 0x0000004E}, {0x00001E46, 0x00000323},-{0x00001E47, 0x0000006E}, {0x00001E47, 0x00000323}, {0x00001E48, 0x0000004E}, {0x00001E48, 0x00000331},-{0x00001E49, 0x0000006E}, {0x00001E49, 0x00000331}, {0x00001E4A, 0x0000004E}, {0x00001E4A, 0x0000032D},-{0x00001E4B, 0x0000006E}, {0x00001E4B, 0x0000032D}, {0x00001E4C, 0x0000004F}, {0x00001E4C, 0x00000303},-{0x00001E4C, 0x00000301}, {0x00001E4D, 0x0000006F}, {0x00001E4D, 0x00000303}, {0x00001E4D, 0x00000301},-{0x00001E4E, 0x0000004F}, {0x00001E4E, 0x00000303}, {0x00001E4E, 0x00000308}, {0x00001E4F, 0x0000006F},-{0x00001E4F, 0x00000303}, {0x00001E4F, 0x00000308}, {0x00001E50, 0x0000004F}, {0x00001E50, 0x00000304},-{0x00001E50, 0x00000300}, {0x00001E51, 0x0000006F}, {0x00001E51, 0x00000304}, {0x00001E51, 0x00000300},-{0x00001E52, 0x0000004F}, {0x00001E52, 0x00000304}, {0x00001E52, 0x00000301}, {0x00001E53, 0x0000006F},-{0x00001E53, 0x00000304}, {0x00001E53, 0x00000301}, {0x00001E54, 0x00000050}, {0x00001E54, 0x00000301},-{0x00001E55, 0x00000070}, {0x00001E55, 0x00000301}, {0x00001E56, 0x00000050}, {0x00001E56, 0x00000307},-{0x00001E57, 0x00000070}, {0x00001E57, 0x00000307}, {0x00001E58, 0x00000052}, {0x00001E58, 0x00000307},-{0x00001E59, 0x00000072}, {0x00001E59, 0x00000307}, {0x00001E5A, 0x00000052}, {0x00001E5A, 0x00000323},-{0x00001E5B, 0x00000072}, {0x00001E5B, 0x00000323}, {0x00001E5C, 0x00000052}, {0x00001E5C, 0x00000323},-{0x00001E5C, 0x00000304}, {0x00001E5D, 0x00000072}, {0x00001E5D, 0x00000323}, {0x00001E5D, 0x00000304},-{0x00001E5E, 0x00000052}, {0x00001E5E, 0x00000331}, {0x00001E5F, 0x00000072}, {0x00001E5F, 0x00000331},-{0x00001E60, 0x00000053}, {0x00001E60, 0x00000307}, {0x00001E61, 0x00000073}, {0x00001E61, 0x00000307},-{0x00001E62, 0x00000053}, {0x00001E62, 0x00000323}, {0x00001E63, 0x00000073}, {0x00001E63, 0x00000323},-{0x00001E64, 0x00000053}, {0x00001E64, 0x00000301}, {0x00001E64, 0x00000307}, {0x00001E65, 0x00000073},-{0x00001E65, 0x00000301}, {0x00001E65, 0x00000307}, {0x00001E66, 0x00000053}, {0x00001E66, 0x0000030C},-{0x00001E66, 0x00000307}, {0x00001E67, 0x00000073}, {0x00001E67, 0x0000030C}, {0x00001E67, 0x00000307},-{0x00001E68, 0x00000053}, {0x00001E68, 0x00000323}, {0x00001E68, 0x00000307}, {0x00001E69, 0x00000073},-{0x00001E69, 0x00000323}, {0x00001E69, 0x00000307}, {0x00001E6A, 0x00000054}, {0x00001E6A, 0x00000307},-{0x00001E6B, 0x00000074}, {0x00001E6B, 0x00000307}, {0x00001E6C, 0x00000054}, {0x00001E6C, 0x00000323},-{0x00001E6D, 0x00000074}, {0x00001E6D, 0x00000323}, {0x00001E6E, 0x00000054}, {0x00001E6E, 0x00000331},-{0x00001E6F, 0x00000074}, {0x00001E6F, 0x00000331}, {0x00001E70, 0x00000054}, {0x00001E70, 0x0000032D},-{0x00001E71, 0x00000074}, {0x00001E71, 0x0000032D}, {0x00001E72, 0x00000055}, {0x00001E72, 0x00000324},-{0x00001E73, 0x00000075}, {0x00001E73, 0x00000324}, {0x00001E74, 0x00000055}, {0x00001E74, 0x00000330},-{0x00001E75, 0x00000075}, {0x00001E75, 0x00000330}, {0x00001E76, 0x00000055}, {0x00001E76, 0x0000032D},-{0x00001E77, 0x00000075}, {0x00001E77, 0x0000032D}, {0x00001E78, 0x00000055}, {0x00001E78, 0x00000303},-{0x00001E78, 0x00000301}, {0x00001E79, 0x00000075}, {0x00001E79, 0x00000303}, {0x00001E79, 0x00000301},-{0x00001E7A, 0x00000055}, {0x00001E7A, 0x00000304}, {0x00001E7A, 0x00000308}, {0x00001E7B, 0x00000075},-{0x00001E7B, 0x00000304}, {0x00001E7B, 0x00000308}, {0x00001E7C, 0x00000056}, {0x00001E7C, 0x00000303},-{0x00001E7D, 0x00000076}, {0x00001E7D, 0x00000303}, {0x00001E7E, 0x00000056}, {0x00001E7E, 0x00000323},-{0x00001E7F, 0x00000076}, {0x00001E7F, 0x00000323}, {0x00001E80, 0x00000057}, {0x00001E80, 0x00000300},-{0x00001E81, 0x00000077}, {0x00001E81, 0x00000300}, {0x00001E82, 0x00000057}, {0x00001E82, 0x00000301},-{0x00001E83, 0x00000077}, {0x00001E83, 0x00000301}, {0x00001E84, 0x00000057}, {0x00001E84, 0x00000308},-{0x00001E85, 0x00000077}, {0x00001E85, 0x00000308}, {0x00001E86, 0x00000057}, {0x00001E86, 0x00000307},-{0x00001E87, 0x00000077}, {0x00001E87, 0x00000307}, {0x00001E88, 0x00000057}, {0x00001E88, 0x00000323},-{0x00001E89, 0x00000077}, {0x00001E89, 0x00000323}, {0x00001E8A, 0x00000058}, {0x00001E8A, 0x00000307},-{0x00001E8B, 0x00000078}, {0x00001E8B, 0x00000307}, {0x00001E8C, 0x00000058}, {0x00001E8C, 0x00000308},-{0x00001E8D, 0x00000078}, {0x00001E8D, 0x00000308}, {0x00001E8E, 0x00000059}, {0x00001E8E, 0x00000307},-{0x00001E8F, 0x00000079}, {0x00001E8F, 0x00000307}, {0x00001E90, 0x0000005A}, {0x00001E90, 0x00000302},-{0x00001E91, 0x0000007A}, {0x00001E91, 0x00000302}, {0x00001E92, 0x0000005A}, {0x00001E92, 0x00000323},-{0x00001E93, 0x0000007A}, {0x00001E93, 0x00000323}, {0x00001E94, 0x0000005A}, {0x00001E94, 0x00000331},-{0x00001E95, 0x0000007A}, {0x00001E95, 0x00000331}, {0x00001E96, 0x00000068}, {0x00001E96, 0x00000331},-{0x00001E97, 0x00000074}, {0x00001E97, 0x00000308}, {0x00001E98, 0x00000077}, {0x00001E98, 0x0000030A},-{0x00001E99, 0x00000079}, {0x00001E99, 0x0000030A}, {0x00001E9B, 0x0000017F}, {0x00001E9B, 0x00000307},-{0x00001EA0, 0x00000041}, {0x00001EA0, 0x00000323}, {0x00001EA1, 0x00000061}, {0x00001EA1, 0x00000323},-{0x00001EA2, 0x00000041}, {0x00001EA2, 0x00000309}, {0x00001EA3, 0x00000061}, {0x00001EA3, 0x00000309},-{0x00001EA4, 0x00000041}, {0x00001EA4, 0x00000302}, {0x00001EA4, 0x00000301}, {0x00001EA5, 0x00000061},-{0x00001EA5, 0x00000302}, {0x00001EA5, 0x00000301}, {0x00001EA6, 0x00000041}, {0x00001EA6, 0x00000302},-{0x00001EA6, 0x00000300}, {0x00001EA7, 0x00000061}, {0x00001EA7, 0x00000302}, {0x00001EA7, 0x00000300},-{0x00001EA8, 0x00000041}, {0x00001EA8, 0x00000302}, {0x00001EA8, 0x00000309}, {0x00001EA9, 0x00000061},-{0x00001EA9, 0x00000302}, {0x00001EA9, 0x00000309}, {0x00001EAA, 0x00000041}, {0x00001EAA, 0x00000302},-{0x00001EAA, 0x00000303}, {0x00001EAB, 0x00000061}, {0x00001EAB, 0x00000302}, {0x00001EAB, 0x00000303},-{0x00001EAC, 0x00000041}, {0x00001EAC, 0x00000323}, {0x00001EAC, 0x00000302}, {0x00001EAD, 0x00000061},-{0x00001EAD, 0x00000323}, {0x00001EAD, 0x00000302}, {0x00001EAE, 0x00000041}, {0x00001EAE, 0x00000306},-{0x00001EAE, 0x00000301}, {0x00001EAF, 0x00000061}, {0x00001EAF, 0x00000306}, {0x00001EAF, 0x00000301},-{0x00001EB0, 0x00000041}, {0x00001EB0, 0x00000306}, {0x00001EB0, 0x00000300}, {0x00001EB1, 0x00000061},-{0x00001EB1, 0x00000306}, {0x00001EB1, 0x00000300}, {0x00001EB2, 0x00000041}, {0x00001EB2, 0x00000306},-{0x00001EB2, 0x00000309}, {0x00001EB3, 0x00000061}, {0x00001EB3, 0x00000306}, {0x00001EB3, 0x00000309},-{0x00001EB4, 0x00000041}, {0x00001EB4, 0x00000306}, {0x00001EB4, 0x00000303}, {0x00001EB5, 0x00000061},-{0x00001EB5, 0x00000306}, {0x00001EB5, 0x00000303}, {0x00001EB6, 0x00000041}, {0x00001EB6, 0x00000323},-{0x00001EB6, 0x00000306}, {0x00001EB7, 0x00000061}, {0x00001EB7, 0x00000323}, {0x00001EB7, 0x00000306},-{0x00001EB8, 0x00000045}, {0x00001EB8, 0x00000323}, {0x00001EB9, 0x00000065}, {0x00001EB9, 0x00000323},-{0x00001EBA, 0x00000045}, {0x00001EBA, 0x00000309}, {0x00001EBB, 0x00000065}, {0x00001EBB, 0x00000309},-{0x00001EBC, 0x00000045}, {0x00001EBC, 0x00000303}, {0x00001EBD, 0x00000065}, {0x00001EBD, 0x00000303},-{0x00001EBE, 0x00000045}, {0x00001EBE, 0x00000302}, {0x00001EBE, 0x00000301}, {0x00001EBF, 0x00000065},-{0x00001EBF, 0x00000302}, {0x00001EBF, 0x00000301}, {0x00001EC0, 0x00000045}, {0x00001EC0, 0x00000302},-{0x00001EC0, 0x00000300}, {0x00001EC1, 0x00000065}, {0x00001EC1, 0x00000302}, {0x00001EC1, 0x00000300},-{0x00001EC2, 0x00000045}, {0x00001EC2, 0x00000302}, {0x00001EC2, 0x00000309}, {0x00001EC3, 0x00000065},-{0x00001EC3, 0x00000302}, {0x00001EC3, 0x00000309}, {0x00001EC4, 0x00000045}, {0x00001EC4, 0x00000302},-{0x00001EC4, 0x00000303}, {0x00001EC5, 0x00000065}, {0x00001EC5, 0x00000302}, {0x00001EC5, 0x00000303},-{0x00001EC6, 0x00000045}, {0x00001EC6, 0x00000323}, {0x00001EC6, 0x00000302}, {0x00001EC7, 0x00000065},-{0x00001EC7, 0x00000323}, {0x00001EC7, 0x00000302}, {0x00001EC8, 0x00000049}, {0x00001EC8, 0x00000309},-{0x00001EC9, 0x00000069}, {0x00001EC9, 0x00000309}, {0x00001ECA, 0x00000049}, {0x00001ECA, 0x00000323},-{0x00001ECB, 0x00000069}, {0x00001ECB, 0x00000323}, {0x00001ECC, 0x0000004F}, {0x00001ECC, 0x00000323},-{0x00001ECD, 0x0000006F}, {0x00001ECD, 0x00000323}, {0x00001ECE, 0x0000004F}, {0x00001ECE, 0x00000309},-{0x00001ECF, 0x0000006F}, {0x00001ECF, 0x00000309}, {0x00001ED0, 0x0000004F}, {0x00001ED0, 0x00000302},-{0x00001ED0, 0x00000301}, {0x00001ED1, 0x0000006F}, {0x00001ED1, 0x00000302}, {0x00001ED1, 0x00000301},-{0x00001ED2, 0x0000004F}, {0x00001ED2, 0x00000302}, {0x00001ED2, 0x00000300}, {0x00001ED3, 0x0000006F},-{0x00001ED3, 0x00000302}, {0x00001ED3, 0x00000300}, {0x00001ED4, 0x0000004F}, {0x00001ED4, 0x00000302},-{0x00001ED4, 0x00000309}, {0x00001ED5, 0x0000006F}, {0x00001ED5, 0x00000302}, {0x00001ED5, 0x00000309},-{0x00001ED6, 0x0000004F}, {0x00001ED6, 0x00000302}, {0x00001ED6, 0x00000303}, {0x00001ED7, 0x0000006F},-{0x00001ED7, 0x00000302}, {0x00001ED7, 0x00000303}, {0x00001ED8, 0x0000004F}, {0x00001ED8, 0x00000323},-{0x00001ED8, 0x00000302}, {0x00001ED9, 0x0000006F}, {0x00001ED9, 0x00000323}, {0x00001ED9, 0x00000302},-{0x00001EDA, 0x0000004F}, {0x00001EDA, 0x0000031B}, {0x00001EDA, 0x00000301}, {0x00001EDB, 0x0000006F},-{0x00001EDB, 0x0000031B}, {0x00001EDB, 0x00000301}, {0x00001EDC, 0x0000004F}, {0x00001EDC, 0x0000031B},-{0x00001EDC, 0x00000300}, {0x00001EDD, 0x0000006F}, {0x00001EDD, 0x0000031B}, {0x00001EDD, 0x00000300},-{0x00001EDE, 0x0000004F}, {0x00001EDE, 0x0000031B}, {0x00001EDE, 0x00000309}, {0x00001EDF, 0x0000006F},-{0x00001EDF, 0x0000031B}, {0x00001EDF, 0x00000309}, {0x00001EE0, 0x0000004F}, {0x00001EE0, 0x0000031B},-{0x00001EE0, 0x00000303}, {0x00001EE1, 0x0000006F}, {0x00001EE1, 0x0000031B}, {0x00001EE1, 0x00000303},-{0x00001EE2, 0x0000004F}, {0x00001EE2, 0x0000031B}, {0x00001EE2, 0x00000323}, {0x00001EE3, 0x0000006F},-{0x00001EE3, 0x0000031B}, {0x00001EE3, 0x00000323}, {0x00001EE4, 0x00000055}, {0x00001EE4, 0x00000323},-{0x00001EE5, 0x00000075}, {0x00001EE5, 0x00000323}, {0x00001EE6, 0x00000055}, {0x00001EE6, 0x00000309},-{0x00001EE7, 0x00000075}, {0x00001EE7, 0x00000309}, {0x00001EE8, 0x00000055}, {0x00001EE8, 0x0000031B},-{0x00001EE8, 0x00000301}, {0x00001EE9, 0x00000075}, {0x00001EE9, 0x0000031B}, {0x00001EE9, 0x00000301},-{0x00001EEA, 0x00000055}, {0x00001EEA, 0x0000031B}, {0x00001EEA, 0x00000300}, {0x00001EEB, 0x00000075},-{0x00001EEB, 0x0000031B}, {0x00001EEB, 0x00000300}, {0x00001EEC, 0x00000055}, {0x00001EEC, 0x0000031B},-{0x00001EEC, 0x00000309}, {0x00001EED, 0x00000075}, {0x00001EED, 0x0000031B}, {0x00001EED, 0x00000309},-{0x00001EEE, 0x00000055}, {0x00001EEE, 0x0000031B}, {0x00001EEE, 0x00000303}, {0x00001EEF, 0x00000075},-{0x00001EEF, 0x0000031B}, {0x00001EEF, 0x00000303}, {0x00001EF0, 0x00000055}, {0x00001EF0, 0x0000031B},-{0x00001EF0, 0x00000323}, {0x00001EF1, 0x00000075}, {0x00001EF1, 0x0000031B}, {0x00001EF1, 0x00000323},-{0x00001EF2, 0x00000059}, {0x00001EF2, 0x00000300}, {0x00001EF3, 0x00000079}, {0x00001EF3, 0x00000300},-{0x00001EF4, 0x00000059}, {0x00001EF4, 0x00000323}, {0x00001EF5, 0x00000079}, {0x00001EF5, 0x00000323},-{0x00001EF6, 0x00000059}, {0x00001EF6, 0x00000309}, {0x00001EF7, 0x00000079}, {0x00001EF7, 0x00000309},-{0x00001EF8, 0x00000059}, {0x00001EF8, 0x00000303}, {0x00001EF9, 0x00000079}, {0x00001EF9, 0x00000303},-{0x00001F00, 0x000003B1}, {0x00001F00, 0x00000313}, {0x00001F01, 0x000003B1}, {0x00001F01, 0x00000314},-{0x00001F02, 0x000003B1}, {0x00001F02, 0x00000313}, {0x00001F02, 0x00000300}, {0x00001F03, 0x000003B1},-{0x00001F03, 0x00000314}, {0x00001F03, 0x00000300}, {0x00001F04, 0x000003B1}, {0x00001F04, 0x00000313},-{0x00001F04, 0x00000301}, {0x00001F05, 0x000003B1}, {0x00001F05, 0x00000314}, {0x00001F05, 0x00000301},-{0x00001F06, 0x000003B1}, {0x00001F06, 0x00000313}, {0x00001F06, 0x00000342}, {0x00001F07, 0x000003B1},-{0x00001F07, 0x00000314}, {0x00001F07, 0x00000342}, {0x00001F08, 0x00000391}, {0x00001F08, 0x00000313},-{0x00001F09, 0x00000391}, {0x00001F09, 0x00000314}, {0x00001F0A, 0x00000391}, {0x00001F0A, 0x00000313},-{0x00001F0A, 0x00000300}, {0x00001F0B, 0x00000391}, {0x00001F0B, 0x00000314}, {0x00001F0B, 0x00000300},-{0x00001F0C, 0x00000391}, {0x00001F0C, 0x00000313}, {0x00001F0C, 0x00000301}, {0x00001F0D, 0x00000391},-{0x00001F0D, 0x00000314}, {0x00001F0D, 0x00000301}, {0x00001F0E, 0x00000391}, {0x00001F0E, 0x00000313},-{0x00001F0E, 0x00000342}, {0x00001F0F, 0x00000391}, {0x00001F0F, 0x00000314}, {0x00001F0F, 0x00000342},-{0x00001F10, 0x000003B5}, {0x00001F10, 0x00000313}, {0x00001F11, 0x000003B5}, {0x00001F11, 0x00000314},-{0x00001F12, 0x000003B5}, {0x00001F12, 0x00000313}, {0x00001F12, 0x00000300}, {0x00001F13, 0x000003B5},-{0x00001F13, 0x00000314}, {0x00001F13, 0x00000300}, {0x00001F14, 0x000003B5}, {0x00001F14, 0x00000313},-{0x00001F14, 0x00000301}, {0x00001F15, 0x000003B5}, {0x00001F15, 0x00000314}, {0x00001F15, 0x00000301},-{0x00001F18, 0x00000395}, {0x00001F18, 0x00000313}, {0x00001F19, 0x00000395}, {0x00001F19, 0x00000314},-{0x00001F1A, 0x00000395}, {0x00001F1A, 0x00000313}, {0x00001F1A, 0x00000300}, {0x00001F1B, 0x00000395},-{0x00001F1B, 0x00000314}, {0x00001F1B, 0x00000300}, {0x00001F1C, 0x00000395}, {0x00001F1C, 0x00000313},-{0x00001F1C, 0x00000301}, {0x00001F1D, 0x00000395}, {0x00001F1D, 0x00000314}, {0x00001F1D, 0x00000301},-{0x00001F20, 0x000003B7}, {0x00001F20, 0x00000313}, {0x00001F21, 0x000003B7}, {0x00001F21, 0x00000314},-{0x00001F22, 0x000003B7}, {0x00001F22, 0x00000313}, {0x00001F22, 0x00000300}, {0x00001F23, 0x000003B7},-{0x00001F23, 0x00000314}, {0x00001F23, 0x00000300}, {0x00001F24, 0x000003B7}, {0x00001F24, 0x00000313},-{0x00001F24, 0x00000301}, {0x00001F25, 0x000003B7}, {0x00001F25, 0x00000314}, {0x00001F25, 0x00000301},-{0x00001F26, 0x000003B7}, {0x00001F26, 0x00000313}, {0x00001F26, 0x00000342}, {0x00001F27, 0x000003B7},-{0x00001F27, 0x00000314}, {0x00001F27, 0x00000342}, {0x00001F28, 0x00000397}, {0x00001F28, 0x00000313},-{0x00001F29, 0x00000397}, {0x00001F29, 0x00000314}, {0x00001F2A, 0x00000397}, {0x00001F2A, 0x00000313},-{0x00001F2A, 0x00000300}, {0x00001F2B, 0x00000397}, {0x00001F2B, 0x00000314}, {0x00001F2B, 0x00000300},-{0x00001F2C, 0x00000397}, {0x00001F2C, 0x00000313}, {0x00001F2C, 0x00000301}, {0x00001F2D, 0x00000397},-{0x00001F2D, 0x00000314}, {0x00001F2D, 0x00000301}, {0x00001F2E, 0x00000397}, {0x00001F2E, 0x00000313},-{0x00001F2E, 0x00000342}, {0x00001F2F, 0x00000397}, {0x00001F2F, 0x00000314}, {0x00001F2F, 0x00000342},-{0x00001F30, 0x000003B9}, {0x00001F30, 0x00000313}, {0x00001F31, 0x000003B9}, {0x00001F31, 0x00000314},-{0x00001F32, 0x000003B9}, {0x00001F32, 0x00000313}, {0x00001F32, 0x00000300}, {0x00001F33, 0x000003B9},-{0x00001F33, 0x00000314}, {0x00001F33, 0x00000300}, {0x00001F34, 0x000003B9}, {0x00001F34, 0x00000313},-{0x00001F34, 0x00000301}, {0x00001F35, 0x000003B9}, {0x00001F35, 0x00000314}, {0x00001F35, 0x00000301},-{0x00001F36, 0x000003B9}, {0x00001F36, 0x00000313}, {0x00001F36, 0x00000342}, {0x00001F37, 0x000003B9},-{0x00001F37, 0x00000314}, {0x00001F37, 0x00000342}, {0x00001F38, 0x00000399}, {0x00001F38, 0x00000313},-{0x00001F39, 0x00000399}, {0x00001F39, 0x00000314}, {0x00001F3A, 0x00000399}, {0x00001F3A, 0x00000313},-{0x00001F3A, 0x00000300}, {0x00001F3B, 0x00000399}, {0x00001F3B, 0x00000314}, {0x00001F3B, 0x00000300},-{0x00001F3C, 0x00000399}, {0x00001F3C, 0x00000313}, {0x00001F3C, 0x00000301}, {0x00001F3D, 0x00000399},-{0x00001F3D, 0x00000314}, {0x00001F3D, 0x00000301}, {0x00001F3E, 0x00000399}, {0x00001F3E, 0x00000313},-{0x00001F3E, 0x00000342}, {0x00001F3F, 0x00000399}, {0x00001F3F, 0x00000314}, {0x00001F3F, 0x00000342},-{0x00001F40, 0x000003BF}, {0x00001F40, 0x00000313}, {0x00001F41, 0x000003BF}, {0x00001F41, 0x00000314},-{0x00001F42, 0x000003BF}, {0x00001F42, 0x00000313}, {0x00001F42, 0x00000300}, {0x00001F43, 0x000003BF},-{0x00001F43, 0x00000314}, {0x00001F43, 0x00000300}, {0x00001F44, 0x000003BF}, {0x00001F44, 0x00000313},-{0x00001F44, 0x00000301}, {0x00001F45, 0x000003BF}, {0x00001F45, 0x00000314}, {0x00001F45, 0x00000301},-{0x00001F48, 0x0000039F}, {0x00001F48, 0x00000313}, {0x00001F49, 0x0000039F}, {0x00001F49, 0x00000314},-{0x00001F4A, 0x0000039F}, {0x00001F4A, 0x00000313}, {0x00001F4A, 0x00000300}, {0x00001F4B, 0x0000039F},-{0x00001F4B, 0x00000314}, {0x00001F4B, 0x00000300}, {0x00001F4C, 0x0000039F}, {0x00001F4C, 0x00000313},-{0x00001F4C, 0x00000301}, {0x00001F4D, 0x0000039F}, {0x00001F4D, 0x00000314}, {0x00001F4D, 0x00000301},-{0x00001F50, 0x000003C5}, {0x00001F50, 0x00000313}, {0x00001F51, 0x000003C5}, {0x00001F51, 0x00000314},-{0x00001F52, 0x000003C5}, {0x00001F52, 0x00000313}, {0x00001F52, 0x00000300}, {0x00001F53, 0x000003C5},-{0x00001F53, 0x00000314}, {0x00001F53, 0x00000300}, {0x00001F54, 0x000003C5}, {0x00001F54, 0x00000313},-{0x00001F54, 0x00000301}, {0x00001F55, 0x000003C5}, {0x00001F55, 0x00000314}, {0x00001F55, 0x00000301},-{0x00001F56, 0x000003C5}, {0x00001F56, 0x00000313}, {0x00001F56, 0x00000342}, {0x00001F57, 0x000003C5},-{0x00001F57, 0x00000314}, {0x00001F57, 0x00000342}, {0x00001F59, 0x000003A5}, {0x00001F59, 0x00000314},-{0x00001F5B, 0x000003A5}, {0x00001F5B, 0x00000314}, {0x00001F5B, 0x00000300}, {0x00001F5D, 0x000003A5},-{0x00001F5D, 0x00000314}, {0x00001F5D, 0x00000301}, {0x00001F5F, 0x000003A5}, {0x00001F5F, 0x00000314},-{0x00001F5F, 0x00000342}, {0x00001F60, 0x000003C9}, {0x00001F60, 0x00000313}, {0x00001F61, 0x000003C9},-{0x00001F61, 0x00000314}, {0x00001F62, 0x000003C9}, {0x00001F62, 0x00000313}, {0x00001F62, 0x00000300},-{0x00001F63, 0x000003C9}, {0x00001F63, 0x00000314}, {0x00001F63, 0x00000300}, {0x00001F64, 0x000003C9},-{0x00001F64, 0x00000313}, {0x00001F64, 0x00000301}, {0x00001F65, 0x000003C9}, {0x00001F65, 0x00000314},-{0x00001F65, 0x00000301}, {0x00001F66, 0x000003C9}, {0x00001F66, 0x00000313}, {0x00001F66, 0x00000342},-{0x00001F67, 0x000003C9}, {0x00001F67, 0x00000314}, {0x00001F67, 0x00000342}, {0x00001F68, 0x000003A9},-{0x00001F68, 0x00000313}, {0x00001F69, 0x000003A9}, {0x00001F69, 0x00000314}, {0x00001F6A, 0x000003A9},-{0x00001F6A, 0x00000313}, {0x00001F6A, 0x00000300}, {0x00001F6B, 0x000003A9}, {0x00001F6B, 0x00000314},-{0x00001F6B, 0x00000300}, {0x00001F6C, 0x000003A9}, {0x00001F6C, 0x00000313}, {0x00001F6C, 0x00000301},-{0x00001F6D, 0x000003A9}, {0x00001F6D, 0x00000314}, {0x00001F6D, 0x00000301}, {0x00001F6E, 0x000003A9},-{0x00001F6E, 0x00000313}, {0x00001F6E, 0x00000342}, {0x00001F6F, 0x000003A9}, {0x00001F6F, 0x00000314},-{0x00001F6F, 0x00000342}, {0x00001F70, 0x000003B1}, {0x00001F70, 0x00000300}, {0x00001F71, 0x000003B1},-{0x00001F71, 0x00000301}, {0x00001F72, 0x000003B5}, {0x00001F72, 0x00000300}, {0x00001F73, 0x000003B5},-{0x00001F73, 0x00000301}, {0x00001F74, 0x000003B7}, {0x00001F74, 0x00000300}, {0x00001F75, 0x000003B7},-{0x00001F75, 0x00000301}, {0x00001F76, 0x000003B9}, {0x00001F76, 0x00000300}, {0x00001F77, 0x000003B9},-{0x00001F77, 0x00000301}, {0x00001F78, 0x000003BF}, {0x00001F78, 0x00000300}, {0x00001F79, 0x000003BF},-{0x00001F79, 0x00000301}, {0x00001F7A, 0x000003C5}, {0x00001F7A, 0x00000300}, {0x00001F7B, 0x000003C5},-{0x00001F7B, 0x00000301}, {0x00001F7C, 0x000003C9}, {0x00001F7C, 0x00000300}, {0x00001F7D, 0x000003C9},-{0x00001F7D, 0x00000301}, {0x00001F80, 0x000003B1}, {0x00001F80, 0x00000313}, {0x00001F80, 0x00000345},-{0x00001F81, 0x000003B1}, {0x00001F81, 0x00000314}, {0x00001F81, 0x00000345}, {0x00001F82, 0x000003B1},-{0x00001F82, 0x00000313}, {0x00001F82, 0x00000300}, {0x00001F82, 0x00000345}, {0x00001F83, 0x000003B1},-{0x00001F83, 0x00000314}, {0x00001F83, 0x00000300}, {0x00001F83, 0x00000345}, {0x00001F84, 0x000003B1},-{0x00001F84, 0x00000313}, {0x00001F84, 0x00000301}, {0x00001F84, 0x00000345}, {0x00001F85, 0x000003B1},-{0x00001F85, 0x00000314}, {0x00001F85, 0x00000301}, {0x00001F85, 0x00000345}, {0x00001F86, 0x000003B1},-{0x00001F86, 0x00000313}, {0x00001F86, 0x00000342}, {0x00001F86, 0x00000345}, {0x00001F87, 0x000003B1},-{0x00001F87, 0x00000314}, {0x00001F87, 0x00000342}, {0x00001F87, 0x00000345}, {0x00001F88, 0x00000391},-{0x00001F88, 0x00000313}, {0x00001F88, 0x00000345}, {0x00001F89, 0x00000391}, {0x00001F89, 0x00000314},-{0x00001F89, 0x00000345}, {0x00001F8A, 0x00000391}, {0x00001F8A, 0x00000313}, {0x00001F8A, 0x00000300},-{0x00001F8A, 0x00000345}, {0x00001F8B, 0x00000391}, {0x00001F8B, 0x00000314}, {0x00001F8B, 0x00000300},-{0x00001F8B, 0x00000345}, {0x00001F8C, 0x00000391}, {0x00001F8C, 0x00000313}, {0x00001F8C, 0x00000301},-{0x00001F8C, 0x00000345}, {0x00001F8D, 0x00000391}, {0x00001F8D, 0x00000314}, {0x00001F8D, 0x00000301},-{0x00001F8D, 0x00000345}, {0x00001F8E, 0x00000391}, {0x00001F8E, 0x00000313}, {0x00001F8E, 0x00000342},-{0x00001F8E, 0x00000345}, {0x00001F8F, 0x00000391}, {0x00001F8F, 0x00000314}, {0x00001F8F, 0x00000342},-{0x00001F8F, 0x00000345}, {0x00001F90, 0x000003B7}, {0x00001F90, 0x00000313}, {0x00001F90, 0x00000345},-{0x00001F91, 0x000003B7}, {0x00001F91, 0x00000314}, {0x00001F91, 0x00000345}, {0x00001F92, 0x000003B7},-{0x00001F92, 0x00000313}, {0x00001F92, 0x00000300}, {0x00001F92, 0x00000345}, {0x00001F93, 0x000003B7},-{0x00001F93, 0x00000314}, {0x00001F93, 0x00000300}, {0x00001F93, 0x00000345}, {0x00001F94, 0x000003B7},-{0x00001F94, 0x00000313}, {0x00001F94, 0x00000301}, {0x00001F94, 0x00000345}, {0x00001F95, 0x000003B7},-{0x00001F95, 0x00000314}, {0x00001F95, 0x00000301}, {0x00001F95, 0x00000345}, {0x00001F96, 0x000003B7},-{0x00001F96, 0x00000313}, {0x00001F96, 0x00000342}, {0x00001F96, 0x00000345}, {0x00001F97, 0x000003B7},-{0x00001F97, 0x00000314}, {0x00001F97, 0x00000342}, {0x00001F97, 0x00000345}, {0x00001F98, 0x00000397},-{0x00001F98, 0x00000313}, {0x00001F98, 0x00000345}, {0x00001F99, 0x00000397}, {0x00001F99, 0x00000314},-{0x00001F99, 0x00000345}, {0x00001F9A, 0x00000397}, {0x00001F9A, 0x00000313}, {0x00001F9A, 0x00000300},-{0x00001F9A, 0x00000345}, {0x00001F9B, 0x00000397}, {0x00001F9B, 0x00000314}, {0x00001F9B, 0x00000300},-{0x00001F9B, 0x00000345}, {0x00001F9C, 0x00000397}, {0x00001F9C, 0x00000313}, {0x00001F9C, 0x00000301},-{0x00001F9C, 0x00000345}, {0x00001F9D, 0x00000397}, {0x00001F9D, 0x00000314}, {0x00001F9D, 0x00000301},-{0x00001F9D, 0x00000345}, {0x00001F9E, 0x00000397}, {0x00001F9E, 0x00000313}, {0x00001F9E, 0x00000342},-{0x00001F9E, 0x00000345}, {0x00001F9F, 0x00000397}, {0x00001F9F, 0x00000314}, {0x00001F9F, 0x00000342},-{0x00001F9F, 0x00000345}, {0x00001FA0, 0x000003C9}, {0x00001FA0, 0x00000313}, {0x00001FA0, 0x00000345},-{0x00001FA1, 0x000003C9}, {0x00001FA1, 0x00000314}, {0x00001FA1, 0x00000345}, {0x00001FA2, 0x000003C9},-{0x00001FA2, 0x00000313}, {0x00001FA2, 0x00000300}, {0x00001FA2, 0x00000345}, {0x00001FA3, 0x000003C9},-{0x00001FA3, 0x00000314}, {0x00001FA3, 0x00000300}, {0x00001FA3, 0x00000345}, {0x00001FA4, 0x000003C9},-{0x00001FA4, 0x00000313}, {0x00001FA4, 0x00000301}, {0x00001FA4, 0x00000345}, {0x00001FA5, 0x000003C9},-{0x00001FA5, 0x00000314}, {0x00001FA5, 0x00000301}, {0x00001FA5, 0x00000345}, {0x00001FA6, 0x000003C9},-{0x00001FA6, 0x00000313}, {0x00001FA6, 0x00000342}, {0x00001FA6, 0x00000345}, {0x00001FA7, 0x000003C9},-{0x00001FA7, 0x00000314}, {0x00001FA7, 0x00000342}, {0x00001FA7, 0x00000345}, {0x00001FA8, 0x000003A9},-{0x00001FA8, 0x00000313}, {0x00001FA8, 0x00000345}, {0x00001FA9, 0x000003A9}, {0x00001FA9, 0x00000314},-{0x00001FA9, 0x00000345}, {0x00001FAA, 0x000003A9}, {0x00001FAA, 0x00000313}, {0x00001FAA, 0x00000300},-{0x00001FAA, 0x00000345}, {0x00001FAB, 0x000003A9}, {0x00001FAB, 0x00000314}, {0x00001FAB, 0x00000300},-{0x00001FAB, 0x00000345}, {0x00001FAC, 0x000003A9}, {0x00001FAC, 0x00000313}, {0x00001FAC, 0x00000301},-{0x00001FAC, 0x00000345}, {0x00001FAD, 0x000003A9}, {0x00001FAD, 0x00000314}, {0x00001FAD, 0x00000301},-{0x00001FAD, 0x00000345}, {0x00001FAE, 0x000003A9}, {0x00001FAE, 0x00000313}, {0x00001FAE, 0x00000342},-{0x00001FAE, 0x00000345}, {0x00001FAF, 0x000003A9}, {0x00001FAF, 0x00000314}, {0x00001FAF, 0x00000342},-{0x00001FAF, 0x00000345}, {0x00001FB0, 0x000003B1}, {0x00001FB0, 0x00000306}, {0x00001FB1, 0x000003B1},-{0x00001FB1, 0x00000304}, {0x00001FB2, 0x000003B1}, {0x00001FB2, 0x00000300}, {0x00001FB2, 0x00000345},-{0x00001FB3, 0x000003B1}, {0x00001FB3, 0x00000345}, {0x00001FB4, 0x000003B1}, {0x00001FB4, 0x00000301},-{0x00001FB4, 0x00000345}, {0x00001FB6, 0x000003B1}, {0x00001FB6, 0x00000342}, {0x00001FB7, 0x000003B1},-{0x00001FB7, 0x00000342}, {0x00001FB7, 0x00000345}, {0x00001FB8, 0x00000391}, {0x00001FB8, 0x00000306},-{0x00001FB9, 0x00000391}, {0x00001FB9, 0x00000304}, {0x00001FBA, 0x00000391}, {0x00001FBA, 0x00000300},-{0x00001FBB, 0x00000391}, {0x00001FBB, 0x00000301}, {0x00001FBC, 0x00000391}, {0x00001FBC, 0x00000345},-{0x00001FBE, 0x000003B9}, {0x00001FC1, 0x000000A8}, {0x00001FC1, 0x00000342}, {0x00001FC2, 0x000003B7},-{0x00001FC2, 0x00000300}, {0x00001FC2, 0x00000345}, {0x00001FC3, 0x000003B7}, {0x00001FC3, 0x00000345},-{0x00001FC4, 0x000003B7}, {0x00001FC4, 0x00000301}, {0x00001FC4, 0x00000345}, {0x00001FC6, 0x000003B7},-{0x00001FC6, 0x00000342}, {0x00001FC7, 0x000003B7}, {0x00001FC7, 0x00000342}, {0x00001FC7, 0x00000345},-{0x00001FC8, 0x00000395}, {0x00001FC8, 0x00000300}, {0x00001FC9, 0x00000395}, {0x00001FC9, 0x00000301},-{0x00001FCA, 0x00000397}, {0x00001FCA, 0x00000300}, {0x00001FCB, 0x00000397}, {0x00001FCB, 0x00000301},-{0x00001FCC, 0x00000397}, {0x00001FCC, 0x00000345}, {0x00001FCD, 0x00001FBF}, {0x00001FCD, 0x00000300},-{0x00001FCE, 0x00001FBF}, {0x00001FCE, 0x00000301}, {0x00001FCF, 0x00001FBF}, {0x00001FCF, 0x00000342},-{0x00001FD0, 0x000003B9}, {0x00001FD0, 0x00000306}, {0x00001FD1, 0x000003B9}, {0x00001FD1, 0x00000304},-{0x00001FD2, 0x000003B9}, {0x00001FD2, 0x00000308}, {0x00001FD2, 0x00000300}, {0x00001FD3, 0x000003B9},-{0x00001FD3, 0x00000308}, {0x00001FD3, 0x00000301}, {0x00001FD6, 0x000003B9}, {0x00001FD6, 0x00000342},-{0x00001FD7, 0x000003B9}, {0x00001FD7, 0x00000308}, {0x00001FD7, 0x00000342}, {0x00001FD8, 0x00000399},-{0x00001FD8, 0x00000306}, {0x00001FD9, 0x00000399}, {0x00001FD9, 0x00000304}, {0x00001FDA, 0x00000399},-{0x00001FDA, 0x00000300}, {0x00001FDB, 0x00000399}, {0x00001FDB, 0x00000301}, {0x00001FDD, 0x00001FFE},-{0x00001FDD, 0x00000300}, {0x00001FDE, 0x00001FFE}, {0x00001FDE, 0x00000301}, {0x00001FDF, 0x00001FFE},-{0x00001FDF, 0x00000342}, {0x00001FE0, 0x000003C5}, {0x00001FE0, 0x00000306}, {0x00001FE1, 0x000003C5},-{0x00001FE1, 0x00000304}, {0x00001FE2, 0x000003C5}, {0x00001FE2, 0x00000308}, {0x00001FE2, 0x00000300},-{0x00001FE3, 0x000003C5}, {0x00001FE3, 0x00000308}, {0x00001FE3, 0x00000301}, {0x00001FE4, 0x000003C1},-{0x00001FE4, 0x00000313}, {0x00001FE5, 0x000003C1}, {0x00001FE5, 0x00000314}, {0x00001FE6, 0x000003C5},-{0x00001FE6, 0x00000342}, {0x00001FE7, 0x000003C5}, {0x00001FE7, 0x00000308}, {0x00001FE7, 0x00000342},-{0x00001FE8, 0x000003A5}, {0x00001FE8, 0x00000306}, {0x00001FE9, 0x000003A5}, {0x00001FE9, 0x00000304},-{0x00001FEA, 0x000003A5}, {0x00001FEA, 0x00000300}, {0x00001FEB, 0x000003A5}, {0x00001FEB, 0x00000301},-{0x00001FEC, 0x000003A1}, {0x00001FEC, 0x00000314}, {0x00001FED, 0x000000A8}, {0x00001FED, 0x00000300},-{0x00001FEE, 0x000000A8}, {0x00001FEE, 0x00000301}, {0x00001FEF, 0x00000060}, {0x00001FF2, 0x000003C9},-{0x00001FF2, 0x00000300}, {0x00001FF2, 0x00000345}, {0x00001FF3, 0x000003C9}, {0x00001FF3, 0x00000345},-{0x00001FF4, 0x000003C9}, {0x00001FF4, 0x00000301}, {0x00001FF4, 0x00000345}, {0x00001FF6, 0x000003C9},-{0x00001FF6, 0x00000342}, {0x00001FF7, 0x000003C9}, {0x00001FF7, 0x00000342}, {0x00001FF7, 0x00000345},-{0x00001FF8, 0x0000039F}, {0x00001FF8, 0x00000300}, {0x00001FF9, 0x0000039F}, {0x00001FF9, 0x00000301},-{0x00001FFA, 0x000003A9}, {0x00001FFA, 0x00000300}, {0x00001FFB, 0x000003A9}, {0x00001FFB, 0x00000301},-{0x00001FFC, 0x000003A9}, {0x00001FFC, 0x00000345}, {0x00001FFD, 0x000000B4}, {0x00002000, 0x00002002},-{0x00002001, 0x00002003}, {0x00002126, 0x000003A9}, {0x0000212A, 0x0000004B}, {0x0000212B, 0x00000041},-{0x0000212B, 0x0000030A}, {0x0000219A, 0x00002190}, {0x0000219A, 0x00000338}, {0x0000219B, 0x00002192},-{0x0000219B, 0x00000338}, {0x000021AE, 0x00002194}, {0x000021AE, 0x00000338}, {0x000021CD, 0x000021D0},-{0x000021CD, 0x00000338}, {0x000021CE, 0x000021D4}, {0x000021CE, 0x00000338}, {0x000021CF, 0x000021D2},-{0x000021CF, 0x00000338}, {0x00002204, 0x00002203}, {0x00002204, 0x00000338}, {0x00002209, 0x00002208},-{0x00002209, 0x00000338}, {0x0000220C, 0x0000220B}, {0x0000220C, 0x00000338}, {0x00002224, 0x00002223},-{0x00002224, 0x00000338}, {0x00002226, 0x00002225}, {0x00002226, 0x00000338}, {0x00002241, 0x0000223C},-{0x00002241, 0x00000338}, {0x00002244, 0x00002243}, {0x00002244, 0x00000338}, {0x00002247, 0x00002245},-{0x00002247, 0x00000338}, {0x00002249, 0x00002248}, {0x00002249, 0x00000338}, {0x00002260, 0x0000003D},-{0x00002260, 0x00000338}, {0x00002262, 0x00002261}, {0x00002262, 0x00000338}, {0x0000226D, 0x0000224D},-{0x0000226D, 0x00000338}, {0x0000226E, 0x0000003C}, {0x0000226E, 0x00000338}, {0x0000226F, 0x0000003E},-{0x0000226F, 0x00000338}, {0x00002270, 0x00002264}, {0x00002270, 0x00000338}, {0x00002271, 0x00002265},-{0x00002271, 0x00000338}, {0x00002274, 0x00002272}, {0x00002274, 0x00000338}, {0x00002275, 0x00002273},-{0x00002275, 0x00000338}, {0x00002278, 0x00002276}, {0x00002278, 0x00000338}, {0x00002279, 0x00002277},-{0x00002279, 0x00000338}, {0x00002280, 0x0000227A}, {0x00002280, 0x00000338}, {0x00002281, 0x0000227B},-{0x00002281, 0x00000338}, {0x00002284, 0x00002282}, {0x00002284, 0x00000338}, {0x00002285, 0x00002283},-{0x00002285, 0x00000338}, {0x00002288, 0x00002286}, {0x00002288, 0x00000338}, {0x00002289, 0x00002287},-{0x00002289, 0x00000338}, {0x000022AC, 0x000022A2}, {0x000022AC, 0x00000338}, {0x000022AD, 0x000022A8},-{0x000022AD, 0x00000338}, {0x000022AE, 0x000022A9}, {0x000022AE, 0x00000338}, {0x000022AF, 0x000022AB},-{0x000022AF, 0x00000338}, {0x000022E0, 0x0000227C}, {0x000022E0, 0x00000338}, {0x000022E1, 0x0000227D},-{0x000022E1, 0x00000338}, {0x000022E2, 0x00002291}, {0x000022E2, 0x00000338}, {0x000022E3, 0x00002292},-{0x000022E3, 0x00000338}, {0x000022EA, 0x000022B2}, {0x000022EA, 0x00000338}, {0x000022EB, 0x000022B3},-{0x000022EB, 0x00000338}, {0x000022EC, 0x000022B4}, {0x000022EC, 0x00000338}, {0x000022ED, 0x000022B5},-{0x000022ED, 0x00000338}, {0x00002329, 0x00003008}, {0x0000232A, 0x00003009}, {0x00002ADC, 0x00002ADD},-{0x00002ADC, 0x00000338}, {0x0000304C, 0x0000304B}, {0x0000304C, 0x00003099}, {0x0000304E, 0x0000304D},-{0x0000304E, 0x00003099}, {0x00003050, 0x0000304F}, {0x00003050, 0x00003099}, {0x00003052, 0x00003051},-{0x00003052, 0x00003099}, {0x00003054, 0x00003053}, {0x00003054, 0x00003099}, {0x00003056, 0x00003055},-{0x00003056, 0x00003099}, {0x00003058, 0x00003057}, {0x00003058, 0x00003099}, {0x0000305A, 0x00003059},-{0x0000305A, 0x00003099}, {0x0000305C, 0x0000305B}, {0x0000305C, 0x00003099}, {0x0000305E, 0x0000305D},-{0x0000305E, 0x00003099}, {0x00003060, 0x0000305F}, {0x00003060, 0x00003099}, {0x00003062, 0x00003061},-{0x00003062, 0x00003099}, {0x00003065, 0x00003064}, {0x00003065, 0x00003099}, {0x00003067, 0x00003066},-{0x00003067, 0x00003099}, {0x00003069, 0x00003068}, {0x00003069, 0x00003099}, {0x00003070, 0x0000306F},-{0x00003070, 0x00003099}, {0x00003071, 0x0000306F}, {0x00003071, 0x0000309A}, {0x00003073, 0x00003072},-{0x00003073, 0x00003099}, {0x00003074, 0x00003072}, {0x00003074, 0x0000309A}, {0x00003076, 0x00003075},-{0x00003076, 0x00003099}, {0x00003077, 0x00003075}, {0x00003077, 0x0000309A}, {0x00003079, 0x00003078},-{0x00003079, 0x00003099}, {0x0000307A, 0x00003078}, {0x0000307A, 0x0000309A}, {0x0000307C, 0x0000307B},-{0x0000307C, 0x00003099}, {0x0000307D, 0x0000307B}, {0x0000307D, 0x0000309A}, {0x00003094, 0x00003046},-{0x00003094, 0x00003099}, {0x0000309E, 0x0000309D}, {0x0000309E, 0x00003099}, {0x000030AC, 0x000030AB},-{0x000030AC, 0x00003099}, {0x000030AE, 0x000030AD}, {0x000030AE, 0x00003099}, {0x000030B0, 0x000030AF},-{0x000030B0, 0x00003099}, {0x000030B2, 0x000030B1}, {0x000030B2, 0x00003099}, {0x000030B4, 0x000030B3},-{0x000030B4, 0x00003099}, {0x000030B6, 0x000030B5}, {0x000030B6, 0x00003099}, {0x000030B8, 0x000030B7},-{0x000030B8, 0x00003099}, {0x000030BA, 0x000030B9}, {0x000030BA, 0x00003099}, {0x000030BC, 0x000030BB},-{0x000030BC, 0x00003099}, {0x000030BE, 0x000030BD}, {0x000030BE, 0x00003099}, {0x000030C0, 0x000030BF},-{0x000030C0, 0x00003099}, {0x000030C2, 0x000030C1}, {0x000030C2, 0x00003099}, {0x000030C5, 0x000030C4},-{0x000030C5, 0x00003099}, {0x000030C7, 0x000030C6}, {0x000030C7, 0x00003099}, {0x000030C9, 0x000030C8},-{0x000030C9, 0x00003099}, {0x000030D0, 0x000030CF}, {0x000030D0, 0x00003099}, {0x000030D1, 0x000030CF},-{0x000030D1, 0x0000309A}, {0x000030D3, 0x000030D2}, {0x000030D3, 0x00003099}, {0x000030D4, 0x000030D2},-{0x000030D4, 0x0000309A}, {0x000030D6, 0x000030D5}, {0x000030D6, 0x00003099}, {0x000030D7, 0x000030D5},-{0x000030D7, 0x0000309A}, {0x000030D9, 0x000030D8}, {0x000030D9, 0x00003099}, {0x000030DA, 0x000030D8},-{0x000030DA, 0x0000309A}, {0x000030DC, 0x000030DB}, {0x000030DC, 0x00003099}, {0x000030DD, 0x000030DB},-{0x000030DD, 0x0000309A}, {0x000030F4, 0x000030A6}, {0x000030F4, 0x00003099}, {0x000030F7, 0x000030EF},-{0x000030F7, 0x00003099}, {0x000030F8, 0x000030F0}, {0x000030F8, 0x00003099}, {0x000030F9, 0x000030F1},-{0x000030F9, 0x00003099}, {0x000030FA, 0x000030F2}, {0x000030FA, 0x00003099}, {0x000030FE, 0x000030FD},-{0x000030FE, 0x00003099}, {0x0000F900, 0x00008C48}, {0x0000F901, 0x000066F4}, {0x0000F902, 0x00008ECA},-{0x0000F903, 0x00008CC8}, {0x0000F904, 0x00006ED1}, {0x0000F905, 0x00004E32}, {0x0000F906, 0x000053E5},-{0x0000F907, 0x00009F9C}, {0x0000F908, 0x00009F9C}, {0x0000F909, 0x00005951}, {0x0000F90A, 0x000091D1},-{0x0000F90B, 0x00005587}, {0x0000F90C, 0x00005948}, {0x0000F90D, 0x000061F6}, {0x0000F90E, 0x00007669},-{0x0000F90F, 0x00007F85}, {0x0000F910, 0x0000863F}, {0x0000F911, 0x000087BA}, {0x0000F912, 0x000088F8},-{0x0000F913, 0x0000908F}, {0x0000F914, 0x00006A02}, {0x0000F915, 0x00006D1B}, {0x0000F916, 0x000070D9},-{0x0000F917, 0x000073DE}, {0x0000F918, 0x0000843D}, {0x0000F919, 0x0000916A}, {0x0000F91A, 0x000099F1},-{0x0000F91B, 0x00004E82}, {0x0000F91C, 0x00005375}, {0x0000F91D, 0x00006B04}, {0x0000F91E, 0x0000721B},-{0x0000F91F, 0x0000862D}, {0x0000F920, 0x00009E1E}, {0x0000F921, 0x00005D50}, {0x0000F922, 0x00006FEB},-{0x0000F923, 0x000085CD}, {0x0000F924, 0x00008964}, {0x0000F925, 0x000062C9}, {0x0000F926, 0x000081D8},-{0x0000F927, 0x0000881F}, {0x0000F928, 0x00005ECA}, {0x0000F929, 0x00006717}, {0x0000F92A, 0x00006D6A},-{0x0000F92B, 0x000072FC}, {0x0000F92C, 0x000090CE}, {0x0000F92D, 0x00004F86}, {0x0000F92E, 0x000051B7},-{0x0000F92F, 0x000052DE}, {0x0000F930, 0x000064C4}, {0x0000F931, 0x00006AD3}, {0x0000F932, 0x00007210},-{0x0000F933, 0x000076E7}, {0x0000F934, 0x00008001}, {0x0000F935, 0x00008606}, {0x0000F936, 0x0000865C},-{0x0000F937, 0x00008DEF}, {0x0000F938, 0x00009732}, {0x0000F939, 0x00009B6F}, {0x0000F93A, 0x00009DFA},-{0x0000F93B, 0x0000788C}, {0x0000F93C, 0x0000797F}, {0x0000F93D, 0x00007DA0}, {0x0000F93E, 0x000083C9},-{0x0000F93F, 0x00009304}, {0x0000F940, 0x00009E7F}, {0x0000F941, 0x00008AD6}, {0x0000F942, 0x000058DF},-{0x0000F943, 0x00005F04}, {0x0000F944, 0x00007C60}, {0x0000F945, 0x0000807E}, {0x0000F946, 0x00007262},-{0x0000F947, 0x000078CA}, {0x0000F948, 0x00008CC2}, {0x0000F949, 0x000096F7}, {0x0000F94A, 0x000058D8},-{0x0000F94B, 0x00005C62}, {0x0000F94C, 0x00006A13}, {0x0000F94D, 0x00006DDA}, {0x0000F94E, 0x00006F0F},-{0x0000F94F, 0x00007D2F}, {0x0000F950, 0x00007E37}, {0x0000F951, 0x0000964B}, {0x0000F952, 0x000052D2},-{0x0000F953, 0x0000808B}, {0x0000F954, 0x000051DC}, {0x0000F955, 0x000051CC}, {0x0000F956, 0x00007A1C},-{0x0000F957, 0x00007DBE}, {0x0000F958, 0x000083F1}, {0x0000F959, 0x00009675}, {0x0000F95A, 0x00008B80},-{0x0000F95B, 0x000062CF}, {0x0000F95C, 0x00006A02}, {0x0000F95D, 0x00008AFE}, {0x0000F95E, 0x00004E39},-{0x0000F95F, 0x00005BE7}, {0x0000F960, 0x00006012}, {0x0000F961, 0x00007387}, {0x0000F962, 0x00007570},-{0x0000F963, 0x00005317}, {0x0000F964, 0x000078FB}, {0x0000F965, 0x00004FBF}, {0x0000F966, 0x00005FA9},-{0x0000F967, 0x00004E0D}, {0x0000F968, 0x00006CCC}, {0x0000F969, 0x00006578}, {0x0000F96A, 0x00007D22},-{0x0000F96B, 0x000053C3}, {0x0000F96C, 0x0000585E}, {0x0000F96D, 0x00007701}, {0x0000F96E, 0x00008449},-{0x0000F96F, 0x00008AAA}, {0x0000F970, 0x00006BBA}, {0x0000F971, 0x00008FB0}, {0x0000F972, 0x00006C88},-{0x0000F973, 0x000062FE}, {0x0000F974, 0x000082E5}, {0x0000F975, 0x000063A0}, {0x0000F976, 0x00007565},-{0x0000F977, 0x00004EAE}, {0x0000F978, 0x00005169}, {0x0000F979, 0x000051C9}, {0x0000F97A, 0x00006881},-{0x0000F97B, 0x00007CE7}, {0x0000F97C, 0x0000826F}, {0x0000F97D, 0x00008AD2}, {0x0000F97E, 0x000091CF},-{0x0000F97F, 0x000052F5}, {0x0000F980, 0x00005442}, {0x0000F981, 0x00005973}, {0x0000F982, 0x00005EEC},-{0x0000F983, 0x000065C5}, {0x0000F984, 0x00006FFE}, {0x0000F985, 0x0000792A}, {0x0000F986, 0x000095AD},-{0x0000F987, 0x00009A6A}, {0x0000F988, 0x00009E97}, {0x0000F989, 0x00009ECE}, {0x0000F98A, 0x0000529B},-{0x0000F98B, 0x000066C6}, {0x0000F98C, 0x00006B77}, {0x0000F98D, 0x00008F62}, {0x0000F98E, 0x00005E74},-{0x0000F98F, 0x00006190}, {0x0000F990, 0x00006200}, {0x0000F991, 0x0000649A}, {0x0000F992, 0x00006F23},-{0x0000F993, 0x00007149}, {0x0000F994, 0x00007489}, {0x0000F995, 0x000079CA}, {0x0000F996, 0x00007DF4},-{0x0000F997, 0x0000806F}, {0x0000F998, 0x00008F26}, {0x0000F999, 0x000084EE}, {0x0000F99A, 0x00009023},-{0x0000F99B, 0x0000934A}, {0x0000F99C, 0x00005217}, {0x0000F99D, 0x000052A3}, {0x0000F99E, 0x000054BD},-{0x0000F99F, 0x000070C8}, {0x0000F9A0, 0x000088C2}, {0x0000F9A1, 0x00008AAA}, {0x0000F9A2, 0x00005EC9},-{0x0000F9A3, 0x00005FF5}, {0x0000F9A4, 0x0000637B}, {0x0000F9A5, 0x00006BAE}, {0x0000F9A6, 0x00007C3E},-{0x0000F9A7, 0x00007375}, {0x0000F9A8, 0x00004EE4}, {0x0000F9A9, 0x000056F9}, {0x0000F9AA, 0x00005BE7},-{0x0000F9AB, 0x00005DBA}, {0x0000F9AC, 0x0000601C}, {0x0000F9AD, 0x000073B2}, {0x0000F9AE, 0x00007469},-{0x0000F9AF, 0x00007F9A}, {0x0000F9B0, 0x00008046}, {0x0000F9B1, 0x00009234}, {0x0000F9B2, 0x000096F6},-{0x0000F9B3, 0x00009748}, {0x0000F9B4, 0x00009818}, {0x0000F9B5, 0x00004F8B}, {0x0000F9B6, 0x000079AE},-{0x0000F9B7, 0x000091B4}, {0x0000F9B8, 0x000096B8}, {0x0000F9B9, 0x000060E1}, {0x0000F9BA, 0x00004E86},-{0x0000F9BB, 0x000050DA}, {0x0000F9BC, 0x00005BEE}, {0x0000F9BD, 0x00005C3F}, {0x0000F9BE, 0x00006599},-{0x0000F9BF, 0x00006A02}, {0x0000F9C0, 0x000071CE}, {0x0000F9C1, 0x00007642}, {0x0000F9C2, 0x000084FC},-{0x0000F9C3, 0x0000907C}, {0x0000F9C4, 0x00009F8D}, {0x0000F9C5, 0x00006688}, {0x0000F9C6, 0x0000962E},-{0x0000F9C7, 0x00005289}, {0x0000F9C8, 0x0000677B}, {0x0000F9C9, 0x000067F3}, {0x0000F9CA, 0x00006D41},-{0x0000F9CB, 0x00006E9C}, {0x0000F9CC, 0x00007409}, {0x0000F9CD, 0x00007559}, {0x0000F9CE, 0x0000786B},-{0x0000F9CF, 0x00007D10}, {0x0000F9D0, 0x0000985E}, {0x0000F9D1, 0x0000516D}, {0x0000F9D2, 0x0000622E},-{0x0000F9D3, 0x00009678}, {0x0000F9D4, 0x0000502B}, {0x0000F9D5, 0x00005D19}, {0x0000F9D6, 0x00006DEA},-{0x0000F9D7, 0x00008F2A}, {0x0000F9D8, 0x00005F8B}, {0x0000F9D9, 0x00006144}, {0x0000F9DA, 0x00006817},-{0x0000F9DB, 0x00007387}, {0x0000F9DC, 0x00009686}, {0x0000F9DD, 0x00005229}, {0x0000F9DE, 0x0000540F},-{0x0000F9DF, 0x00005C65}, {0x0000F9E0, 0x00006613}, {0x0000F9E1, 0x0000674E}, {0x0000F9E2, 0x000068A8},-{0x0000F9E3, 0x00006CE5}, {0x0000F9E4, 0x00007406}, {0x0000F9E5, 0x000075E2}, {0x0000F9E6, 0x00007F79},-{0x0000F9E7, 0x000088CF}, {0x0000F9E8, 0x000088E1}, {0x0000F9E9, 0x000091CC}, {0x0000F9EA, 0x000096E2},-{0x0000F9EB, 0x0000533F}, {0x0000F9EC, 0x00006EBA}, {0x0000F9ED, 0x0000541D}, {0x0000F9EE, 0x000071D0},-{0x0000F9EF, 0x00007498}, {0x0000F9F0, 0x000085FA}, {0x0000F9F1, 0x000096A3}, {0x0000F9F2, 0x00009C57},-{0x0000F9F3, 0x00009E9F}, {0x0000F9F4, 0x00006797}, {0x0000F9F5, 0x00006DCB}, {0x0000F9F6, 0x000081E8},-{0x0000F9F7, 0x00007ACB}, {0x0000F9F8, 0x00007B20}, {0x0000F9F9, 0x00007C92}, {0x0000F9FA, 0x000072C0},-{0x0000F9FB, 0x00007099}, {0x0000F9FC, 0x00008B58}, {0x0000F9FD, 0x00004EC0}, {0x0000F9FE, 0x00008336},-{0x0000F9FF, 0x0000523A}, {0x0000FA00, 0x00005207}, {0x0000FA01, 0x00005EA6}, {0x0000FA02, 0x000062D3},-{0x0000FA03, 0x00007CD6}, {0x0000FA04, 0x00005B85}, {0x0000FA05, 0x00006D1E}, {0x0000FA06, 0x000066B4},-{0x0000FA07, 0x00008F3B}, {0x0000FA08, 0x0000884C}, {0x0000FA09, 0x0000964D}, {0x0000FA0A, 0x0000898B},-{0x0000FA0B, 0x00005ED3}, {0x0000FA0C, 0x00005140}, {0x0000FA0D, 0x000055C0}, {0x0000FA10, 0x0000585A},-{0x0000FA12, 0x00006674}, {0x0000FA15, 0x000051DE}, {0x0000FA16, 0x0000732A}, {0x0000FA17, 0x000076CA},-{0x0000FA18, 0x0000793C}, {0x0000FA19, 0x0000795E}, {0x0000FA1A, 0x00007965}, {0x0000FA1B, 0x0000798F},-{0x0000FA1C, 0x00009756}, {0x0000FA1D, 0x00007CBE}, {0x0000FA1E, 0x00007FBD}, {0x0000FA20, 0x00008612},-{0x0000FA22, 0x00008AF8}, {0x0000FA25, 0x00009038}, {0x0000FA26, 0x000090FD}, {0x0000FA2A, 0x000098EF},-{0x0000FA2B, 0x000098FC}, {0x0000FA2C, 0x00009928}, {0x0000FA2D, 0x00009DB4}, {0x0000FA2E, 0x000090DE},-{0x0000FA2F, 0x000096B7}, {0x0000FA30, 0x00004FAE}, {0x0000FA31, 0x000050E7}, {0x0000FA32, 0x0000514D},-{0x0000FA33, 0x000052C9}, {0x0000FA34, 0x000052E4}, {0x0000FA35, 0x00005351}, {0x0000FA36, 0x0000559D},-{0x0000FA37, 0x00005606}, {0x0000FA38, 0x00005668}, {0x0000FA39, 0x00005840}, {0x0000FA3A, 0x000058A8},-{0x0000FA3B, 0x00005C64}, {0x0000FA3C, 0x00005C6E}, {0x0000FA3D, 0x00006094}, {0x0000FA3E, 0x00006168},-{0x0000FA3F, 0x0000618E}, {0x0000FA40, 0x000061F2}, {0x0000FA41, 0x0000654F}, {0x0000FA42, 0x000065E2},-{0x0000FA43, 0x00006691}, {0x0000FA44, 0x00006885}, {0x0000FA45, 0x00006D77}, {0x0000FA46, 0x00006E1A},-{0x0000FA47, 0x00006F22}, {0x0000FA48, 0x0000716E}, {0x0000FA49, 0x0000722B}, {0x0000FA4A, 0x00007422},-{0x0000FA4B, 0x00007891}, {0x0000FA4C, 0x0000793E}, {0x0000FA4D, 0x00007949}, {0x0000FA4E, 0x00007948},-{0x0000FA4F, 0x00007950}, {0x0000FA50, 0x00007956}, {0x0000FA51, 0x0000795D}, {0x0000FA52, 0x0000798D},-{0x0000FA53, 0x0000798E}, {0x0000FA54, 0x00007A40}, {0x0000FA55, 0x00007A81}, {0x0000FA56, 0x00007BC0},-{0x0000FA57, 0x00007DF4}, {0x0000FA58, 0x00007E09}, {0x0000FA59, 0x00007E41}, {0x0000FA5A, 0x00007F72},-{0x0000FA5B, 0x00008005}, {0x0000FA5C, 0x000081ED}, {0x0000FA5D, 0x00008279}, {0x0000FA5E, 0x00008279},-{0x0000FA5F, 0x00008457}, {0x0000FA60, 0x00008910}, {0x0000FA61, 0x00008996}, {0x0000FA62, 0x00008B01},-{0x0000FA63, 0x00008B39}, {0x0000FA64, 0x00008CD3}, {0x0000FA65, 0x00008D08}, {0x0000FA66, 0x00008FB6},-{0x0000FA67, 0x00009038}, {0x0000FA68, 0x000096E3}, {0x0000FA69, 0x000097FF}, {0x0000FA6A, 0x0000983B},-{0x0000FA6B, 0x00006075}, {0x0000FA6C, 0x000242EE}, {0x0000FA6D, 0x00008218}, {0x0000FA70, 0x00004E26},-{0x0000FA71, 0x000051B5}, {0x0000FA72, 0x00005168}, {0x0000FA73, 0x00004F80}, {0x0000FA74, 0x00005145},-{0x0000FA75, 0x00005180}, {0x0000FA76, 0x000052C7}, {0x0000FA77, 0x000052FA}, {0x0000FA78, 0x0000559D},-{0x0000FA79, 0x00005555}, {0x0000FA7A, 0x00005599}, {0x0000FA7B, 0x000055E2}, {0x0000FA7C, 0x0000585A},-{0x0000FA7D, 0x000058B3}, {0x0000FA7E, 0x00005944}, {0x0000FA7F, 0x00005954}, {0x0000FA80, 0x00005A62},-{0x0000FA81, 0x00005B28}, {0x0000FA82, 0x00005ED2}, {0x0000FA83, 0x00005ED9}, {0x0000FA84, 0x00005F69},-{0x0000FA85, 0x00005FAD}, {0x0000FA86, 0x000060D8}, {0x0000FA87, 0x0000614E}, {0x0000FA88, 0x00006108},-{0x0000FA89, 0x0000618E}, {0x0000FA8A, 0x00006160}, {0x0000FA8B, 0x000061F2}, {0x0000FA8C, 0x00006234},-{0x0000FA8D, 0x000063C4}, {0x0000FA8E, 0x0000641C}, {0x0000FA8F, 0x00006452}, {0x0000FA90, 0x00006556},-{0x0000FA91, 0x00006674}, {0x0000FA92, 0x00006717}, {0x0000FA93, 0x0000671B}, {0x0000FA94, 0x00006756},-{0x0000FA95, 0x00006B79}, {0x0000FA96, 0x00006BBA}, {0x0000FA97, 0x00006D41}, {0x0000FA98, 0x00006EDB},-{0x0000FA99, 0x00006ECB}, {0x0000FA9A, 0x00006F22}, {0x0000FA9B, 0x0000701E}, {0x0000FA9C, 0x0000716E},-{0x0000FA9D, 0x000077A7}, {0x0000FA9E, 0x00007235}, {0x0000FA9F, 0x000072AF}, {0x0000FAA0, 0x0000732A},-{0x0000FAA1, 0x00007471}, {0x0000FAA2, 0x00007506}, {0x0000FAA3, 0x0000753B}, {0x0000FAA4, 0x0000761D},-{0x0000FAA5, 0x0000761F}, {0x0000FAA6, 0x000076CA}, {0x0000FAA7, 0x000076DB}, {0x0000FAA8, 0x000076F4},-{0x0000FAA9, 0x0000774A}, {0x0000FAAA, 0x00007740}, {0x0000FAAB, 0x000078CC}, {0x0000FAAC, 0x00007AB1},-{0x0000FAAD, 0x00007BC0}, {0x0000FAAE, 0x00007C7B}, {0x0000FAAF, 0x00007D5B}, {0x0000FAB0, 0x00007DF4},-{0x0000FAB1, 0x00007F3E}, {0x0000FAB2, 0x00008005}, {0x0000FAB3, 0x00008352}, {0x0000FAB4, 0x000083EF},-{0x0000FAB5, 0x00008779}, {0x0000FAB6, 0x00008941}, {0x0000FAB7, 0x00008986}, {0x0000FAB8, 0x00008996},-{0x0000FAB9, 0x00008ABF}, {0x0000FABA, 0x00008AF8}, {0x0000FABB, 0x00008ACB}, {0x0000FABC, 0x00008B01},-{0x0000FABD, 0x00008AFE}, {0x0000FABE, 0x00008AED}, {0x0000FABF, 0x00008B39}, {0x0000FAC0, 0x00008B8A},-{0x0000FAC1, 0x00008D08}, {0x0000FAC2, 0x00008F38}, {0x0000FAC3, 0x00009072}, {0x0000FAC4, 0x00009199},-{0x0000FAC5, 0x00009276}, {0x0000FAC6, 0x0000967C}, {0x0000FAC7, 0x000096E3}, {0x0000FAC8, 0x00009756},-{0x0000FAC9, 0x000097DB}, {0x0000FACA, 0x000097FF}, {0x0000FACB, 0x0000980B}, {0x0000FACC, 0x0000983B},-{0x0000FACD, 0x00009B12}, {0x0000FACE, 0x00009F9C}, {0x0000FACF, 0x0002284A}, {0x0000FAD0, 0x00022844},-{0x0000FAD1, 0x000233D5}, {0x0000FAD2, 0x00003B9D}, {0x0000FAD3, 0x00004018}, {0x0000FAD4, 0x00004039},-{0x0000FAD5, 0x00025249}, {0x0000FAD6, 0x00025CD0}, {0x0000FAD7, 0x00027ED3}, {0x0000FAD8, 0x00009F43},-{0x0000FAD9, 0x00009F8E}, {0x0000FB1D, 0x000005D9}, {0x0000FB1D, 0x000005B4}, {0x0000FB1F, 0x000005F2},-{0x0000FB1F, 0x000005B7}, {0x0000FB2A, 0x000005E9}, {0x0000FB2A, 0x000005C1}, {0x0000FB2B, 0x000005E9},-{0x0000FB2B, 0x000005C2}, {0x0000FB2C, 0x000005E9}, {0x0000FB2C, 0x000005BC}, {0x0000FB2C, 0x000005C1},-{0x0000FB2D, 0x000005E9}, {0x0000FB2D, 0x000005BC}, {0x0000FB2D, 0x000005C2}, {0x0000FB2E, 0x000005D0},-{0x0000FB2E, 0x000005B7}, {0x0000FB2F, 0x000005D0}, {0x0000FB2F, 0x000005B8}, {0x0000FB30, 0x000005D0},-{0x0000FB30, 0x000005BC}, {0x0000FB31, 0x000005D1}, {0x0000FB31, 0x000005BC}, {0x0000FB32, 0x000005D2},-{0x0000FB32, 0x000005BC}, {0x0000FB33, 0x000005D3}, {0x0000FB33, 0x000005BC}, {0x0000FB34, 0x000005D4},-{0x0000FB34, 0x000005BC}, {0x0000FB35, 0x000005D5}, {0x0000FB35, 0x000005BC}, {0x0000FB36, 0x000005D6},-{0x0000FB36, 0x000005BC}, {0x0000FB38, 0x000005D8}, {0x0000FB38, 0x000005BC}, {0x0000FB39, 0x000005D9},-{0x0000FB39, 0x000005BC}, {0x0000FB3A, 0x000005DA}, {0x0000FB3A, 0x000005BC}, {0x0000FB3B, 0x000005DB},-{0x0000FB3B, 0x000005BC}, {0x0000FB3C, 0x000005DC}, {0x0000FB3C, 0x000005BC}, {0x0000FB3E, 0x000005DE},-{0x0000FB3E, 0x000005BC}, {0x0000FB40, 0x000005E0}, {0x0000FB40, 0x000005BC}, {0x0000FB41, 0x000005E1},-{0x0000FB41, 0x000005BC}, {0x0000FB43, 0x000005E3}, {0x0000FB43, 0x000005BC}, {0x0000FB44, 0x000005E4},-{0x0000FB44, 0x000005BC}, {0x0000FB46, 0x000005E6}, {0x0000FB46, 0x000005BC}, {0x0000FB47, 0x000005E7},-{0x0000FB47, 0x000005BC}, {0x0000FB48, 0x000005E8}, {0x0000FB48, 0x000005BC}, {0x0000FB49, 0x000005E9},-{0x0000FB49, 0x000005BC}, {0x0000FB4A, 0x000005EA}, {0x0000FB4A, 0x000005BC}, {0x0000FB4B, 0x000005D5},-{0x0000FB4B, 0x000005B9}, {0x0000FB4C, 0x000005D1}, {0x0000FB4C, 0x000005BF}, {0x0000FB4D, 0x000005DB},-{0x0000FB4D, 0x000005BF}, {0x0000FB4E, 0x000005E4}, {0x0000FB4E, 0x000005BF}, {0x0001109A, 0x00011099},-{0x0001109A, 0x000110BA}, {0x0001109C, 0x0001109B}, {0x0001109C, 0x000110BA}, {0x000110AB, 0x000110A5},-{0x000110AB, 0x000110BA}, {0x0001112E, 0x00011131}, {0x0001112E, 0x00011127}, {0x0001112F, 0x00011132},-{0x0001112F, 0x00011127}, {0x0001134B, 0x00011347}, {0x0001134B, 0x0001133E}, {0x0001134C, 0x00011347},-{0x0001134C, 0x00011357}, {0x000114BB, 0x000114B9}, {0x000114BB, 0x000114BA}, {0x000114BC, 0x000114B9},-{0x000114BC, 0x000114B0}, {0x000114BE, 0x000114B9}, {0x000114BE, 0x000114BD}, {0x000115BA, 0x000115B8},-{0x000115BA, 0x000115AF}, {0x000115BB, 0x000115B9}, {0x000115BB, 0x000115AF}, {0x0001D15E, 0x0001D157},-{0x0001D15E, 0x0001D165}, {0x0001D15F, 0x0001D158}, {0x0001D15F, 0x0001D165}, {0x0001D160, 0x0001D158},-{0x0001D160, 0x0001D165}, {0x0001D160, 0x0001D16E}, {0x0001D161, 0x0001D158}, {0x0001D161, 0x0001D165},-{0x0001D161, 0x0001D16F}, {0x0001D162, 0x0001D158}, {0x0001D162, 0x0001D165}, {0x0001D162, 0x0001D170},-{0x0001D163, 0x0001D158}, {0x0001D163, 0x0001D165}, {0x0001D163, 0x0001D171}, {0x0001D164, 0x0001D158},-{0x0001D164, 0x0001D165}, {0x0001D164, 0x0001D172}, {0x0001D1BB, 0x0001D1B9}, {0x0001D1BB, 0x0001D165},-{0x0001D1BC, 0x0001D1BA}, {0x0001D1BC, 0x0001D165}, {0x0001D1BD, 0x0001D1B9}, {0x0001D1BD, 0x0001D165},-{0x0001D1BD, 0x0001D16E}, {0x0001D1BE, 0x0001D1BA}, {0x0001D1BE, 0x0001D165}, {0x0001D1BE, 0x0001D16E},-{0x0001D1BF, 0x0001D1B9}, {0x0001D1BF, 0x0001D165}, {0x0001D1BF, 0x0001D16F}, {0x0001D1C0, 0x0001D1BA},-{0x0001D1C0, 0x0001D165}, {0x0001D1C0, 0x0001D16F}, {0x0002F800, 0x00004E3D}, {0x0002F801, 0x00004E38},-{0x0002F802, 0x00004E41}, {0x0002F803, 0x00020122}, {0x0002F804, 0x00004F60}, {0x0002F805, 0x00004FAE},-{0x0002F806, 0x00004FBB}, {0x0002F807, 0x00005002}, {0x0002F808, 0x0000507A}, {0x0002F809, 0x00005099},-{0x0002F80A, 0x000050E7}, {0x0002F80B, 0x000050CF}, {0x0002F80C, 0x0000349E}, {0x0002F80D, 0x0002063A},-{0x0002F80E, 0x0000514D}, {0x0002F80F, 0x00005154}, {0x0002F810, 0x00005164}, {0x0002F811, 0x00005177},-{0x0002F812, 0x0002051C}, {0x0002F813, 0x000034B9}, {0x0002F814, 0x00005167}, {0x0002F815, 0x0000518D},-{0x0002F816, 0x0002054B}, {0x0002F817, 0x00005197}, {0x0002F818, 0x000051A4}, {0x0002F819, 0x00004ECC},-{0x0002F81A, 0x000051AC}, {0x0002F81B, 0x000051B5}, {0x0002F81C, 0x000291DF}, {0x0002F81D, 0x000051F5},-{0x0002F81E, 0x00005203}, {0x0002F81F, 0x000034DF}, {0x0002F820, 0x0000523B}, {0x0002F821, 0x00005246},-{0x0002F822, 0x00005272}, {0x0002F823, 0x00005277}, {0x0002F824, 0x00003515}, {0x0002F825, 0x000052C7},-{0x0002F826, 0x000052C9}, {0x0002F827, 0x000052E4}, {0x0002F828, 0x000052FA}, {0x0002F829, 0x00005305},-{0x0002F82A, 0x00005306}, {0x0002F82B, 0x00005317}, {0x0002F82C, 0x00005349}, {0x0002F82D, 0x00005351},-{0x0002F82E, 0x0000535A}, {0x0002F82F, 0x00005373}, {0x0002F830, 0x0000537D}, {0x0002F831, 0x0000537F},-{0x0002F832, 0x0000537F}, {0x0002F833, 0x0000537F}, {0x0002F834, 0x00020A2C}, {0x0002F835, 0x00007070},-{0x0002F836, 0x000053CA}, {0x0002F837, 0x000053DF}, {0x0002F838, 0x00020B63}, {0x0002F839, 0x000053EB},-{0x0002F83A, 0x000053F1}, {0x0002F83B, 0x00005406}, {0x0002F83C, 0x0000549E}, {0x0002F83D, 0x00005438},-{0x0002F83E, 0x00005448}, {0x0002F83F, 0x00005468}, {0x0002F840, 0x000054A2}, {0x0002F841, 0x000054F6},-{0x0002F842, 0x00005510}, {0x0002F843, 0x00005553}, {0x0002F844, 0x00005563}, {0x0002F845, 0x00005584},-{0x0002F846, 0x00005584}, {0x0002F847, 0x00005599}, {0x0002F848, 0x000055AB}, {0x0002F849, 0x000055B3},-{0x0002F84A, 0x000055C2}, {0x0002F84B, 0x00005716}, {0x0002F84C, 0x00005606}, {0x0002F84D, 0x00005717},-{0x0002F84E, 0x00005651}, {0x0002F84F, 0x00005674}, {0x0002F850, 0x00005207}, {0x0002F851, 0x000058EE},-{0x0002F852, 0x000057CE}, {0x0002F853, 0x000057F4}, {0x0002F854, 0x0000580D}, {0x0002F855, 0x0000578B},-{0x0002F856, 0x00005832}, {0x0002F857, 0x00005831}, {0x0002F858, 0x000058AC}, {0x0002F859, 0x000214E4},-{0x0002F85A, 0x000058F2}, {0x0002F85B, 0x000058F7}, {0x0002F85C, 0x00005906}, {0x0002F85D, 0x0000591A},-{0x0002F85E, 0x00005922}, {0x0002F85F, 0x00005962}, {0x0002F860, 0x000216A8}, {0x0002F861, 0x000216EA},-{0x0002F862, 0x000059EC}, {0x0002F863, 0x00005A1B}, {0x0002F864, 0x00005A27}, {0x0002F865, 0x000059D8},-{0x0002F866, 0x00005A66}, {0x0002F867, 0x000036EE}, {0x0002F868, 0x000036FC}, {0x0002F869, 0x00005B08},-{0x0002F86A, 0x00005B3E}, {0x0002F86B, 0x00005B3E}, {0x0002F86C, 0x000219C8}, {0x0002F86D, 0x00005BC3},-{0x0002F86E, 0x00005BD8}, {0x0002F86F, 0x00005BE7}, {0x0002F870, 0x00005BF3}, {0x0002F871, 0x00021B18},-{0x0002F872, 0x00005BFF}, {0x0002F873, 0x00005C06}, {0x0002F874, 0x00005F53}, {0x0002F875, 0x00005C22},-{0x0002F876, 0x00003781}, {0x0002F877, 0x00005C60}, {0x0002F878, 0x00005C6E}, {0x0002F879, 0x00005CC0},-{0x0002F87A, 0x00005C8D}, {0x0002F87B, 0x00021DE4}, {0x0002F87C, 0x00005D43}, {0x0002F87D, 0x00021DE6},-{0x0002F87E, 0x00005D6E}, {0x0002F87F, 0x00005D6B}, {0x0002F880, 0x00005D7C}, {0x0002F881, 0x00005DE1},-{0x0002F882, 0x00005DE2}, {0x0002F883, 0x0000382F}, {0x0002F884, 0x00005DFD}, {0x0002F885, 0x00005E28},-{0x0002F886, 0x00005E3D}, {0x0002F887, 0x00005E69}, {0x0002F888, 0x00003862}, {0x0002F889, 0x00022183},-{0x0002F88A, 0x0000387C}, {0x0002F88B, 0x00005EB0}, {0x0002F88C, 0x00005EB3}, {0x0002F88D, 0x00005EB6},-{0x0002F88E, 0x00005ECA}, {0x0002F88F, 0x0002A392}, {0x0002F890, 0x00005EFE}, {0x0002F891, 0x00022331},-{0x0002F892, 0x00022331}, {0x0002F893, 0x00008201}, {0x0002F894, 0x00005F22}, {0x0002F895, 0x00005F22},-{0x0002F896, 0x000038C7}, {0x0002F897, 0x000232B8}, {0x0002F898, 0x000261DA}, {0x0002F899, 0x00005F62},-{0x0002F89A, 0x00005F6B}, {0x0002F89B, 0x000038E3}, {0x0002F89C, 0x00005F9A}, {0x0002F89D, 0x00005FCD},-{0x0002F89E, 0x00005FD7}, {0x0002F89F, 0x00005FF9}, {0x0002F8A0, 0x00006081}, {0x0002F8A1, 0x0000393A},-{0x0002F8A2, 0x0000391C}, {0x0002F8A3, 0x00006094}, {0x0002F8A4, 0x000226D4}, {0x0002F8A5, 0x000060C7},-{0x0002F8A6, 0x00006148}, {0x0002F8A7, 0x0000614C}, {0x0002F8A8, 0x0000614E}, {0x0002F8A9, 0x0000614C},-{0x0002F8AA, 0x0000617A}, {0x0002F8AB, 0x0000618E}, {0x0002F8AC, 0x000061B2}, {0x0002F8AD, 0x000061A4},-{0x0002F8AE, 0x000061AF}, {0x0002F8AF, 0x000061DE}, {0x0002F8B0, 0x000061F2}, {0x0002F8B1, 0x000061F6},-{0x0002F8B2, 0x00006210}, {0x0002F8B3, 0x0000621B}, {0x0002F8B4, 0x0000625D}, {0x0002F8B5, 0x000062B1},-{0x0002F8B6, 0x000062D4}, {0x0002F8B7, 0x00006350}, {0x0002F8B8, 0x00022B0C}, {0x0002F8B9, 0x0000633D},-{0x0002F8BA, 0x000062FC}, {0x0002F8BB, 0x00006368}, {0x0002F8BC, 0x00006383}, {0x0002F8BD, 0x000063E4},-{0x0002F8BE, 0x00022BF1}, {0x0002F8BF, 0x00006422}, {0x0002F8C0, 0x000063C5}, {0x0002F8C1, 0x000063A9},-{0x0002F8C2, 0x00003A2E}, {0x0002F8C3, 0x00006469}, {0x0002F8C4, 0x0000647E}, {0x0002F8C5, 0x0000649D},-{0x0002F8C6, 0x00006477}, {0x0002F8C7, 0x00003A6C}, {0x0002F8C8, 0x0000654F}, {0x0002F8C9, 0x0000656C},-{0x0002F8CA, 0x0002300A}, {0x0002F8CB, 0x000065E3}, {0x0002F8CC, 0x000066F8}, {0x0002F8CD, 0x00006649},-{0x0002F8CE, 0x00003B19}, {0x0002F8CF, 0x00006691}, {0x0002F8D0, 0x00003B08}, {0x0002F8D1, 0x00003AE4},-{0x0002F8D2, 0x00005192}, {0x0002F8D3, 0x00005195}, {0x0002F8D4, 0x00006700}, {0x0002F8D5, 0x0000669C},-{0x0002F8D6, 0x000080AD}, {0x0002F8D7, 0x000043D9}, {0x0002F8D8, 0x00006717}, {0x0002F8D9, 0x0000671B},-{0x0002F8DA, 0x00006721}, {0x0002F8DB, 0x0000675E}, {0x0002F8DC, 0x00006753}, {0x0002F8DD, 0x000233C3},-{0x0002F8DE, 0x00003B49}, {0x0002F8DF, 0x000067FA}, {0x0002F8E0, 0x00006785}, {0x0002F8E1, 0x00006852},-{0x0002F8E2, 0x00006885}, {0x0002F8E3, 0x0002346D}, {0x0002F8E4, 0x0000688E}, {0x0002F8E5, 0x0000681F},-{0x0002F8E6, 0x00006914}, {0x0002F8E7, 0x00003B9D}, {0x0002F8E8, 0x00006942}, {0x0002F8E9, 0x000069A3},-{0x0002F8EA, 0x000069EA}, {0x0002F8EB, 0x00006AA8}, {0x0002F8EC, 0x000236A3}, {0x0002F8ED, 0x00006ADB},-{0x0002F8EE, 0x00003C18}, {0x0002F8EF, 0x00006B21}, {0x0002F8F0, 0x000238A7}, {0x0002F8F1, 0x00006B54},-{0x0002F8F2, 0x00003C4E}, {0x0002F8F3, 0x00006B72}, {0x0002F8F4, 0x00006B9F}, {0x0002F8F5, 0x00006BBA},-{0x0002F8F6, 0x00006BBB}, {0x0002F8F7, 0x00023A8D}, {0x0002F8F8, 0x00021D0B}, {0x0002F8F9, 0x00023AFA},-{0x0002F8FA, 0x00006C4E}, {0x0002F8FB, 0x00023CBC}, {0x0002F8FC, 0x00006CBF}, {0x0002F8FD, 0x00006CCD},-{0x0002F8FE, 0x00006C67}, {0x0002F8FF, 0x00006D16}, {0x0002F900, 0x00006D3E}, {0x0002F901, 0x00006D77},-{0x0002F902, 0x00006D41}, {0x0002F903, 0x00006D69}, {0x0002F904, 0x00006D78}, {0x0002F905, 0x00006D85},-{0x0002F906, 0x00023D1E}, {0x0002F907, 0x00006D34}, {0x0002F908, 0x00006E2F}, {0x0002F909, 0x00006E6E},-{0x0002F90A, 0x00003D33}, {0x0002F90B, 0x00006ECB}, {0x0002F90C, 0x00006EC7}, {0x0002F90D, 0x00023ED1},-{0x0002F90E, 0x00006DF9}, {0x0002F90F, 0x00006F6E}, {0x0002F910, 0x00023F5E}, {0x0002F911, 0x00023F8E},-{0x0002F912, 0x00006FC6}, {0x0002F913, 0x00007039}, {0x0002F914, 0x0000701E}, {0x0002F915, 0x0000701B},-{0x0002F916, 0x00003D96}, {0x0002F917, 0x0000704A}, {0x0002F918, 0x0000707D}, {0x0002F919, 0x00007077},-{0x0002F91A, 0x000070AD}, {0x0002F91B, 0x00020525}, {0x0002F91C, 0x00007145}, {0x0002F91D, 0x00024263},-{0x0002F91E, 0x0000719C}, {0x0002F91F, 0x000243AB}, {0x0002F920, 0x00007228}, {0x0002F921, 0x00007235},-{0x0002F922, 0x00007250}, {0x0002F923, 0x00024608}, {0x0002F924, 0x00007280}, {0x0002F925, 0x00007295},-{0x0002F926, 0x00024735}, {0x0002F927, 0x00024814}, {0x0002F928, 0x0000737A}, {0x0002F929, 0x0000738B},-{0x0002F92A, 0x00003EAC}, {0x0002F92B, 0x000073A5}, {0x0002F92C, 0x00003EB8}, {0x0002F92D, 0x00003EB8},-{0x0002F92E, 0x00007447}, {0x0002F92F, 0x0000745C}, {0x0002F930, 0x00007471}, {0x0002F931, 0x00007485},-{0x0002F932, 0x000074CA}, {0x0002F933, 0x00003F1B}, {0x0002F934, 0x00007524}, {0x0002F935, 0x00024C36},-{0x0002F936, 0x0000753E}, {0x0002F937, 0x00024C92}, {0x0002F938, 0x00007570}, {0x0002F939, 0x0002219F},-{0x0002F93A, 0x00007610}, {0x0002F93B, 0x00024FA1}, {0x0002F93C, 0x00024FB8}, {0x0002F93D, 0x00025044},-{0x0002F93E, 0x00003FFC}, {0x0002F93F, 0x00004008}, {0x0002F940, 0x000076F4}, {0x0002F941, 0x000250F3},-{0x0002F942, 0x000250F2}, {0x0002F943, 0x00025119}, {0x0002F944, 0x00025133}, {0x0002F945, 0x0000771E},-{0x0002F946, 0x0000771F}, {0x0002F947, 0x0000771F}, {0x0002F948, 0x0000774A}, {0x0002F949, 0x00004039},-{0x0002F94A, 0x0000778B}, {0x0002F94B, 0x00004046}, {0x0002F94C, 0x00004096}, {0x0002F94D, 0x0002541D},-{0x0002F94E, 0x0000784E}, {0x0002F94F, 0x0000788C}, {0x0002F950, 0x000078CC}, {0x0002F951, 0x000040E3},-{0x0002F952, 0x00025626}, {0x0002F953, 0x00007956}, {0x0002F954, 0x0002569A}, {0x0002F955, 0x000256C5},-{0x0002F956, 0x0000798F}, {0x0002F957, 0x000079EB}, {0x0002F958, 0x0000412F}, {0x0002F959, 0x00007A40},-{0x0002F95A, 0x00007A4A}, {0x0002F95B, 0x00007A4F}, {0x0002F95C, 0x0002597C}, {0x0002F95D, 0x00025AA7},-{0x0002F95E, 0x00025AA7}, {0x0002F95F, 0x00007AEE}, {0x0002F960, 0x00004202}, {0x0002F961, 0x00025BAB},-{0x0002F962, 0x00007BC6}, {0x0002F963, 0x00007BC9}, {0x0002F964, 0x00004227}, {0x0002F965, 0x00025C80},-{0x0002F966, 0x00007CD2}, {0x0002F967, 0x000042A0}, {0x0002F968, 0x00007CE8}, {0x0002F969, 0x00007CE3},-{0x0002F96A, 0x00007D00}, {0x0002F96B, 0x00025F86}, {0x0002F96C, 0x00007D63}, {0x0002F96D, 0x00004301},-{0x0002F96E, 0x00007DC7}, {0x0002F96F, 0x00007E02}, {0x0002F970, 0x00007E45}, {0x0002F971, 0x00004334},-{0x0002F972, 0x00026228}, {0x0002F973, 0x00026247}, {0x0002F974, 0x00004359}, {0x0002F975, 0x000262D9},-{0x0002F976, 0x00007F7A}, {0x0002F977, 0x0002633E}, {0x0002F978, 0x00007F95}, {0x0002F979, 0x00007FFA},-{0x0002F97A, 0x00008005}, {0x0002F97B, 0x000264DA}, {0x0002F97C, 0x00026523}, {0x0002F97D, 0x00008060},-{0x0002F97E, 0x000265A8}, {0x0002F97F, 0x00008070}, {0x0002F980, 0x0002335F}, {0x0002F981, 0x000043D5},-{0x0002F982, 0x000080B2}, {0x0002F983, 0x00008103}, {0x0002F984, 0x0000440B}, {0x0002F985, 0x0000813E},-{0x0002F986, 0x00005AB5}, {0x0002F987, 0x000267A7}, {0x0002F988, 0x000267B5}, {0x0002F989, 0x00023393},-{0x0002F98A, 0x0002339C}, {0x0002F98B, 0x00008201}, {0x0002F98C, 0x00008204}, {0x0002F98D, 0x00008F9E},-{0x0002F98E, 0x0000446B}, {0x0002F98F, 0x00008291}, {0x0002F990, 0x0000828B}, {0x0002F991, 0x0000829D},-{0x0002F992, 0x000052B3}, {0x0002F993, 0x000082B1}, {0x0002F994, 0x000082B3}, {0x0002F995, 0x000082BD},-{0x0002F996, 0x000082E6}, {0x0002F997, 0x00026B3C}, {0x0002F998, 0x000082E5}, {0x0002F999, 0x0000831D},-{0x0002F99A, 0x00008363}, {0x0002F99B, 0x000083AD}, {0x0002F99C, 0x00008323}, {0x0002F99D, 0x000083BD},-{0x0002F99E, 0x000083E7}, {0x0002F99F, 0x00008457}, {0x0002F9A0, 0x00008353}, {0x0002F9A1, 0x000083CA},-{0x0002F9A2, 0x000083CC}, {0x0002F9A3, 0x000083DC}, {0x0002F9A4, 0x00026C36}, {0x0002F9A5, 0x00026D6B},-{0x0002F9A6, 0x00026CD5}, {0x0002F9A7, 0x0000452B}, {0x0002F9A8, 0x000084F1}, {0x0002F9A9, 0x000084F3},-{0x0002F9AA, 0x00008516}, {0x0002F9AB, 0x000273CA}, {0x0002F9AC, 0x00008564}, {0x0002F9AD, 0x00026F2C},-{0x0002F9AE, 0x0000455D}, {0x0002F9AF, 0x00004561}, {0x0002F9B0, 0x00026FB1}, {0x0002F9B1, 0x000270D2},-{0x0002F9B2, 0x0000456B}, {0x0002F9B3, 0x00008650}, {0x0002F9B4, 0x0000865C}, {0x0002F9B5, 0x00008667},-{0x0002F9B6, 0x00008669}, {0x0002F9B7, 0x000086A9}, {0x0002F9B8, 0x00008688}, {0x0002F9B9, 0x0000870E},-{0x0002F9BA, 0x000086E2}, {0x0002F9BB, 0x00008779}, {0x0002F9BC, 0x00008728}, {0x0002F9BD, 0x0000876B},-{0x0002F9BE, 0x00008786}, {0x0002F9BF, 0x000045D7}, {0x0002F9C0, 0x000087E1}, {0x0002F9C1, 0x00008801},-{0x0002F9C2, 0x000045F9}, {0x0002F9C3, 0x00008860}, {0x0002F9C4, 0x00008863}, {0x0002F9C5, 0x00027667},-{0x0002F9C6, 0x000088D7}, {0x0002F9C7, 0x000088DE}, {0x0002F9C8, 0x00004635}, {0x0002F9C9, 0x000088FA},-{0x0002F9CA, 0x000034BB}, {0x0002F9CB, 0x000278AE}, {0x0002F9CC, 0x00027966}, {0x0002F9CD, 0x000046BE},-{0x0002F9CE, 0x000046C7}, {0x0002F9CF, 0x00008AA0}, {0x0002F9D0, 0x00008AED}, {0x0002F9D1, 0x00008B8A},-{0x0002F9D2, 0x00008C55}, {0x0002F9D3, 0x00027CA8}, {0x0002F9D4, 0x00008CAB}, {0x0002F9D5, 0x00008CC1},-{0x0002F9D6, 0x00008D1B}, {0x0002F9D7, 0x00008D77}, {0x0002F9D8, 0x00027F2F}, {0x0002F9D9, 0x00020804},-{0x0002F9DA, 0x00008DCB}, {0x0002F9DB, 0x00008DBC}, {0x0002F9DC, 0x00008DF0}, {0x0002F9DD, 0x000208DE},-{0x0002F9DE, 0x00008ED4}, {0x0002F9DF, 0x00008F38}, {0x0002F9E0, 0x000285D2}, {0x0002F9E1, 0x000285ED},-{0x0002F9E2, 0x00009094}, {0x0002F9E3, 0x000090F1}, {0x0002F9E4, 0x00009111}, {0x0002F9E5, 0x0002872E},-{0x0002F9E6, 0x0000911B}, {0x0002F9E7, 0x00009238}, {0x0002F9E8, 0x000092D7}, {0x0002F9E9, 0x000092D8},-{0x0002F9EA, 0x0000927C}, {0x0002F9EB, 0x000093F9}, {0x0002F9EC, 0x00009415}, {0x0002F9ED, 0x00028BFA},-{0x0002F9EE, 0x0000958B}, {0x0002F9EF, 0x00004995}, {0x0002F9F0, 0x000095B7}, {0x0002F9F1, 0x00028D77},-{0x0002F9F2, 0x000049E6}, {0x0002F9F3, 0x000096C3}, {0x0002F9F4, 0x00005DB2}, {0x0002F9F5, 0x00009723},-{0x0002F9F6, 0x00029145}, {0x0002F9F7, 0x0002921A}, {0x0002F9F8, 0x00004A6E}, {0x0002F9F9, 0x00004A76},-{0x0002F9FA, 0x000097E0}, {0x0002F9FB, 0x0002940A}, {0x0002F9FC, 0x00004AB2}, {0x0002F9FD, 0x00029496},-{0x0002F9FE, 0x0000980B}, {0x0002F9FF, 0x0000980B}, {0x0002FA00, 0x00009829}, {0x0002FA01, 0x000295B6},-{0x0002FA02, 0x000098E2}, {0x0002FA03, 0x00004B33}, {0x0002FA04, 0x00009929}, {0x0002FA05, 0x000099A7},-{0x0002FA06, 0x000099C2}, {0x0002FA07, 0x000099FE}, {0x0002FA08, 0x00004BCE}, {0x0002FA09, 0x00029B30},-{0x0002FA0A, 0x00009B12}, {0x0002FA0B, 0x00009C40}, {0x0002FA0C, 0x00009CFD}, {0x0002FA0D, 0x00004CCE},-{0x0002FA0E, 0x00004CED}, {0x0002FA0F, 0x00009D67}, {0x0002FA10, 0x0002A0CE}, {0x0002FA11, 0x00004CF8},-{0x0002FA12, 0x0002A105}, {0x0002FA13, 0x0002A20E}, {0x0002FA14, 0x0002A291}, {0x0002FA15, 0x00009EBB},-{0x0002FA16, 0x00004D56}, {0x0002FA17, 0x00009EF9}, {0x0002FA18, 0x00009EFE}, {0x0002FA19, 0x00009F05},-{0x0002FA1A, 0x00009F0F}, {0x0002FA1B, 0x00009F16}, {0x0002FA1D, 0x0002A600},+const std::vector<range_nfd> unicode_ranges_nfd = { // start, last, nfd+{0x000000, 0x000000, 0x000000},+{0x0000C0, 0x0000C5, 0x000041},+{0x0000C7, 0x0000C7, 0x000043},+{0x0000C8, 0x0000CB, 0x000045},+{0x0000CC, 0x0000CF, 0x000049},+{0x0000D1, 0x0000D1, 0x00004E},+{0x0000D2, 0x0000D6, 0x00004F},+{0x0000D9, 0x0000DC, 0x000055},+{0x0000DD, 0x0000DD, 0x000059},+{0x0000E0, 0x0000E5, 0x000061},+{0x0000E7, 0x0000E7, 0x000063},+{0x0000E8, 0x0000EB, 0x000065},+{0x0000EC, 0x0000EF, 0x000069},+{0x0000F1, 0x0000F1, 0x00006E},+{0x0000F2, 0x0000F6, 0x00006F},+{0x0000F9, 0x0000FC, 0x000075},+{0x0000FD, 0x0000FD, 0x000079},+{0x0000FF, 0x0000FF, 0x000079},+{0x000100, 0x000100, 0x000041},+{0x000101, 0x000101, 0x000061},+{0x000102, 0x000102, 0x000041},+{0x000103, 0x000103, 0x000061},+{0x000104, 0x000104, 0x000041},+{0x000105, 0x000105, 0x000061},+{0x000106, 0x000106, 0x000043},+{0x000107, 0x000107, 0x000063},+{0x000108, 0x000108, 0x000043},+{0x000109, 0x000109, 0x000063},+{0x00010A, 0x00010A, 0x000043},+{0x00010B, 0x00010B, 0x000063},+{0x00010C, 0x00010C, 0x000043},+{0x00010D, 0x00010D, 0x000063},+{0x00010E, 0x00010E, 0x000044},+{0x00010F, 0x00010F, 0x000064},+{0x000112, 0x000112, 0x000045},+{0x000113, 0x000113, 0x000065},+{0x000114, 0x000114, 0x000045},+{0x000115, 0x000115, 0x000065},+{0x000116, 0x000116, 0x000045},+{0x000117, 0x000117, 0x000065},+{0x000118, 0x000118, 0x000045},+{0x000119, 0x000119, 0x000065},+{0x00011A, 0x00011A, 0x000045},+{0x00011B, 0x00011B, 0x000065},+{0x00011C, 0x00011C, 0x000047},+{0x00011D, 0x00011D, 0x000067},+{0x00011E, 0x00011E, 0x000047},+{0x00011F, 0x00011F, 0x000067},+{0x000120, 0x000120, 0x000047},+{0x000121, 0x000121, 0x000067},+{0x000122, 0x000122, 0x000047},+{0x000123, 0x000123, 0x000067},+{0x000124, 0x000124, 0x000048},+{0x000125, 0x000125, 0x000068},+{0x000128, 0x000128, 0x000049},+{0x000129, 0x000129, 0x000069},+{0x00012A, 0x00012A, 0x000049},+{0x00012B, 0x00012B, 0x000069},+{0x00012C, 0x00012C, 0x000049},+{0x00012D, 0x00012D, 0x000069},+{0x00012E, 0x00012E, 0x000049},+{0x00012F, 0x00012F, 0x000069},+{0x000130, 0x000130, 0x000049},+{0x000134, 0x000134, 0x00004A},+{0x000135, 0x000135, 0x00006A},+{0x000136, 0x000136, 0x00004B},+{0x000137, 0x000137, 0x00006B},+{0x000139, 0x000139, 0x00004C},+{0x00013A, 0x00013A, 0x00006C},+{0x00013B, 0x00013B, 0x00004C},+{0x00013C, 0x00013C, 0x00006C},+{0x00013D, 0x00013D, 0x00004C},+{0x00013E, 0x00013E, 0x00006C},+{0x000143, 0x000143, 0x00004E},+{0x000144, 0x000144, 0x00006E},+{0x000145, 0x000145, 0x00004E},+{0x000146, 0x000146, 0x00006E},+{0x000147, 0x000147, 0x00004E},+{0x000148, 0x000148, 0x00006E},+{0x00014C, 0x00014C, 0x00004F},+{0x00014D, 0x00014D, 0x00006F},+{0x00014E, 0x00014E, 0x00004F},+{0x00014F, 0x00014F, 0x00006F},+{0x000150, 0x000150, 0x00004F},+{0x000151, 0x000151, 0x00006F},+{0x000154, 0x000154, 0x000052},+{0x000155, 0x000155, 0x000072},+{0x000156, 0x000156, 0x000052},+{0x000157, 0x000157, 0x000072},+{0x000158, 0x000158, 0x000052},+{0x000159, 0x000159, 0x000072},+{0x00015A, 0x00015A, 0x000053},+{0x00015B, 0x00015B, 0x000073},+{0x00015C, 0x00015C, 0x000053},+{0x00015D, 0x00015D, 0x000073},+{0x00015E, 0x00015E, 0x000053},+{0x00015F, 0x00015F, 0x000073},+{0x000160, 0x000160, 0x000053},+{0x000161, 0x000161, 0x000073},+{0x000162, 0x000162, 0x000054},+{0x000163, 0x000163, 0x000074},+{0x000164, 0x000164, 0x000054},+{0x000165, 0x000165, 0x000074},+{0x000168, 0x000168, 0x000055},+{0x000169, 0x000169, 0x000075},+{0x00016A, 0x00016A, 0x000055},+{0x00016B, 0x00016B, 0x000075},+{0x00016C, 0x00016C, 0x000055},+{0x00016D, 0x00016D, 0x000075},+{0x00016E, 0x00016E, 0x000055},+{0x00016F, 0x00016F, 0x000075},+{0x000170, 0x000170, 0x000055},+{0x000171, 0x000171, 0x000075},+{0x000172, 0x000172, 0x000055},+{0x000173, 0x000173, 0x000075},+{0x000174, 0x000174, 0x000057},+{0x000175, 0x000175, 0x000077},+{0x000176, 0x000176, 0x000059},+{0x000177, 0x000177, 0x000079},+{0x000178, 0x000178, 0x000059},+{0x000179, 0x000179, 0x00005A},+{0x00017A, 0x00017A, 0x00007A},+{0x00017B, 0x00017B, 0x00005A},+{0x00017C, 0x00017C, 0x00007A},+{0x00017D, 0x00017D, 0x00005A},+{0x00017E, 0x00017E, 0x00007A},+{0x0001A0, 0x0001A0, 0x00004F},+{0x0001A1, 0x0001A1, 0x00006F},+{0x0001AF, 0x0001AF, 0x000055},+{0x0001B0, 0x0001B0, 0x000075},+{0x0001CD, 0x0001CD, 0x000041},+{0x0001CE, 0x0001CE, 0x000061},+{0x0001CF, 0x0001CF, 0x000049},+{0x0001D0, 0x0001D0, 0x000069},+{0x0001D1, 0x0001D1, 0x00004F},+{0x0001D2, 0x0001D2, 0x00006F},+{0x0001D3, 0x0001D3, 0x000055},+{0x0001D4, 0x0001D4, 0x000075},+{0x0001D5, 0x0001D5, 0x000055},+{0x0001D6, 0x0001D6, 0x000075},+{0x0001D7, 0x0001D7, 0x000055},+{0x0001D8, 0x0001D8, 0x000075},+{0x0001D9, 0x0001D9, 0x000055},+{0x0001DA, 0x0001DA, 0x000075},+{0x0001DB, 0x0001DB, 0x000055},+{0x0001DC, 0x0001DC, 0x000075},+{0x0001DE, 0x0001DE, 0x000041},+{0x0001DF, 0x0001DF, 0x000061},+{0x0001E0, 0x0001E0, 0x000041},+{0x0001E1, 0x0001E1, 0x000061},+{0x0001E2, 0x0001E2, 0x0000C6},+{0x0001E3, 0x0001E3, 0x0000E6},+{0x0001E6, 0x0001E6, 0x000047},+{0x0001E7, 0x0001E7, 0x000067},+{0x0001E8, 0x0001E8, 0x00004B},+{0x0001E9, 0x0001E9, 0x00006B},+{0x0001EA, 0x0001EA, 0x00004F},+{0x0001EB, 0x0001EB, 0x00006F},+{0x0001EC, 0x0001EC, 0x00004F},+{0x0001ED, 0x0001ED, 0x00006F},+{0x0001EE, 0x0001EE, 0x0001B7},+{0x0001EF, 0x0001EF, 0x000292},+{0x0001F0, 0x0001F0, 0x00006A},+{0x0001F4, 0x0001F4, 0x000047},+{0x0001F5, 0x0001F5, 0x000067},+{0x0001F8, 0x0001F8, 0x00004E},+{0x0001F9, 0x0001F9, 0x00006E},+{0x0001FA, 0x0001FA, 0x000041},+{0x0001FB, 0x0001FB, 0x000061},+{0x0001FC, 0x0001FC, 0x0000C6},+{0x0001FD, 0x0001FD, 0x0000E6},+{0x0001FE, 0x0001FE, 0x0000D8},+{0x0001FF, 0x0001FF, 0x0000F8},+{0x000200, 0x000200, 0x000041},+{0x000201, 0x000201, 0x000061},+{0x000202, 0x000202, 0x000041},+{0x000203, 0x000203, 0x000061},+{0x000204, 0x000204, 0x000045},+{0x000205, 0x000205, 0x000065},+{0x000206, 0x000206, 0x000045},+{0x000207, 0x000207, 0x000065},+{0x000208, 0x000208, 0x000049},+{0x000209, 0x000209, 0x000069},+{0x00020A, 0x00020A, 0x000049},+{0x00020B, 0x00020B, 0x000069},+{0x00020C, 0x00020C, 0x00004F},+{0x00020D, 0x00020D, 0x00006F},+{0x00020E, 0x00020E, 0x00004F},+{0x00020F, 0x00020F, 0x00006F},+{0x000210, 0x000210, 0x000052},+{0x000211, 0x000211, 0x000072},+{0x000212, 0x000212, 0x000052},+{0x000213, 0x000213, 0x000072},+{0x000214, 0x000214, 0x000055},+{0x000215, 0x000215, 0x000075},+{0x000216, 0x000216, 0x000055},+{0x000217, 0x000217, 0x000075},+{0x000218, 0x000218, 0x000053},+{0x000219, 0x000219, 0x000073},+{0x00021A, 0x00021A, 0x000054},+{0x00021B, 0x00021B, 0x000074},+{0x00021E, 0x00021E, 0x000048},+{0x00021F, 0x00021F, 0x000068},+{0x000226, 0x000226, 0x000041},+{0x000227, 0x000227, 0x000061},+{0x000228, 0x000228, 0x000045},+{0x000229, 0x000229, 0x000065},+{0x00022A, 0x00022A, 0x00004F},+{0x00022B, 0x00022B, 0x00006F},+{0x00022C, 0x00022C, 0x00004F},+{0x00022D, 0x00022D, 0x00006F},+{0x00022E, 0x00022E, 0x00004F},+{0x00022F, 0x00022F, 0x00006F},+{0x000230, 0x000230, 0x00004F},+{0x000231, 0x000231, 0x00006F},+{0x000232, 0x000232, 0x000059},+{0x000233, 0x000233, 0x000079},+{0x000340, 0x000340, 0x000300},+{0x000341, 0x000341, 0x000301},+{0x000343, 0x000343, 0x000313},+{0x000344, 0x000344, 0x000308},+{0x000374, 0x000374, 0x0002B9},+{0x00037E, 0x00037E, 0x00003B},+{0x000385, 0x000385, 0x0000A8},+{0x000386, 0x000386, 0x000391},+{0x000387, 0x000387, 0x0000B7},+{0x000388, 0x000388, 0x000395},+{0x000389, 0x000389, 0x000397},+{0x00038A, 0x00038A, 0x000399},+{0x00038C, 0x00038C, 0x00039F},+{0x00038E, 0x00038E, 0x0003A5},+{0x00038F, 0x00038F, 0x0003A9},+{0x000390, 0x000390, 0x0003B9},+{0x0003AA, 0x0003AA, 0x000399},+{0x0003AB, 0x0003AB, 0x0003A5},+{0x0003AC, 0x0003AC, 0x0003B1},+{0x0003AD, 0x0003AD, 0x0003B5},+{0x0003AE, 0x0003AE, 0x0003B7},+{0x0003AF, 0x0003AF, 0x0003B9},+{0x0003B0, 0x0003B0, 0x0003C5},+{0x0003CA, 0x0003CA, 0x0003B9},+{0x0003CB, 0x0003CB, 0x0003C5},+{0x0003CC, 0x0003CC, 0x0003BF},+{0x0003CD, 0x0003CD, 0x0003C5},+{0x0003CE, 0x0003CE, 0x0003C9},+{0x0003D3, 0x0003D4, 0x0003D2},+{0x000400, 0x000401, 0x000415},+{0x000403, 0x000403, 0x000413},+{0x000407, 0x000407, 0x000406},+{0x00040C, 0x00040C, 0x00041A},+{0x00040D, 0x00040D, 0x000418},+{0x00040E, 0x00040E, 0x000423},+{0x000419, 0x000419, 0x000418},+{0x000439, 0x000439, 0x000438},+{0x000450, 0x000451, 0x000435},+{0x000453, 0x000453, 0x000433},+{0x000457, 0x000457, 0x000456},+{0x00045C, 0x00045C, 0x00043A},+{0x00045D, 0x00045D, 0x000438},+{0x00045E, 0x00045E, 0x000443},+{0x000476, 0x000476, 0x000474},+{0x000477, 0x000477, 0x000475},+{0x0004C1, 0x0004C1, 0x000416},+{0x0004C2, 0x0004C2, 0x000436},+{0x0004D0, 0x0004D0, 0x000410},+{0x0004D1, 0x0004D1, 0x000430},+{0x0004D2, 0x0004D2, 0x000410},+{0x0004D3, 0x0004D3, 0x000430},+{0x0004D6, 0x0004D6, 0x000415},+{0x0004D7, 0x0004D7, 0x000435},+{0x0004DA, 0x0004DA, 0x0004D8},+{0x0004DB, 0x0004DB, 0x0004D9},+{0x0004DC, 0x0004DC, 0x000416},+{0x0004DD, 0x0004DD, 0x000436},+{0x0004DE, 0x0004DE, 0x000417},+{0x0004DF, 0x0004DF, 0x000437},+{0x0004E2, 0x0004E2, 0x000418},+{0x0004E3, 0x0004E3, 0x000438},+{0x0004E4, 0x0004E4, 0x000418},+{0x0004E5, 0x0004E5, 0x000438},+{0x0004E6, 0x0004E6, 0x00041E},+{0x0004E7, 0x0004E7, 0x00043E},+{0x0004EA, 0x0004EA, 0x0004E8},+{0x0004EB, 0x0004EB, 0x0004E9},+{0x0004EC, 0x0004EC, 0x00042D},+{0x0004ED, 0x0004ED, 0x00044D},+{0x0004EE, 0x0004EE, 0x000423},+{0x0004EF, 0x0004EF, 0x000443},+{0x0004F0, 0x0004F0, 0x000423},+{0x0004F1, 0x0004F1, 0x000443},+{0x0004F2, 0x0004F2, 0x000423},+{0x0004F3, 0x0004F3, 0x000443},+{0x0004F4, 0x0004F4, 0x000427},+{0x0004F5, 0x0004F5, 0x000447},+{0x0004F8, 0x0004F8, 0x00042B},+{0x0004F9, 0x0004F9, 0x00044B},+{0x000622, 0x000623, 0x000627},+{0x000624, 0x000624, 0x000648},+{0x000625, 0x000625, 0x000627},+{0x000626, 0x000626, 0x00064A},+{0x0006C0, 0x0006C0, 0x0006D5},+{0x0006C2, 0x0006C2, 0x0006C1},+{0x0006D3, 0x0006D3, 0x0006D2},+{0x000929, 0x000929, 0x000928},+{0x000931, 0x000931, 0x000930},+{0x000934, 0x000934, 0x000933},+{0x000958, 0x000958, 0x000915},+{0x000959, 0x000959, 0x000916},+{0x00095A, 0x00095A, 0x000917},+{0x00095B, 0x00095B, 0x00091C},+{0x00095C, 0x00095C, 0x000921},+{0x00095D, 0x00095D, 0x000922},+{0x00095E, 0x00095E, 0x00092B},+{0x00095F, 0x00095F, 0x00092F},+{0x0009CB, 0x0009CC, 0x0009C7},+{0x0009DC, 0x0009DC, 0x0009A1},+{0x0009DD, 0x0009DD, 0x0009A2},+{0x0009DF, 0x0009DF, 0x0009AF},+{0x000A33, 0x000A33, 0x000A32},+{0x000A36, 0x000A36, 0x000A38},+{0x000A59, 0x000A59, 0x000A16},+{0x000A5A, 0x000A5A, 0x000A17},+{0x000A5B, 0x000A5B, 0x000A1C},+{0x000A5E, 0x000A5E, 0x000A2B},+{0x000B48, 0x000B48, 0x000B47},+{0x000B4B, 0x000B4C, 0x000B47},+{0x000B5C, 0x000B5C, 0x000B21},+{0x000B5D, 0x000B5D, 0x000B22},+{0x000B94, 0x000B94, 0x000B92},+{0x000BCA, 0x000BCA, 0x000BC6},+{0x000BCB, 0x000BCB, 0x000BC7},+{0x000BCC, 0x000BCC, 0x000BC6},+{0x000C48, 0x000C48, 0x000C46},+{0x000CC0, 0x000CC0, 0x000CBF},+{0x000CC7, 0x000CC8, 0x000CC6},+{0x000CCA, 0x000CCB, 0x000CC6},+{0x000D4A, 0x000D4A, 0x000D46},+{0x000D4B, 0x000D4B, 0x000D47},+{0x000D4C, 0x000D4C, 0x000D46},+{0x000DDA, 0x000DDA, 0x000DD9},+{0x000DDC, 0x000DDE, 0x000DD9},+{0x000F43, 0x000F43, 0x000F42},+{0x000F4D, 0x000F4D, 0x000F4C},+{0x000F52, 0x000F52, 0x000F51},+{0x000F57, 0x000F57, 0x000F56},+{0x000F5C, 0x000F5C, 0x000F5B},+{0x000F69, 0x000F69, 0x000F40},+{0x000F73, 0x000F73, 0x000F71},+{0x000F75, 0x000F75, 0x000F71},+{0x000F76, 0x000F76, 0x000FB2},+{0x000F78, 0x000F78, 0x000FB3},+{0x000F81, 0x000F81, 0x000F71},+{0x000F93, 0x000F93, 0x000F92},+{0x000F9D, 0x000F9D, 0x000F9C},+{0x000FA2, 0x000FA2, 0x000FA1},+{0x000FA7, 0x000FA7, 0x000FA6},+{0x000FAC, 0x000FAC, 0x000FAB},+{0x000FB9, 0x000FB9, 0x000F90},+{0x001026, 0x001026, 0x001025},+{0x001B06, 0x001B06, 0x001B05},+{0x001B08, 0x001B08, 0x001B07},+{0x001B0A, 0x001B0A, 0x001B09},+{0x001B0C, 0x001B0C, 0x001B0B},+{0x001B0E, 0x001B0E, 0x001B0D},+{0x001B12, 0x001B12, 0x001B11},+{0x001B3B, 0x001B3B, 0x001B3A},+{0x001B3D, 0x001B3D, 0x001B3C},+{0x001B40, 0x001B40, 0x001B3E},+{0x001B41, 0x001B41, 0x001B3F},+{0x001B43, 0x001B43, 0x001B42},+{0x001E00, 0x001E00, 0x000041},+{0x001E01, 0x001E01, 0x000061},+{0x001E02, 0x001E02, 0x000042},+{0x001E03, 0x001E03, 0x000062},+{0x001E04, 0x001E04, 0x000042},+{0x001E05, 0x001E05, 0x000062},+{0x001E06, 0x001E06, 0x000042},+{0x001E07, 0x001E07, 0x000062},+{0x001E08, 0x001E08, 0x000043},+{0x001E09, 0x001E09, 0x000063},+{0x001E0A, 0x001E0A, 0x000044},+{0x001E0B, 0x001E0B, 0x000064},+{0x001E0C, 0x001E0C, 0x000044},+{0x001E0D, 0x001E0D, 0x000064},+{0x001E0E, 0x001E0E, 0x000044},+{0x001E0F, 0x001E0F, 0x000064},+{0x001E10, 0x001E10, 0x000044},+{0x001E11, 0x001E11, 0x000064},+{0x001E12, 0x001E12, 0x000044},+{0x001E13, 0x001E13, 0x000064},+{0x001E14, 0x001E14, 0x000045},+{0x001E15, 0x001E15, 0x000065},+{0x001E16, 0x001E16, 0x000045},+{0x001E17, 0x001E17, 0x000065},+{0x001E18, 0x001E18, 0x000045},+{0x001E19, 0x001E19, 0x000065},+{0x001E1A, 0x001E1A, 0x000045},+{0x001E1B, 0x001E1B, 0x000065},+{0x001E1C, 0x001E1C, 0x000045},+{0x001E1D, 0x001E1D, 0x000065},+{0x001E1E, 0x001E1E, 0x000046},+{0x001E1F, 0x001E1F, 0x000066},+{0x001E20, 0x001E20, 0x000047},+{0x001E21, 0x001E21, 0x000067},+{0x001E22, 0x001E22, 0x000048},+{0x001E23, 0x001E23, 0x000068},+{0x001E24, 0x001E24, 0x000048},+{0x001E25, 0x001E25, 0x000068},+{0x001E26, 0x001E26, 0x000048},+{0x001E27, 0x001E27, 0x000068},+{0x001E28, 0x001E28, 0x000048},+{0x001E29, 0x001E29, 0x000068},+{0x001E2A, 0x001E2A, 0x000048},+{0x001E2B, 0x001E2B, 0x000068},+{0x001E2C, 0x001E2C, 0x000049},+{0x001E2D, 0x001E2D, 0x000069},+{0x001E2E, 0x001E2E, 0x000049},+{0x001E2F, 0x001E2F, 0x000069},+{0x001E30, 0x001E30, 0x00004B},+{0x001E31, 0x001E31, 0x00006B},+{0x001E32, 0x001E32, 0x00004B},+{0x001E33, 0x001E33, 0x00006B},+{0x001E34, 0x001E34, 0x00004B},+{0x001E35, 0x001E35, 0x00006B},+{0x001E36, 0x001E36, 0x00004C},+{0x001E37, 0x001E37, 0x00006C},+{0x001E38, 0x001E38, 0x00004C},+{0x001E39, 0x001E39, 0x00006C},+{0x001E3A, 0x001E3A, 0x00004C},+{0x001E3B, 0x001E3B, 0x00006C},+{0x001E3C, 0x001E3C, 0x00004C},+{0x001E3D, 0x001E3D, 0x00006C},+{0x001E3E, 0x001E3E, 0x00004D},+{0x001E3F, 0x001E3F, 0x00006D},+{0x001E40, 0x001E40, 0x00004D},+{0x001E41, 0x001E41, 0x00006D},+{0x001E42, 0x001E42, 0x00004D},+{0x001E43, 0x001E43, 0x00006D},+{0x001E44, 0x001E44, 0x00004E},+{0x001E45, 0x001E45, 0x00006E},+{0x001E46, 0x001E46, 0x00004E},+{0x001E47, 0x001E47, 0x00006E},+{0x001E48, 0x001E48, 0x00004E},+{0x001E49, 0x001E49, 0x00006E},+{0x001E4A, 0x001E4A, 0x00004E},+{0x001E4B, 0x001E4B, 0x00006E},+{0x001E4C, 0x001E4C, 0x00004F},+{0x001E4D, 0x001E4D, 0x00006F},+{0x001E4E, 0x001E4E, 0x00004F},+{0x001E4F, 0x001E4F, 0x00006F},+{0x001E50, 0x001E50, 0x00004F},+{0x001E51, 0x001E51, 0x00006F},+{0x001E52, 0x001E52, 0x00004F},+{0x001E53, 0x001E53, 0x00006F},+{0x001E54, 0x001E54, 0x000050},+{0x001E55, 0x001E55, 0x000070},+{0x001E56, 0x001E56, 0x000050},+{0x001E57, 0x001E57, 0x000070},+{0x001E58, 0x001E58, 0x000052},+{0x001E59, 0x001E59, 0x000072},+{0x001E5A, 0x001E5A, 0x000052},+{0x001E5B, 0x001E5B, 0x000072},+{0x001E5C, 0x001E5C, 0x000052},+{0x001E5D, 0x001E5D, 0x000072},+{0x001E5E, 0x001E5E, 0x000052},+{0x001E5F, 0x001E5F, 0x000072},+{0x001E60, 0x001E60, 0x000053},+{0x001E61, 0x001E61, 0x000073},+{0x001E62, 0x001E62, 0x000053},+{0x001E63, 0x001E63, 0x000073},+{0x001E64, 0x001E64, 0x000053},+{0x001E65, 0x001E65, 0x000073},+{0x001E66, 0x001E66, 0x000053},+{0x001E67, 0x001E67, 0x000073},+{0x001E68, 0x001E68, 0x000053},+{0x001E69, 0x001E69, 0x000073},+{0x001E6A, 0x001E6A, 0x000054},+{0x001E6B, 0x001E6B, 0x000074},+{0x001E6C, 0x001E6C, 0x000054},+{0x001E6D, 0x001E6D, 0x000074},+{0x001E6E, 0x001E6E, 0x000054},+{0x001E6F, 0x001E6F, 0x000074},+{0x001E70, 0x001E70, 0x000054},+{0x001E71, 0x001E71, 0x000074},+{0x001E72, 0x001E72, 0x000055},+{0x001E73, 0x001E73, 0x000075},+{0x001E74, 0x001E74, 0x000055},+{0x001E75, 0x001E75, 0x000075},+{0x001E76, 0x001E76, 0x000055},+{0x001E77, 0x001E77, 0x000075},+{0x001E78, 0x001E78, 0x000055},+{0x001E79, 0x001E79, 0x000075},+{0x001E7A, 0x001E7A, 0x000055},+{0x001E7B, 0x001E7B, 0x000075},+{0x001E7C, 0x001E7C, 0x000056},+{0x001E7D, 0x001E7D, 0x000076},+{0x001E7E, 0x001E7E, 0x000056},+{0x001E7F, 0x001E7F, 0x000076},+{0x001E80, 0x001E80, 0x000057},+{0x001E81, 0x001E81, 0x000077},+{0x001E82, 0x001E82, 0x000057},+{0x001E83, 0x001E83, 0x000077},+{0x001E84, 0x001E84, 0x000057},+{0x001E85, 0x001E85, 0x000077},+{0x001E86, 0x001E86, 0x000057},+{0x001E87, 0x001E87, 0x000077},+{0x001E88, 0x001E88, 0x000057},+{0x001E89, 0x001E89, 0x000077},+{0x001E8A, 0x001E8A, 0x000058},+{0x001E8B, 0x001E8B, 0x000078},+{0x001E8C, 0x001E8C, 0x000058},+{0x001E8D, 0x001E8D, 0x000078},+{0x001E8E, 0x001E8E, 0x000059},+{0x001E8F, 0x001E8F, 0x000079},+{0x001E90, 0x001E90, 0x00005A},+{0x001E91, 0x001E91, 0x00007A},+{0x001E92, 0x001E92, 0x00005A},+{0x001E93, 0x001E93, 0x00007A},+{0x001E94, 0x001E94, 0x00005A},+{0x001E95, 0x001E95, 0x00007A},+{0x001E96, 0x001E96, 0x000068},+{0x001E97, 0x001E97, 0x000074},+{0x001E98, 0x001E98, 0x000077},+{0x001E99, 0x001E99, 0x000079},+{0x001E9B, 0x001E9B, 0x00017F},+{0x001EA0, 0x001EA0, 0x000041},+{0x001EA1, 0x001EA1, 0x000061},+{0x001EA2, 0x001EA2, 0x000041},+{0x001EA3, 0x001EA3, 0x000061},+{0x001EA4, 0x001EA4, 0x000041},+{0x001EA5, 0x001EA5, 0x000061},+{0x001EA6, 0x001EA6, 0x000041},+{0x001EA7, 0x001EA7, 0x000061},+{0x001EA8, 0x001EA8, 0x000041},+{0x001EA9, 0x001EA9, 0x000061},+{0x001EAA, 0x001EAA, 0x000041},+{0x001EAB, 0x001EAB, 0x000061},+{0x001EAC, 0x001EAC, 0x000041},+{0x001EAD, 0x001EAD, 0x000061},+{0x001EAE, 0x001EAE, 0x000041},+{0x001EAF, 0x001EAF, 0x000061},+{0x001EB0, 0x001EB0, 0x000041},+{0x001EB1, 0x001EB1, 0x000061},+{0x001EB2, 0x001EB2, 0x000041},+{0x001EB3, 0x001EB3, 0x000061},+{0x001EB4, 0x001EB4, 0x000041},+{0x001EB5, 0x001EB5, 0x000061},+{0x001EB6, 0x001EB6, 0x000041},+{0x001EB7, 0x001EB7, 0x000061},+{0x001EB8, 0x001EB8, 0x000045},+{0x001EB9, 0x001EB9, 0x000065},+{0x001EBA, 0x001EBA, 0x000045},+{0x001EBB, 0x001EBB, 0x000065},+{0x001EBC, 0x001EBC, 0x000045},+{0x001EBD, 0x001EBD, 0x000065},+{0x001EBE, 0x001EBE, 0x000045},+{0x001EBF, 0x001EBF, 0x000065},+{0x001EC0, 0x001EC0, 0x000045},+{0x001EC1, 0x001EC1, 0x000065},+{0x001EC2, 0x001EC2, 0x000045},+{0x001EC3, 0x001EC3, 0x000065},+{0x001EC4, 0x001EC4, 0x000045},+{0x001EC5, 0x001EC5, 0x000065},+{0x001EC6, 0x001EC6, 0x000045},+{0x001EC7, 0x001EC7, 0x000065},+{0x001EC8, 0x001EC8, 0x000049},+{0x001EC9, 0x001EC9, 0x000069},+{0x001ECA, 0x001ECA, 0x000049},+{0x001ECB, 0x001ECB, 0x000069},+{0x001ECC, 0x001ECC, 0x00004F},+{0x001ECD, 0x001ECD, 0x00006F},+{0x001ECE, 0x001ECE, 0x00004F},+{0x001ECF, 0x001ECF, 0x00006F},+{0x001ED0, 0x001ED0, 0x00004F},+{0x001ED1, 0x001ED1, 0x00006F},+{0x001ED2, 0x001ED2, 0x00004F},+{0x001ED3, 0x001ED3, 0x00006F},+{0x001ED4, 0x001ED4, 0x00004F},+{0x001ED5, 0x001ED5, 0x00006F},+{0x001ED6, 0x001ED6, 0x00004F},+{0x001ED7, 0x001ED7, 0x00006F},+{0x001ED8, 0x001ED8, 0x00004F},+{0x001ED9, 0x001ED9, 0x00006F},+{0x001EDA, 0x001EDA, 0x00004F},+{0x001EDB, 0x001EDB, 0x00006F},+{0x001EDC, 0x001EDC, 0x00004F},+{0x001EDD, 0x001EDD, 0x00006F},+{0x001EDE, 0x001EDE, 0x00004F},+{0x001EDF, 0x001EDF, 0x00006F},+{0x001EE0, 0x001EE0, 0x00004F},+{0x001EE1, 0x001EE1, 0x00006F},+{0x001EE2, 0x001EE2, 0x00004F},+{0x001EE3, 0x001EE3, 0x00006F},+{0x001EE4, 0x001EE4, 0x000055},+{0x001EE5, 0x001EE5, 0x000075},+{0x001EE6, 0x001EE6, 0x000055},+{0x001EE7, 0x001EE7, 0x000075},+{0x001EE8, 0x001EE8, 0x000055},+{0x001EE9, 0x001EE9, 0x000075},+{0x001EEA, 0x001EEA, 0x000055},+{0x001EEB, 0x001EEB, 0x000075},+{0x001EEC, 0x001EEC, 0x000055},+{0x001EED, 0x001EED, 0x000075},+{0x001EEE, 0x001EEE, 0x000055},+{0x001EEF, 0x001EEF, 0x000075},+{0x001EF0, 0x001EF0, 0x000055},+{0x001EF1, 0x001EF1, 0x000075},+{0x001EF2, 0x001EF2, 0x000059},+{0x001EF3, 0x001EF3, 0x000079},+{0x001EF4, 0x001EF4, 0x000059},+{0x001EF5, 0x001EF5, 0x000079},+{0x001EF6, 0x001EF6, 0x000059},+{0x001EF7, 0x001EF7, 0x000079},+{0x001EF8, 0x001EF8, 0x000059},+{0x001EF9, 0x001EF9, 0x000079},+{0x001F00, 0x001F07, 0x0003B1},+{0x001F08, 0x001F0F, 0x000391},+{0x001F10, 0x001F15, 0x0003B5},+{0x001F18, 0x001F1D, 0x000395},+{0x001F20, 0x001F27, 0x0003B7},+{0x001F28, 0x001F2F, 0x000397},+{0x001F30, 0x001F37, 0x0003B9},+{0x001F38, 0x001F3F, 0x000399},+{0x001F40, 0x001F45, 0x0003BF},+{0x001F48, 0x001F4D, 0x00039F},+{0x001F50, 0x001F57, 0x0003C5},+{0x001F59, 0x001F59, 0x0003A5},+{0x001F5B, 0x001F5B, 0x0003A5},+{0x001F5D, 0x001F5D, 0x0003A5},+{0x001F5F, 0x001F5F, 0x0003A5},+{0x001F60, 0x001F67, 0x0003C9},+{0x001F68, 0x001F6F, 0x0003A9},+{0x001F70, 0x001F71, 0x0003B1},+{0x001F72, 0x001F73, 0x0003B5},+{0x001F74, 0x001F75, 0x0003B7},+{0x001F76, 0x001F77, 0x0003B9},+{0x001F78, 0x001F79, 0x0003BF},+{0x001F7A, 0x001F7B, 0x0003C5},+{0x001F7C, 0x001F7D, 0x0003C9},+{0x001F80, 0x001F87, 0x0003B1},+{0x001F88, 0x001F8F, 0x000391},+{0x001F90, 0x001F97, 0x0003B7},+{0x001F98, 0x001F9F, 0x000397},+{0x001FA0, 0x001FA7, 0x0003C9},+{0x001FA8, 0x001FAF, 0x0003A9},+{0x001FB0, 0x001FB4, 0x0003B1},+{0x001FB6, 0x001FB7, 0x0003B1},+{0x001FB8, 0x001FBC, 0x000391},+{0x001FBE, 0x001FBE, 0x0003B9},+{0x001FC1, 0x001FC1, 0x0000A8},+{0x001FC2, 0x001FC4, 0x0003B7},+{0x001FC6, 0x001FC7, 0x0003B7},+{0x001FC8, 0x001FC9, 0x000395},+{0x001FCA, 0x001FCC, 0x000397},+{0x001FCD, 0x001FCF, 0x001FBF},+{0x001FD0, 0x001FD3, 0x0003B9},+{0x001FD6, 0x001FD7, 0x0003B9},+{0x001FD8, 0x001FDB, 0x000399},+{0x001FDD, 0x001FDF, 0x001FFE},+{0x001FE0, 0x001FE3, 0x0003C5},+{0x001FE4, 0x001FE5, 0x0003C1},+{0x001FE6, 0x001FE7, 0x0003C5},+{0x001FE8, 0x001FEB, 0x0003A5},+{0x001FEC, 0x001FEC, 0x0003A1},+{0x001FED, 0x001FEE, 0x0000A8},+{0x001FEF, 0x001FEF, 0x000060},+{0x001FF2, 0x001FF4, 0x0003C9},+{0x001FF6, 0x001FF7, 0x0003C9},+{0x001FF8, 0x001FF9, 0x00039F},+{0x001FFA, 0x001FFC, 0x0003A9},+{0x001FFD, 0x001FFD, 0x0000B4},+{0x002000, 0x002000, 0x002002},+{0x002001, 0x002001, 0x002003},+{0x002126, 0x002126, 0x0003A9},+{0x00212A, 0x00212A, 0x00004B},+{0x00212B, 0x00212B, 0x000041},+{0x00219A, 0x00219A, 0x002190},+{0x00219B, 0x00219B, 0x002192},+{0x0021AE, 0x0021AE, 0x002194},+{0x0021CD, 0x0021CD, 0x0021D0},+{0x0021CE, 0x0021CE, 0x0021D4},+{0x0021CF, 0x0021CF, 0x0021D2},+{0x002204, 0x002204, 0x002203},+{0x002209, 0x002209, 0x002208},+{0x00220C, 0x00220C, 0x00220B},+{0x002224, 0x002224, 0x002223},+{0x002226, 0x002226, 0x002225},+{0x002241, 0x002241, 0x00223C},+{0x002244, 0x002244, 0x002243},+{0x002247, 0x002247, 0x002245},+{0x002249, 0x002249, 0x002248},+{0x002260, 0x002260, 0x00003D},+{0x002262, 0x002262, 0x002261},+{0x00226D, 0x00226D, 0x00224D},+{0x00226E, 0x00226E, 0x00003C},+{0x00226F, 0x00226F, 0x00003E},+{0x002270, 0x002270, 0x002264},+{0x002271, 0x002271, 0x002265},+{0x002274, 0x002274, 0x002272},+{0x002275, 0x002275, 0x002273},+{0x002278, 0x002278, 0x002276},+{0x002279, 0x002279, 0x002277},+{0x002280, 0x002280, 0x00227A},+{0x002281, 0x002281, 0x00227B},+{0x002284, 0x002284, 0x002282},+{0x002285, 0x002285, 0x002283},+{0x002288, 0x002288, 0x002286},+{0x002289, 0x002289, 0x002287},+{0x0022AC, 0x0022AC, 0x0022A2},+{0x0022AD, 0x0022AD, 0x0022A8},+{0x0022AE, 0x0022AE, 0x0022A9},+{0x0022AF, 0x0022AF, 0x0022AB},+{0x0022E0, 0x0022E0, 0x00227C},+{0x0022E1, 0x0022E1, 0x00227D},+{0x0022E2, 0x0022E2, 0x002291},+{0x0022E3, 0x0022E3, 0x002292},+{0x0022EA, 0x0022EA, 0x0022B2},+{0x0022EB, 0x0022EB, 0x0022B3},+{0x0022EC, 0x0022EC, 0x0022B4},+{0x0022ED, 0x0022ED, 0x0022B5},+{0x002329, 0x002329, 0x003008},+{0x00232A, 0x00232A, 0x003009},+{0x002ADC, 0x002ADC, 0x002ADD},+{0x00304C, 0x00304C, 0x00304B},+{0x00304E, 0x00304E, 0x00304D},+{0x003050, 0x003050, 0x00304F},+{0x003052, 0x003052, 0x003051},+{0x003054, 0x003054, 0x003053},+{0x003056, 0x003056, 0x003055},+{0x003058, 0x003058, 0x003057},+{0x00305A, 0x00305A, 0x003059},+{0x00305C, 0x00305C, 0x00305B},+{0x00305E, 0x00305E, 0x00305D},+{0x003060, 0x003060, 0x00305F},+{0x003062, 0x003062, 0x003061},+{0x003065, 0x003065, 0x003064},+{0x003067, 0x003067, 0x003066},+{0x003069, 0x003069, 0x003068},+{0x003070, 0x003071, 0x00306F},+{0x003073, 0x003074, 0x003072},+{0x003076, 0x003077, 0x003075},+{0x003079, 0x00307A, 0x003078},+{0x00307C, 0x00307D, 0x00307B},+{0x003094, 0x003094, 0x003046},+{0x00309E, 0x00309E, 0x00309D},+{0x0030AC, 0x0030AC, 0x0030AB},+{0x0030AE, 0x0030AE, 0x0030AD},+{0x0030B0, 0x0030B0, 0x0030AF},+{0x0030B2, 0x0030B2, 0x0030B1},+{0x0030B4, 0x0030B4, 0x0030B3},+{0x0030B6, 0x0030B6, 0x0030B5},+{0x0030B8, 0x0030B8, 0x0030B7},+{0x0030BA, 0x0030BA, 0x0030B9},+{0x0030BC, 0x0030BC, 0x0030BB},+{0x0030BE, 0x0030BE, 0x0030BD},+{0x0030C0, 0x0030C0, 0x0030BF},+{0x0030C2, 0x0030C2, 0x0030C1},+{0x0030C5, 0x0030C5, 0x0030C4},+{0x0030C7, 0x0030C7, 0x0030C6},+{0x0030C9, 0x0030C9, 0x0030C8},+{0x0030D0, 0x0030D1, 0x0030CF},+{0x0030D3, 0x0030D4, 0x0030D2},+{0x0030D6, 0x0030D7, 0x0030D5},+{0x0030D9, 0x0030DA, 0x0030D8},+{0x0030DC, 0x0030DD, 0x0030DB},+{0x0030F4, 0x0030F4, 0x0030A6},+{0x0030F7, 0x0030F7, 0x0030EF},+{0x0030F8, 0x0030F8, 0x0030F0},+{0x0030F9, 0x0030F9, 0x0030F1},+{0x0030FA, 0x0030FA, 0x0030F2},+{0x0030FE, 0x0030FE, 0x0030FD},+{0x00AC00, 0x00AE4B, 0x001100},+{0x00AE4C, 0x00B097, 0x001101},+{0x00B098, 0x00B2E3, 0x001102},+{0x00B2E4, 0x00B52F, 0x001103},+{0x00B530, 0x00B77B, 0x001104},+{0x00B77C, 0x00B9C7, 0x001105},+{0x00B9C8, 0x00BC13, 0x001106},+{0x00BC14, 0x00BE5F, 0x001107},+{0x00BE60, 0x00C0AB, 0x001108},+{0x00C0AC, 0x00C2F7, 0x001109},+{0x00C2F8, 0x00C543, 0x00110A},+{0x00C544, 0x00C78F, 0x00110B},+{0x00C790, 0x00C9DB, 0x00110C},+{0x00C9DC, 0x00CC27, 0x00110D},+{0x00CC28, 0x00CE73, 0x00110E},+{0x00CE74, 0x00D0BF, 0x00110F},+{0x00D0C0, 0x00D30B, 0x001110},+{0x00D30C, 0x00D557, 0x001111},+{0x00D558, 0x00D7A3, 0x001112},+{0x00F900, 0x00F900, 0x008C48},+{0x00F901, 0x00F901, 0x0066F4},+{0x00F902, 0x00F902, 0x008ECA},+{0x00F903, 0x00F903, 0x008CC8},+{0x00F904, 0x00F904, 0x006ED1},+{0x00F905, 0x00F905, 0x004E32},+{0x00F906, 0x00F906, 0x0053E5},+{0x00F907, 0x00F908, 0x009F9C},+{0x00F909, 0x00F909, 0x005951},+{0x00F90A, 0x00F90A, 0x0091D1},+{0x00F90B, 0x00F90B, 0x005587},+{0x00F90C, 0x00F90C, 0x005948},+{0x00F90D, 0x00F90D, 0x0061F6},+{0x00F90E, 0x00F90E, 0x007669},+{0x00F90F, 0x00F90F, 0x007F85},+{0x00F910, 0x00F910, 0x00863F},+{0x00F911, 0x00F911, 0x0087BA},+{0x00F912, 0x00F912, 0x0088F8},+{0x00F913, 0x00F913, 0x00908F},+{0x00F914, 0x00F914, 0x006A02},+{0x00F915, 0x00F915, 0x006D1B},+{0x00F916, 0x00F916, 0x0070D9},+{0x00F917, 0x00F917, 0x0073DE},+{0x00F918, 0x00F918, 0x00843D},+{0x00F919, 0x00F919, 0x00916A},+{0x00F91A, 0x00F91A, 0x0099F1},+{0x00F91B, 0x00F91B, 0x004E82},+{0x00F91C, 0x00F91C, 0x005375},+{0x00F91D, 0x00F91D, 0x006B04},+{0x00F91E, 0x00F91E, 0x00721B},+{0x00F91F, 0x00F91F, 0x00862D},+{0x00F920, 0x00F920, 0x009E1E},+{0x00F921, 0x00F921, 0x005D50},+{0x00F922, 0x00F922, 0x006FEB},+{0x00F923, 0x00F923, 0x0085CD},+{0x00F924, 0x00F924, 0x008964},+{0x00F925, 0x00F925, 0x0062C9},+{0x00F926, 0x00F926, 0x0081D8},+{0x00F927, 0x00F927, 0x00881F},+{0x00F928, 0x00F928, 0x005ECA},+{0x00F929, 0x00F929, 0x006717},+{0x00F92A, 0x00F92A, 0x006D6A},+{0x00F92B, 0x00F92B, 0x0072FC},+{0x00F92C, 0x00F92C, 0x0090CE},+{0x00F92D, 0x00F92D, 0x004F86},+{0x00F92E, 0x00F92E, 0x0051B7},+{0x00F92F, 0x00F92F, 0x0052DE},+{0x00F930, 0x00F930, 0x0064C4},+{0x00F931, 0x00F931, 0x006AD3},+{0x00F932, 0x00F932, 0x007210},+{0x00F933, 0x00F933, 0x0076E7},+{0x00F934, 0x00F934, 0x008001},+{0x00F935, 0x00F935, 0x008606},+{0x00F936, 0x00F936, 0x00865C},+{0x00F937, 0x00F937, 0x008DEF},+{0x00F938, 0x00F938, 0x009732},+{0x00F939, 0x00F939, 0x009B6F},+{0x00F93A, 0x00F93A, 0x009DFA},+{0x00F93B, 0x00F93B, 0x00788C},+{0x00F93C, 0x00F93C, 0x00797F},+{0x00F93D, 0x00F93D, 0x007DA0},+{0x00F93E, 0x00F93E, 0x0083C9},+{0x00F93F, 0x00F93F, 0x009304},+{0x00F940, 0x00F940, 0x009E7F},+{0x00F941, 0x00F941, 0x008AD6},+{0x00F942, 0x00F942, 0x0058DF},+{0x00F943, 0x00F943, 0x005F04},+{0x00F944, 0x00F944, 0x007C60},+{0x00F945, 0x00F945, 0x00807E},+{0x00F946, 0x00F946, 0x007262},+{0x00F947, 0x00F947, 0x0078CA},+{0x00F948, 0x00F948, 0x008CC2},+{0x00F949, 0x00F949, 0x0096F7},+{0x00F94A, 0x00F94A, 0x0058D8},+{0x00F94B, 0x00F94B, 0x005C62},+{0x00F94C, 0x00F94C, 0x006A13},+{0x00F94D, 0x00F94D, 0x006DDA},+{0x00F94E, 0x00F94E, 0x006F0F},+{0x00F94F, 0x00F94F, 0x007D2F},+{0x00F950, 0x00F950, 0x007E37},+{0x00F951, 0x00F951, 0x00964B},+{0x00F952, 0x00F952, 0x0052D2},+{0x00F953, 0x00F953, 0x00808B},+{0x00F954, 0x00F954, 0x0051DC},+{0x00F955, 0x00F955, 0x0051CC},+{0x00F956, 0x00F956, 0x007A1C},+{0x00F957, 0x00F957, 0x007DBE},+{0x00F958, 0x00F958, 0x0083F1},+{0x00F959, 0x00F959, 0x009675},+{0x00F95A, 0x00F95A, 0x008B80},+{0x00F95B, 0x00F95B, 0x0062CF},+{0x00F95C, 0x00F95C, 0x006A02},+{0x00F95D, 0x00F95D, 0x008AFE},+{0x00F95E, 0x00F95E, 0x004E39},+{0x00F95F, 0x00F95F, 0x005BE7},+{0x00F960, 0x00F960, 0x006012},+{0x00F961, 0x00F961, 0x007387},+{0x00F962, 0x00F962, 0x007570},+{0x00F963, 0x00F963, 0x005317},+{0x00F964, 0x00F964, 0x0078FB},+{0x00F965, 0x00F965, 0x004FBF},+{0x00F966, 0x00F966, 0x005FA9},+{0x00F967, 0x00F967, 0x004E0D},+{0x00F968, 0x00F968, 0x006CCC},+{0x00F969, 0x00F969, 0x006578},+{0x00F96A, 0x00F96A, 0x007D22},+{0x00F96B, 0x00F96B, 0x0053C3},+{0x00F96C, 0x00F96C, 0x00585E},+{0x00F96D, 0x00F96D, 0x007701},+{0x00F96E, 0x00F96E, 0x008449},+{0x00F96F, 0x00F96F, 0x008AAA},+{0x00F970, 0x00F970, 0x006BBA},+{0x00F971, 0x00F971, 0x008FB0},+{0x00F972, 0x00F972, 0x006C88},+{0x00F973, 0x00F973, 0x0062FE},+{0x00F974, 0x00F974, 0x0082E5},+{0x00F975, 0x00F975, 0x0063A0},+{0x00F976, 0x00F976, 0x007565},+{0x00F977, 0x00F977, 0x004EAE},+{0x00F978, 0x00F978, 0x005169},+{0x00F979, 0x00F979, 0x0051C9},+{0x00F97A, 0x00F97A, 0x006881},+{0x00F97B, 0x00F97B, 0x007CE7},+{0x00F97C, 0x00F97C, 0x00826F},+{0x00F97D, 0x00F97D, 0x008AD2},+{0x00F97E, 0x00F97E, 0x0091CF},+{0x00F97F, 0x00F97F, 0x0052F5},+{0x00F980, 0x00F980, 0x005442},+{0x00F981, 0x00F981, 0x005973},+{0x00F982, 0x00F982, 0x005EEC},+{0x00F983, 0x00F983, 0x0065C5},+{0x00F984, 0x00F984, 0x006FFE},+{0x00F985, 0x00F985, 0x00792A},+{0x00F986, 0x00F986, 0x0095AD},+{0x00F987, 0x00F987, 0x009A6A},+{0x00F988, 0x00F988, 0x009E97},+{0x00F989, 0x00F989, 0x009ECE},+{0x00F98A, 0x00F98A, 0x00529B},+{0x00F98B, 0x00F98B, 0x0066C6},+{0x00F98C, 0x00F98C, 0x006B77},+{0x00F98D, 0x00F98D, 0x008F62},+{0x00F98E, 0x00F98E, 0x005E74},+{0x00F98F, 0x00F98F, 0x006190},+{0x00F990, 0x00F990, 0x006200},+{0x00F991, 0x00F991, 0x00649A},+{0x00F992, 0x00F992, 0x006F23},+{0x00F993, 0x00F993, 0x007149},+{0x00F994, 0x00F994, 0x007489},+{0x00F995, 0x00F995, 0x0079CA},+{0x00F996, 0x00F996, 0x007DF4},+{0x00F997, 0x00F997, 0x00806F},+{0x00F998, 0x00F998, 0x008F26},+{0x00F999, 0x00F999, 0x0084EE},+{0x00F99A, 0x00F99A, 0x009023},+{0x00F99B, 0x00F99B, 0x00934A},+{0x00F99C, 0x00F99C, 0x005217},+{0x00F99D, 0x00F99D, 0x0052A3},+{0x00F99E, 0x00F99E, 0x0054BD},+{0x00F99F, 0x00F99F, 0x0070C8},+{0x00F9A0, 0x00F9A0, 0x0088C2},+{0x00F9A1, 0x00F9A1, 0x008AAA},+{0x00F9A2, 0x00F9A2, 0x005EC9},+{0x00F9A3, 0x00F9A3, 0x005FF5},+{0x00F9A4, 0x00F9A4, 0x00637B},+{0x00F9A5, 0x00F9A5, 0x006BAE},+{0x00F9A6, 0x00F9A6, 0x007C3E},+{0x00F9A7, 0x00F9A7, 0x007375},+{0x00F9A8, 0x00F9A8, 0x004EE4},+{0x00F9A9, 0x00F9A9, 0x0056F9},+{0x00F9AA, 0x00F9AA, 0x005BE7},+{0x00F9AB, 0x00F9AB, 0x005DBA},+{0x00F9AC, 0x00F9AC, 0x00601C},+{0x00F9AD, 0x00F9AD, 0x0073B2},+{0x00F9AE, 0x00F9AE, 0x007469},+{0x00F9AF, 0x00F9AF, 0x007F9A},+{0x00F9B0, 0x00F9B0, 0x008046},+{0x00F9B1, 0x00F9B1, 0x009234},+{0x00F9B2, 0x00F9B2, 0x0096F6},+{0x00F9B3, 0x00F9B3, 0x009748},+{0x00F9B4, 0x00F9B4, 0x009818},+{0x00F9B5, 0x00F9B5, 0x004F8B},+{0x00F9B6, 0x00F9B6, 0x0079AE},+{0x00F9B7, 0x00F9B7, 0x0091B4},+{0x00F9B8, 0x00F9B8, 0x0096B8},+{0x00F9B9, 0x00F9B9, 0x0060E1},+{0x00F9BA, 0x00F9BA, 0x004E86},+{0x00F9BB, 0x00F9BB, 0x0050DA},+{0x00F9BC, 0x00F9BC, 0x005BEE},+{0x00F9BD, 0x00F9BD, 0x005C3F},+{0x00F9BE, 0x00F9BE, 0x006599},+{0x00F9BF, 0x00F9BF, 0x006A02},+{0x00F9C0, 0x00F9C0, 0x0071CE},+{0x00F9C1, 0x00F9C1, 0x007642},+{0x00F9C2, 0x00F9C2, 0x0084FC},+{0x00F9C3, 0x00F9C3, 0x00907C},+{0x00F9C4, 0x00F9C4, 0x009F8D},+{0x00F9C5, 0x00F9C5, 0x006688},+{0x00F9C6, 0x00F9C6, 0x00962E},+{0x00F9C7, 0x00F9C7, 0x005289},+{0x00F9C8, 0x00F9C8, 0x00677B},+{0x00F9C9, 0x00F9C9, 0x0067F3},+{0x00F9CA, 0x00F9CA, 0x006D41},+{0x00F9CB, 0x00F9CB, 0x006E9C},+{0x00F9CC, 0x00F9CC, 0x007409},+{0x00F9CD, 0x00F9CD, 0x007559},+{0x00F9CE, 0x00F9CE, 0x00786B},+{0x00F9CF, 0x00F9CF, 0x007D10},+{0x00F9D0, 0x00F9D0, 0x00985E},+{0x00F9D1, 0x00F9D1, 0x00516D},+{0x00F9D2, 0x00F9D2, 0x00622E},+{0x00F9D3, 0x00F9D3, 0x009678},+{0x00F9D4, 0x00F9D4, 0x00502B},+{0x00F9D5, 0x00F9D5, 0x005D19},+{0x00F9D6, 0x00F9D6, 0x006DEA},+{0x00F9D7, 0x00F9D7, 0x008F2A},+{0x00F9D8, 0x00F9D8, 0x005F8B},+{0x00F9D9, 0x00F9D9, 0x006144},+{0x00F9DA, 0x00F9DA, 0x006817},+{0x00F9DB, 0x00F9DB, 0x007387},+{0x00F9DC, 0x00F9DC, 0x009686},+{0x00F9DD, 0x00F9DD, 0x005229},+{0x00F9DE, 0x00F9DE, 0x00540F},+{0x00F9DF, 0x00F9DF, 0x005C65},+{0x00F9E0, 0x00F9E0, 0x006613},+{0x00F9E1, 0x00F9E1, 0x00674E},+{0x00F9E2, 0x00F9E2, 0x0068A8},+{0x00F9E3, 0x00F9E3, 0x006CE5},+{0x00F9E4, 0x00F9E4, 0x007406},+{0x00F9E5, 0x00F9E5, 0x0075E2},+{0x00F9E6, 0x00F9E6, 0x007F79},+{0x00F9E7, 0x00F9E7, 0x0088CF},+{0x00F9E8, 0x00F9E8, 0x0088E1},+{0x00F9E9, 0x00F9E9, 0x0091CC},+{0x00F9EA, 0x00F9EA, 0x0096E2},+{0x00F9EB, 0x00F9EB, 0x00533F},+{0x00F9EC, 0x00F9EC, 0x006EBA},+{0x00F9ED, 0x00F9ED, 0x00541D},+{0x00F9EE, 0x00F9EE, 0x0071D0},+{0x00F9EF, 0x00F9EF, 0x007498},+{0x00F9F0, 0x00F9F0, 0x0085FA},+{0x00F9F1, 0x00F9F1, 0x0096A3},+{0x00F9F2, 0x00F9F2, 0x009C57},+{0x00F9F3, 0x00F9F3, 0x009E9F},+{0x00F9F4, 0x00F9F4, 0x006797},+{0x00F9F5, 0x00F9F5, 0x006DCB},+{0x00F9F6, 0x00F9F6, 0x0081E8},+{0x00F9F7, 0x00F9F7, 0x007ACB},+{0x00F9F8, 0x00F9F8, 0x007B20},+{0x00F9F9, 0x00F9F9, 0x007C92},+{0x00F9FA, 0x00F9FA, 0x0072C0},+{0x00F9FB, 0x00F9FB, 0x007099},+{0x00F9FC, 0x00F9FC, 0x008B58},+{0x00F9FD, 0x00F9FD, 0x004EC0},+{0x00F9FE, 0x00F9FE, 0x008336},+{0x00F9FF, 0x00F9FF, 0x00523A},+{0x00FA00, 0x00FA00, 0x005207},+{0x00FA01, 0x00FA01, 0x005EA6},+{0x00FA02, 0x00FA02, 0x0062D3},+{0x00FA03, 0x00FA03, 0x007CD6},+{0x00FA04, 0x00FA04, 0x005B85},+{0x00FA05, 0x00FA05, 0x006D1E},+{0x00FA06, 0x00FA06, 0x0066B4},+{0x00FA07, 0x00FA07, 0x008F3B},+{0x00FA08, 0x00FA08, 0x00884C},+{0x00FA09, 0x00FA09, 0x00964D},+{0x00FA0A, 0x00FA0A, 0x00898B},+{0x00FA0B, 0x00FA0B, 0x005ED3},+{0x00FA0C, 0x00FA0C, 0x005140},+{0x00FA0D, 0x00FA0D, 0x0055C0},+{0x00FA10, 0x00FA10, 0x00585A},+{0x00FA12, 0x00FA12, 0x006674},+{0x00FA15, 0x00FA15, 0x0051DE},+{0x00FA16, 0x00FA16, 0x00732A},+{0x00FA17, 0x00FA17, 0x0076CA},+{0x00FA18, 0x00FA18, 0x00793C},+{0x00FA19, 0x00FA19, 0x00795E},+{0x00FA1A, 0x00FA1A, 0x007965},+{0x00FA1B, 0x00FA1B, 0x00798F},+{0x00FA1C, 0x00FA1C, 0x009756},+{0x00FA1D, 0x00FA1D, 0x007CBE},+{0x00FA1E, 0x00FA1E, 0x007FBD},+{0x00FA20, 0x00FA20, 0x008612},+{0x00FA22, 0x00FA22, 0x008AF8},+{0x00FA25, 0x00FA25, 0x009038},+{0x00FA26, 0x00FA26, 0x0090FD},+{0x00FA2A, 0x00FA2A, 0x0098EF},+{0x00FA2B, 0x00FA2B, 0x0098FC},+{0x00FA2C, 0x00FA2C, 0x009928},+{0x00FA2D, 0x00FA2D, 0x009DB4},+{0x00FA2E, 0x00FA2E, 0x0090DE},+{0x00FA2F, 0x00FA2F, 0x0096B7},+{0x00FA30, 0x00FA30, 0x004FAE},+{0x00FA31, 0x00FA31, 0x0050E7},+{0x00FA32, 0x00FA32, 0x00514D},+{0x00FA33, 0x00FA33, 0x0052C9},+{0x00FA34, 0x00FA34, 0x0052E4},+{0x00FA35, 0x00FA35, 0x005351},+{0x00FA36, 0x00FA36, 0x00559D},+{0x00FA37, 0x00FA37, 0x005606},+{0x00FA38, 0x00FA38, 0x005668},+{0x00FA39, 0x00FA39, 0x005840},+{0x00FA3A, 0x00FA3A, 0x0058A8},+{0x00FA3B, 0x00FA3B, 0x005C64},+{0x00FA3C, 0x00FA3C, 0x005C6E},+{0x00FA3D, 0x00FA3D, 0x006094},+{0x00FA3E, 0x00FA3E, 0x006168},+{0x00FA3F, 0x00FA3F, 0x00618E},+{0x00FA40, 0x00FA40, 0x0061F2},+{0x00FA41, 0x00FA41, 0x00654F},+{0x00FA42, 0x00FA42, 0x0065E2},+{0x00FA43, 0x00FA43, 0x006691},+{0x00FA44, 0x00FA44, 0x006885},+{0x00FA45, 0x00FA45, 0x006D77},+{0x00FA46, 0x00FA46, 0x006E1A},+{0x00FA47, 0x00FA47, 0x006F22},+{0x00FA48, 0x00FA48, 0x00716E},+{0x00FA49, 0x00FA49, 0x00722B},+{0x00FA4A, 0x00FA4A, 0x007422},+{0x00FA4B, 0x00FA4B, 0x007891},+{0x00FA4C, 0x00FA4C, 0x00793E},+{0x00FA4D, 0x00FA4D, 0x007949},+{0x00FA4E, 0x00FA4E, 0x007948},+{0x00FA4F, 0x00FA4F, 0x007950},+{0x00FA50, 0x00FA50, 0x007956},+{0x00FA51, 0x00FA51, 0x00795D},+{0x00FA52, 0x00FA52, 0x00798D},+{0x00FA53, 0x00FA53, 0x00798E},+{0x00FA54, 0x00FA54, 0x007A40},+{0x00FA55, 0x00FA55, 0x007A81},+{0x00FA56, 0x00FA56, 0x007BC0},+{0x00FA57, 0x00FA57, 0x007DF4},+{0x00FA58, 0x00FA58, 0x007E09},+{0x00FA59, 0x00FA59, 0x007E41},+{0x00FA5A, 0x00FA5A, 0x007F72},+{0x00FA5B, 0x00FA5B, 0x008005},+{0x00FA5C, 0x00FA5C, 0x0081ED},+{0x00FA5D, 0x00FA5E, 0x008279},+{0x00FA5F, 0x00FA5F, 0x008457},+{0x00FA60, 0x00FA60, 0x008910},+{0x00FA61, 0x00FA61, 0x008996},+{0x00FA62, 0x00FA62, 0x008B01},+{0x00FA63, 0x00FA63, 0x008B39},+{0x00FA64, 0x00FA64, 0x008CD3},+{0x00FA65, 0x00FA65, 0x008D08},+{0x00FA66, 0x00FA66, 0x008FB6},+{0x00FA67, 0x00FA67, 0x009038},+{0x00FA68, 0x00FA68, 0x0096E3},+{0x00FA69, 0x00FA69, 0x0097FF},+{0x00FA6A, 0x00FA6A, 0x00983B},+{0x00FA6B, 0x00FA6B, 0x006075},+{0x00FA6C, 0x00FA6C, 0x0242EE},+{0x00FA6D, 0x00FA6D, 0x008218},+{0x00FA70, 0x00FA70, 0x004E26},+{0x00FA71, 0x00FA71, 0x0051B5},+{0x00FA72, 0x00FA72, 0x005168},+{0x00FA73, 0x00FA73, 0x004F80},+{0x00FA74, 0x00FA74, 0x005145},+{0x00FA75, 0x00FA75, 0x005180},+{0x00FA76, 0x00FA76, 0x0052C7},+{0x00FA77, 0x00FA77, 0x0052FA},+{0x00FA78, 0x00FA78, 0x00559D},+{0x00FA79, 0x00FA79, 0x005555},+{0x00FA7A, 0x00FA7A, 0x005599},+{0x00FA7B, 0x00FA7B, 0x0055E2},+{0x00FA7C, 0x00FA7C, 0x00585A},+{0x00FA7D, 0x00FA7D, 0x0058B3},+{0x00FA7E, 0x00FA7E, 0x005944},+{0x00FA7F, 0x00FA7F, 0x005954},+{0x00FA80, 0x00FA80, 0x005A62},+{0x00FA81, 0x00FA81, 0x005B28},+{0x00FA82, 0x00FA82, 0x005ED2},+{0x00FA83, 0x00FA83, 0x005ED9},+{0x00FA84, 0x00FA84, 0x005F69},+{0x00FA85, 0x00FA85, 0x005FAD},+{0x00FA86, 0x00FA86, 0x0060D8},+{0x00FA87, 0x00FA87, 0x00614E},+{0x00FA88, 0x00FA88, 0x006108},+{0x00FA89, 0x00FA89, 0x00618E},+{0x00FA8A, 0x00FA8A, 0x006160},+{0x00FA8B, 0x00FA8B, 0x0061F2},+{0x00FA8C, 0x00FA8C, 0x006234},+{0x00FA8D, 0x00FA8D, 0x0063C4},+{0x00FA8E, 0x00FA8E, 0x00641C},+{0x00FA8F, 0x00FA8F, 0x006452},+{0x00FA90, 0x00FA90, 0x006556},+{0x00FA91, 0x00FA91, 0x006674},+{0x00FA92, 0x00FA92, 0x006717},+{0x00FA93, 0x00FA93, 0x00671B},+{0x00FA94, 0x00FA94, 0x006756},+{0x00FA95, 0x00FA95, 0x006B79},+{0x00FA96, 0x00FA96, 0x006BBA},+{0x00FA97, 0x00FA97, 0x006D41},+{0x00FA98, 0x00FA98, 0x006EDB},+{0x00FA99, 0x00FA99, 0x006ECB},+{0x00FA9A, 0x00FA9A, 0x006F22},+{0x00FA9B, 0x00FA9B, 0x00701E},+{0x00FA9C, 0x00FA9C, 0x00716E},+{0x00FA9D, 0x00FA9D, 0x0077A7},+{0x00FA9E, 0x00FA9E, 0x007235},+{0x00FA9F, 0x00FA9F, 0x0072AF},+{0x00FAA0, 0x00FAA0, 0x00732A},+{0x00FAA1, 0x00FAA1, 0x007471},+{0x00FAA2, 0x00FAA2, 0x007506},+{0x00FAA3, 0x00FAA3, 0x00753B},+{0x00FAA4, 0x00FAA4, 0x00761D},+{0x00FAA5, 0x00FAA5, 0x00761F},+{0x00FAA6, 0x00FAA6, 0x0076CA},+{0x00FAA7, 0x00FAA7, 0x0076DB},+{0x00FAA8, 0x00FAA8, 0x0076F4},+{0x00FAA9, 0x00FAA9, 0x00774A},+{0x00FAAA, 0x00FAAA, 0x007740},+{0x00FAAB, 0x00FAAB, 0x0078CC},+{0x00FAAC, 0x00FAAC, 0x007AB1},+{0x00FAAD, 0x00FAAD, 0x007BC0},+{0x00FAAE, 0x00FAAE, 0x007C7B},+{0x00FAAF, 0x00FAAF, 0x007D5B},+{0x00FAB0, 0x00FAB0, 0x007DF4},+{0x00FAB1, 0x00FAB1, 0x007F3E},+{0x00FAB2, 0x00FAB2, 0x008005},+{0x00FAB3, 0x00FAB3, 0x008352},+{0x00FAB4, 0x00FAB4, 0x0083EF},+{0x00FAB5, 0x00FAB5, 0x008779},+{0x00FAB6, 0x00FAB6, 0x008941},+{0x00FAB7, 0x00FAB7, 0x008986},+{0x00FAB8, 0x00FAB8, 0x008996},+{0x00FAB9, 0x00FAB9, 0x008ABF},+{0x00FABA, 0x00FABA, 0x008AF8},+{0x00FABB, 0x00FABB, 0x008ACB},+{0x00FABC, 0x00FABC, 0x008B01},+{0x00FABD, 0x00FABD, 0x008AFE},+{0x00FABE, 0x00FABE, 0x008AED},+{0x00FABF, 0x00FABF, 0x008B39},+{0x00FAC0, 0x00FAC0, 0x008B8A},+{0x00FAC1, 0x00FAC1, 0x008D08},+{0x00FAC2, 0x00FAC2, 0x008F38},+{0x00FAC3, 0x00FAC3, 0x009072},+{0x00FAC4, 0x00FAC4, 0x009199},+{0x00FAC5, 0x00FAC5, 0x009276},+{0x00FAC6, 0x00FAC6, 0x00967C},+{0x00FAC7, 0x00FAC7, 0x0096E3},+{0x00FAC8, 0x00FAC8, 0x009756},+{0x00FAC9, 0x00FAC9, 0x0097DB},+{0x00FACA, 0x00FACA, 0x0097FF},+{0x00FACB, 0x00FACB, 0x00980B},+{0x00FACC, 0x00FACC, 0x00983B},+{0x00FACD, 0x00FACD, 0x009B12},+{0x00FACE, 0x00FACE, 0x009F9C},+{0x00FACF, 0x00FACF, 0x02284A},+{0x00FAD0, 0x00FAD0, 0x022844},+{0x00FAD1, 0x00FAD1, 0x0233D5},+{0x00FAD2, 0x00FAD2, 0x003B9D},+{0x00FAD3, 0x00FAD3, 0x004018},+{0x00FAD4, 0x00FAD4, 0x004039},+{0x00FAD5, 0x00FAD5, 0x025249},+{0x00FAD6, 0x00FAD6, 0x025CD0},+{0x00FAD7, 0x00FAD7, 0x027ED3},+{0x00FAD8, 0x00FAD8, 0x009F43},+{0x00FAD9, 0x00FAD9, 0x009F8E},+{0x00FB1D, 0x00FB1D, 0x0005D9},+{0x00FB1F, 0x00FB1F, 0x0005F2},+{0x00FB2A, 0x00FB2D, 0x0005E9},+{0x00FB2E, 0x00FB30, 0x0005D0},+{0x00FB31, 0x00FB31, 0x0005D1},+{0x00FB32, 0x00FB32, 0x0005D2},+{0x00FB33, 0x00FB33, 0x0005D3},+{0x00FB34, 0x00FB34, 0x0005D4},+{0x00FB35, 0x00FB35, 0x0005D5},+{0x00FB36, 0x00FB36, 0x0005D6},+{0x00FB38, 0x00FB38, 0x0005D8},+{0x00FB39, 0x00FB39, 0x0005D9},+{0x00FB3A, 0x00FB3A, 0x0005DA},+{0x00FB3B, 0x00FB3B, 0x0005DB},+{0x00FB3C, 0x00FB3C, 0x0005DC},+{0x00FB3E, 0x00FB3E, 0x0005DE},+{0x00FB40, 0x00FB40, 0x0005E0},+{0x00FB41, 0x00FB41, 0x0005E1},+{0x00FB43, 0x00FB43, 0x0005E3},+{0x00FB44, 0x00FB44, 0x0005E4},+{0x00FB46, 0x00FB46, 0x0005E6},+{0x00FB47, 0x00FB47, 0x0005E7},+{0x00FB48, 0x00FB48, 0x0005E8},+{0x00FB49, 0x00FB49, 0x0005E9},+{0x00FB4A, 0x00FB4A, 0x0005EA},+{0x00FB4B, 0x00FB4B, 0x0005D5},+{0x00FB4C, 0x00FB4C, 0x0005D1},+{0x00FB4D, 0x00FB4D, 0x0005DB},+{0x00FB4E, 0x00FB4E, 0x0005E4},+{0x01109A, 0x01109A, 0x011099},+{0x01109C, 0x01109C, 0x01109B},+{0x0110AB, 0x0110AB, 0x0110A5},+{0x01112E, 0x01112E, 0x011131},+{0x01112F, 0x01112F, 0x011132},+{0x01134B, 0x01134C, 0x011347},+{0x0114BB, 0x0114BC, 0x0114B9},+{0x0114BE, 0x0114BE, 0x0114B9},+{0x0115BA, 0x0115BA, 0x0115B8},+{0x0115BB, 0x0115BB, 0x0115B9},+{0x011938, 0x011938, 0x011935},+{0x01D15E, 0x01D15E, 0x01D157},+{0x01D15F, 0x01D164, 0x01D158},+{0x01D1BB, 0x01D1BB, 0x01D1B9},+{0x01D1BC, 0x01D1BC, 0x01D1BA},+{0x01D1BD, 0x01D1BD, 0x01D1B9},+{0x01D1BE, 0x01D1BE, 0x01D1BA},+{0x01D1BF, 0x01D1BF, 0x01D1B9},+{0x01D1C0, 0x01D1C0, 0x01D1BA},+{0x02F800, 0x02F800, 0x004E3D},+{0x02F801, 0x02F801, 0x004E38},+{0x02F802, 0x02F802, 0x004E41},+{0x02F803, 0x02F803, 0x020122},+{0x02F804, 0x02F804, 0x004F60},+{0x02F805, 0x02F805, 0x004FAE},+{0x02F806, 0x02F806, 0x004FBB},+{0x02F807, 0x02F807, 0x005002},+{0x02F808, 0x02F808, 0x00507A},+{0x02F809, 0x02F809, 0x005099},+{0x02F80A, 0x02F80A, 0x0050E7},+{0x02F80B, 0x02F80B, 0x0050CF},+{0x02F80C, 0x02F80C, 0x00349E},+{0x02F80D, 0x02F80D, 0x02063A},+{0x02F80E, 0x02F80E, 0x00514D},+{0x02F80F, 0x02F80F, 0x005154},+{0x02F810, 0x02F810, 0x005164},+{0x02F811, 0x02F811, 0x005177},+{0x02F812, 0x02F812, 0x02051C},+{0x02F813, 0x02F813, 0x0034B9},+{0x02F814, 0x02F814, 0x005167},+{0x02F815, 0x02F815, 0x00518D},+{0x02F816, 0x02F816, 0x02054B},+{0x02F817, 0x02F817, 0x005197},+{0x02F818, 0x02F818, 0x0051A4},+{0x02F819, 0x02F819, 0x004ECC},+{0x02F81A, 0x02F81A, 0x0051AC},+{0x02F81B, 0x02F81B, 0x0051B5},+{0x02F81C, 0x02F81C, 0x0291DF},+{0x02F81D, 0x02F81D, 0x0051F5},+{0x02F81E, 0x02F81E, 0x005203},+{0x02F81F, 0x02F81F, 0x0034DF},+{0x02F820, 0x02F820, 0x00523B},+{0x02F821, 0x02F821, 0x005246},+{0x02F822, 0x02F822, 0x005272},+{0x02F823, 0x02F823, 0x005277},+{0x02F824, 0x02F824, 0x003515},+{0x02F825, 0x02F825, 0x0052C7},+{0x02F826, 0x02F826, 0x0052C9},+{0x02F827, 0x02F827, 0x0052E4},+{0x02F828, 0x02F828, 0x0052FA},+{0x02F829, 0x02F829, 0x005305},+{0x02F82A, 0x02F82A, 0x005306},+{0x02F82B, 0x02F82B, 0x005317},+{0x02F82C, 0x02F82C, 0x005349},+{0x02F82D, 0x02F82D, 0x005351},+{0x02F82E, 0x02F82E, 0x00535A},+{0x02F82F, 0x02F82F, 0x005373},+{0x02F830, 0x02F830, 0x00537D},+{0x02F831, 0x02F833, 0x00537F},+{0x02F834, 0x02F834, 0x020A2C},+{0x02F835, 0x02F835, 0x007070},+{0x02F836, 0x02F836, 0x0053CA},+{0x02F837, 0x02F837, 0x0053DF},+{0x02F838, 0x02F838, 0x020B63},+{0x02F839, 0x02F839, 0x0053EB},+{0x02F83A, 0x02F83A, 0x0053F1},+{0x02F83B, 0x02F83B, 0x005406},+{0x02F83C, 0x02F83C, 0x00549E},+{0x02F83D, 0x02F83D, 0x005438},+{0x02F83E, 0x02F83E, 0x005448},+{0x02F83F, 0x02F83F, 0x005468},+{0x02F840, 0x02F840, 0x0054A2},+{0x02F841, 0x02F841, 0x0054F6},+{0x02F842, 0x02F842, 0x005510},+{0x02F843, 0x02F843, 0x005553},+{0x02F844, 0x02F844, 0x005563},+{0x02F845, 0x02F846, 0x005584},+{0x02F847, 0x02F847, 0x005599},+{0x02F848, 0x02F848, 0x0055AB},+{0x02F849, 0x02F849, 0x0055B3},+{0x02F84A, 0x02F84A, 0x0055C2},+{0x02F84B, 0x02F84B, 0x005716},+{0x02F84C, 0x02F84C, 0x005606},+{0x02F84D, 0x02F84D, 0x005717},+{0x02F84E, 0x02F84E, 0x005651},+{0x02F84F, 0x02F84F, 0x005674},+{0x02F850, 0x02F850, 0x005207},+{0x02F851, 0x02F851, 0x0058EE},+{0x02F852, 0x02F852, 0x0057CE},+{0x02F853, 0x02F853, 0x0057F4},+{0x02F854, 0x02F854, 0x00580D},+{0x02F855, 0x02F855, 0x00578B},+{0x02F856, 0x02F856, 0x005832},+{0x02F857, 0x02F857, 0x005831},+{0x02F858, 0x02F858, 0x0058AC},+{0x02F859, 0x02F859, 0x0214E4},+{0x02F85A, 0x02F85A, 0x0058F2},+{0x02F85B, 0x02F85B, 0x0058F7},+{0x02F85C, 0x02F85C, 0x005906},+{0x02F85D, 0x02F85D, 0x00591A},+{0x02F85E, 0x02F85E, 0x005922},+{0x02F85F, 0x02F85F, 0x005962},+{0x02F860, 0x02F860, 0x0216A8},+{0x02F861, 0x02F861, 0x0216EA},+{0x02F862, 0x02F862, 0x0059EC},+{0x02F863, 0x02F863, 0x005A1B},+{0x02F864, 0x02F864, 0x005A27},+{0x02F865, 0x02F865, 0x0059D8},+{0x02F866, 0x02F866, 0x005A66},+{0x02F867, 0x02F867, 0x0036EE},+{0x02F868, 0x02F868, 0x0036FC},+{0x02F869, 0x02F869, 0x005B08},+{0x02F86A, 0x02F86B, 0x005B3E},+{0x02F86C, 0x02F86C, 0x0219C8},+{0x02F86D, 0x02F86D, 0x005BC3},+{0x02F86E, 0x02F86E, 0x005BD8},+{0x02F86F, 0x02F86F, 0x005BE7},+{0x02F870, 0x02F870, 0x005BF3},+{0x02F871, 0x02F871, 0x021B18},+{0x02F872, 0x02F872, 0x005BFF},+{0x02F873, 0x02F873, 0x005C06},+{0x02F874, 0x02F874, 0x005F53},+{0x02F875, 0x02F875, 0x005C22},+{0x02F876, 0x02F876, 0x003781},+{0x02F877, 0x02F877, 0x005C60},+{0x02F878, 0x02F878, 0x005C6E},+{0x02F879, 0x02F879, 0x005CC0},+{0x02F87A, 0x02F87A, 0x005C8D},+{0x02F87B, 0x02F87B, 0x021DE4},+{0x02F87C, 0x02F87C, 0x005D43},+{0x02F87D, 0x02F87D, 0x021DE6},+{0x02F87E, 0x02F87E, 0x005D6E},+{0x02F87F, 0x02F87F, 0x005D6B},+{0x02F880, 0x02F880, 0x005D7C},+{0x02F881, 0x02F881, 0x005DE1},+{0x02F882, 0x02F882, 0x005DE2},+{0x02F883, 0x02F883, 0x00382F},+{0x02F884, 0x02F884, 0x005DFD},+{0x02F885, 0x02F885, 0x005E28},+{0x02F886, 0x02F886, 0x005E3D},+{0x02F887, 0x02F887, 0x005E69},+{0x02F888, 0x02F888, 0x003862},+{0x02F889, 0x02F889, 0x022183},+{0x02F88A, 0x02F88A, 0x00387C},+{0x02F88B, 0x02F88B, 0x005EB0},+{0x02F88C, 0x02F88C, 0x005EB3},+{0x02F88D, 0x02F88D, 0x005EB6},+{0x02F88E, 0x02F88E, 0x005ECA},+{0x02F88F, 0x02F88F, 0x02A392},+{0x02F890, 0x02F890, 0x005EFE},+{0x02F891, 0x02F892, 0x022331},+{0x02F893, 0x02F893, 0x008201},+{0x02F894, 0x02F895, 0x005F22},+{0x02F896, 0x02F896, 0x0038C7},+{0x02F897, 0x02F897, 0x0232B8},+{0x02F898, 0x02F898, 0x0261DA},+{0x02F899, 0x02F899, 0x005F62},+{0x02F89A, 0x02F89A, 0x005F6B},+{0x02F89B, 0x02F89B, 0x0038E3},+{0x02F89C, 0x02F89C, 0x005F9A},+{0x02F89D, 0x02F89D, 0x005FCD},+{0x02F89E, 0x02F89E, 0x005FD7},+{0x02F89F, 0x02F89F, 0x005FF9},+{0x02F8A0, 0x02F8A0, 0x006081},+{0x02F8A1, 0x02F8A1, 0x00393A},+{0x02F8A2, 0x02F8A2, 0x00391C},+{0x02F8A3, 0x02F8A3, 0x006094},+{0x02F8A4, 0x02F8A4, 0x0226D4},+{0x02F8A5, 0x02F8A5, 0x0060C7},+{0x02F8A6, 0x02F8A6, 0x006148},+{0x02F8A7, 0x02F8A7, 0x00614C},+{0x02F8A8, 0x02F8A8, 0x00614E},+{0x02F8A9, 0x02F8A9, 0x00614C},+{0x02F8AA, 0x02F8AA, 0x00617A},+{0x02F8AB, 0x02F8AB, 0x00618E},+{0x02F8AC, 0x02F8AC, 0x0061B2},+{0x02F8AD, 0x02F8AD, 0x0061A4},+{0x02F8AE, 0x02F8AE, 0x0061AF},+{0x02F8AF, 0x02F8AF, 0x0061DE},+{0x02F8B0, 0x02F8B0, 0x0061F2},+{0x02F8B1, 0x02F8B1, 0x0061F6},+{0x02F8B2, 0x02F8B2, 0x006210},+{0x02F8B3, 0x02F8B3, 0x00621B},+{0x02F8B4, 0x02F8B4, 0x00625D},+{0x02F8B5, 0x02F8B5, 0x0062B1},+{0x02F8B6, 0x02F8B6, 0x0062D4},+{0x02F8B7, 0x02F8B7, 0x006350},+{0x02F8B8, 0x02F8B8, 0x022B0C},+{0x02F8B9, 0x02F8B9, 0x00633D},+{0x02F8BA, 0x02F8BA, 0x0062FC},+{0x02F8BB, 0x02F8BB, 0x006368},+{0x02F8BC, 0x02F8BC, 0x006383},+{0x02F8BD, 0x02F8BD, 0x0063E4},+{0x02F8BE, 0x02F8BE, 0x022BF1},+{0x02F8BF, 0x02F8BF, 0x006422},+{0x02F8C0, 0x02F8C0, 0x0063C5},+{0x02F8C1, 0x02F8C1, 0x0063A9},+{0x02F8C2, 0x02F8C2, 0x003A2E},+{0x02F8C3, 0x02F8C3, 0x006469},+{0x02F8C4, 0x02F8C4, 0x00647E},+{0x02F8C5, 0x02F8C5, 0x00649D},+{0x02F8C6, 0x02F8C6, 0x006477},+{0x02F8C7, 0x02F8C7, 0x003A6C},+{0x02F8C8, 0x02F8C8, 0x00654F},+{0x02F8C9, 0x02F8C9, 0x00656C},+{0x02F8CA, 0x02F8CA, 0x02300A},+{0x02F8CB, 0x02F8CB, 0x0065E3},+{0x02F8CC, 0x02F8CC, 0x0066F8},+{0x02F8CD, 0x02F8CD, 0x006649},+{0x02F8CE, 0x02F8CE, 0x003B19},+{0x02F8CF, 0x02F8CF, 0x006691},+{0x02F8D0, 0x02F8D0, 0x003B08},+{0x02F8D1, 0x02F8D1, 0x003AE4},+{0x02F8D2, 0x02F8D2, 0x005192},+{0x02F8D3, 0x02F8D3, 0x005195},+{0x02F8D4, 0x02F8D4, 0x006700},+{0x02F8D5, 0x02F8D5, 0x00669C},+{0x02F8D6, 0x02F8D6, 0x0080AD},+{0x02F8D7, 0x02F8D7, 0x0043D9},+{0x02F8D8, 0x02F8D8, 0x006717},+{0x02F8D9, 0x02F8D9, 0x00671B},+{0x02F8DA, 0x02F8DA, 0x006721},+{0x02F8DB, 0x02F8DB, 0x00675E},+{0x02F8DC, 0x02F8DC, 0x006753},+{0x02F8DD, 0x02F8DD, 0x0233C3},+{0x02F8DE, 0x02F8DE, 0x003B49},+{0x02F8DF, 0x02F8DF, 0x0067FA},+{0x02F8E0, 0x02F8E0, 0x006785},+{0x02F8E1, 0x02F8E1, 0x006852},+{0x02F8E2, 0x02F8E2, 0x006885},+{0x02F8E3, 0x02F8E3, 0x02346D},+{0x02F8E4, 0x02F8E4, 0x00688E},+{0x02F8E5, 0x02F8E5, 0x00681F},+{0x02F8E6, 0x02F8E6, 0x006914},+{0x02F8E7, 0x02F8E7, 0x003B9D},+{0x02F8E8, 0x02F8E8, 0x006942},+{0x02F8E9, 0x02F8E9, 0x0069A3},+{0x02F8EA, 0x02F8EA, 0x0069EA},+{0x02F8EB, 0x02F8EB, 0x006AA8},+{0x02F8EC, 0x02F8EC, 0x0236A3},+{0x02F8ED, 0x02F8ED, 0x006ADB},+{0x02F8EE, 0x02F8EE, 0x003C18},+{0x02F8EF, 0x02F8EF, 0x006B21},+{0x02F8F0, 0x02F8F0, 0x0238A7},+{0x02F8F1, 0x02F8F1, 0x006B54},+{0x02F8F2, 0x02F8F2, 0x003C4E},+{0x02F8F3, 0x02F8F3, 0x006B72},+{0x02F8F4, 0x02F8F4, 0x006B9F},+{0x02F8F5, 0x02F8F5, 0x006BBA},+{0x02F8F6, 0x02F8F6, 0x006BBB},+{0x02F8F7, 0x02F8F7, 0x023A8D},+{0x02F8F8, 0x02F8F8, 0x021D0B},+{0x02F8F9, 0x02F8F9, 0x023AFA},+{0x02F8FA, 0x02F8FA, 0x006C4E},+{0x02F8FB, 0x02F8FB, 0x023CBC},+{0x02F8FC, 0x02F8FC, 0x006CBF},+{0x02F8FD, 0x02F8FD, 0x006CCD},+{0x02F8FE, 0x02F8FE, 0x006C67},+{0x02F8FF, 0x02F8FF, 0x006D16},+{0x02F900, 0x02F900, 0x006D3E},+{0x02F901, 0x02F901, 0x006D77},+{0x02F902, 0x02F902, 0x006D41},+{0x02F903, 0x02F903, 0x006D69},+{0x02F904, 0x02F904, 0x006D78},+{0x02F905, 0x02F905, 0x006D85},+{0x02F906, 0x02F906, 0x023D1E},+{0x02F907, 0x02F907, 0x006D34},+{0x02F908, 0x02F908, 0x006E2F},+{0x02F909, 0x02F909, 0x006E6E},+{0x02F90A, 0x02F90A, 0x003D33},+{0x02F90B, 0x02F90B, 0x006ECB},+{0x02F90C, 0x02F90C, 0x006EC7},+{0x02F90D, 0x02F90D, 0x023ED1},+{0x02F90E, 0x02F90E, 0x006DF9},+{0x02F90F, 0x02F90F, 0x006F6E},+{0x02F910, 0x02F910, 0x023F5E},+{0x02F911, 0x02F911, 0x023F8E},+{0x02F912, 0x02F912, 0x006FC6},+{0x02F913, 0x02F913, 0x007039},+{0x02F914, 0x02F914, 0x00701E},+{0x02F915, 0x02F915, 0x00701B},+{0x02F916, 0x02F916, 0x003D96},+{0x02F917, 0x02F917, 0x00704A},+{0x02F918, 0x02F918, 0x00707D},+{0x02F919, 0x02F919, 0x007077},+{0x02F91A, 0x02F91A, 0x0070AD},+{0x02F91B, 0x02F91B, 0x020525},+{0x02F91C, 0x02F91C, 0x007145},+{0x02F91D, 0x02F91D, 0x024263},+{0x02F91E, 0x02F91E, 0x00719C},+{0x02F91F, 0x02F91F, 0x0243AB},+{0x02F920, 0x02F920, 0x007228},+{0x02F921, 0x02F921, 0x007235},+{0x02F922, 0x02F922, 0x007250},+{0x02F923, 0x02F923, 0x024608},+{0x02F924, 0x02F924, 0x007280},+{0x02F925, 0x02F925, 0x007295},+{0x02F926, 0x02F926, 0x024735},+{0x02F927, 0x02F927, 0x024814},+{0x02F928, 0x02F928, 0x00737A},+{0x02F929, 0x02F929, 0x00738B},+{0x02F92A, 0x02F92A, 0x003EAC},+{0x02F92B, 0x02F92B, 0x0073A5},+{0x02F92C, 0x02F92D, 0x003EB8},+{0x02F92E, 0x02F92E, 0x007447},+{0x02F92F, 0x02F92F, 0x00745C},+{0x02F930, 0x02F930, 0x007471},+{0x02F931, 0x02F931, 0x007485},+{0x02F932, 0x02F932, 0x0074CA},+{0x02F933, 0x02F933, 0x003F1B},+{0x02F934, 0x02F934, 0x007524},+{0x02F935, 0x02F935, 0x024C36},+{0x02F936, 0x02F936, 0x00753E},+{0x02F937, 0x02F937, 0x024C92},+{0x02F938, 0x02F938, 0x007570},+{0x02F939, 0x02F939, 0x02219F},+{0x02F93A, 0x02F93A, 0x007610},+{0x02F93B, 0x02F93B, 0x024FA1},+{0x02F93C, 0x02F93C, 0x024FB8},+{0x02F93D, 0x02F93D, 0x025044},+{0x02F93E, 0x02F93E, 0x003FFC},+{0x02F93F, 0x02F93F, 0x004008},+{0x02F940, 0x02F940, 0x0076F4},+{0x02F941, 0x02F941, 0x0250F3},+{0x02F942, 0x02F942, 0x0250F2},+{0x02F943, 0x02F943, 0x025119},+{0x02F944, 0x02F944, 0x025133},+{0x02F945, 0x02F945, 0x00771E},+{0x02F946, 0x02F947, 0x00771F},+{0x02F948, 0x02F948, 0x00774A},+{0x02F949, 0x02F949, 0x004039},+{0x02F94A, 0x02F94A, 0x00778B},+{0x02F94B, 0x02F94B, 0x004046},+{0x02F94C, 0x02F94C, 0x004096},+{0x02F94D, 0x02F94D, 0x02541D},+{0x02F94E, 0x02F94E, 0x00784E},+{0x02F94F, 0x02F94F, 0x00788C},+{0x02F950, 0x02F950, 0x0078CC},+{0x02F951, 0x02F951, 0x0040E3},+{0x02F952, 0x02F952, 0x025626},+{0x02F953, 0x02F953, 0x007956},+{0x02F954, 0x02F954, 0x02569A},+{0x02F955, 0x02F955, 0x0256C5},+{0x02F956, 0x02F956, 0x00798F},+{0x02F957, 0x02F957, 0x0079EB},+{0x02F958, 0x02F958, 0x00412F},+{0x02F959, 0x02F959, 0x007A40},+{0x02F95A, 0x02F95A, 0x007A4A},+{0x02F95B, 0x02F95B, 0x007A4F},+{0x02F95C, 0x02F95C, 0x02597C},+{0x02F95D, 0x02F95E, 0x025AA7},+{0x02F95F, 0x02F95F, 0x007AEE},+{0x02F960, 0x02F960, 0x004202},+{0x02F961, 0x02F961, 0x025BAB},+{0x02F962, 0x02F962, 0x007BC6},+{0x02F963, 0x02F963, 0x007BC9},+{0x02F964, 0x02F964, 0x004227},+{0x02F965, 0x02F965, 0x025C80},+{0x02F966, 0x02F966, 0x007CD2},+{0x02F967, 0x02F967, 0x0042A0},+{0x02F968, 0x02F968, 0x007CE8},+{0x02F969, 0x02F969, 0x007CE3},+{0x02F96A, 0x02F96A, 0x007D00},+{0x02F96B, 0x02F96B, 0x025F86},+{0x02F96C, 0x02F96C, 0x007D63},+{0x02F96D, 0x02F96D, 0x004301},+{0x02F96E, 0x02F96E, 0x007DC7},+{0x02F96F, 0x02F96F, 0x007E02},+{0x02F970, 0x02F970, 0x007E45},+{0x02F971, 0x02F971, 0x004334},+{0x02F972, 0x02F972, 0x026228},+{0x02F973, 0x02F973, 0x026247},+{0x02F974, 0x02F974, 0x004359},+{0x02F975, 0x02F975, 0x0262D9},+{0x02F976, 0x02F976, 0x007F7A},+{0x02F977, 0x02F977, 0x02633E},+{0x02F978, 0x02F978, 0x007F95},+{0x02F979, 0x02F979, 0x007FFA},+{0x02F97A, 0x02F97A, 0x008005},+{0x02F97B, 0x02F97B, 0x0264DA},+{0x02F97C, 0x02F97C, 0x026523},+{0x02F97D, 0x02F97D, 0x008060},+{0x02F97E, 0x02F97E, 0x0265A8},+{0x02F97F, 0x02F97F, 0x008070},+{0x02F980, 0x02F980, 0x02335F},+{0x02F981, 0x02F981, 0x0043D5},+{0x02F982, 0x02F982, 0x0080B2},+{0x02F983, 0x02F983, 0x008103},+{0x02F984, 0x02F984, 0x00440B},+{0x02F985, 0x02F985, 0x00813E},+{0x02F986, 0x02F986, 0x005AB5},+{0x02F987, 0x02F987, 0x0267A7},+{0x02F988, 0x02F988, 0x0267B5},+{0x02F989, 0x02F989, 0x023393},+{0x02F98A, 0x02F98A, 0x02339C},+{0x02F98B, 0x02F98B, 0x008201},+{0x02F98C, 0x02F98C, 0x008204},+{0x02F98D, 0x02F98D, 0x008F9E},+{0x02F98E, 0x02F98E, 0x00446B},+{0x02F98F, 0x02F98F, 0x008291},+{0x02F990, 0x02F990, 0x00828B},+{0x02F991, 0x02F991, 0x00829D},+{0x02F992, 0x02F992, 0x0052B3},+{0x02F993, 0x02F993, 0x0082B1},+{0x02F994, 0x02F994, 0x0082B3},+{0x02F995, 0x02F995, 0x0082BD},+{0x02F996, 0x02F996, 0x0082E6},+{0x02F997, 0x02F997, 0x026B3C},+{0x02F998, 0x02F998, 0x0082E5},+{0x02F999, 0x02F999, 0x00831D},+{0x02F99A, 0x02F99A, 0x008363},+{0x02F99B, 0x02F99B, 0x0083AD},+{0x02F99C, 0x02F99C, 0x008323},+{0x02F99D, 0x02F99D, 0x0083BD},+{0x02F99E, 0x02F99E, 0x0083E7},+{0x02F99F, 0x02F99F, 0x008457},+{0x02F9A0, 0x02F9A0, 0x008353},+{0x02F9A1, 0x02F9A1, 0x0083CA},+{0x02F9A2, 0x02F9A2, 0x0083CC},+{0x02F9A3, 0x02F9A3, 0x0083DC},+{0x02F9A4, 0x02F9A4, 0x026C36},+{0x02F9A5, 0x02F9A5, 0x026D6B},+{0x02F9A6, 0x02F9A6, 0x026CD5},+{0x02F9A7, 0x02F9A7, 0x00452B},+{0x02F9A8, 0x02F9A8, 0x0084F1},+{0x02F9A9, 0x02F9A9, 0x0084F3},+{0x02F9AA, 0x02F9AA, 0x008516},+{0x02F9AB, 0x02F9AB, 0x0273CA},+{0x02F9AC, 0x02F9AC, 0x008564},+{0x02F9AD, 0x02F9AD, 0x026F2C},+{0x02F9AE, 0x02F9AE, 0x00455D},+{0x02F9AF, 0x02F9AF, 0x004561},+{0x02F9B0, 0x02F9B0, 0x026FB1},+{0x02F9B1, 0x02F9B1, 0x0270D2},+{0x02F9B2, 0x02F9B2, 0x00456B},+{0x02F9B3, 0x02F9B3, 0x008650},+{0x02F9B4, 0x02F9B4, 0x00865C},+{0x02F9B5, 0x02F9B5, 0x008667},+{0x02F9B6, 0x02F9B6, 0x008669},+{0x02F9B7, 0x02F9B7, 0x0086A9},+{0x02F9B8, 0x02F9B8, 0x008688},+{0x02F9B9, 0x02F9B9, 0x00870E},+{0x02F9BA, 0x02F9BA, 0x0086E2},+{0x02F9BB, 0x02F9BB, 0x008779},+{0x02F9BC, 0x02F9BC, 0x008728},+{0x02F9BD, 0x02F9BD, 0x00876B},+{0x02F9BE, 0x02F9BE, 0x008786},+{0x02F9BF, 0x02F9BF, 0x0045D7},+{0x02F9C0, 0x02F9C0, 0x0087E1},+{0x02F9C1, 0x02F9C1, 0x008801},+{0x02F9C2, 0x02F9C2, 0x0045F9},+{0x02F9C3, 0x02F9C3, 0x008860},+{0x02F9C4, 0x02F9C4, 0x008863},+{0x02F9C5, 0x02F9C5, 0x027667},+{0x02F9C6, 0x02F9C6, 0x0088D7},+{0x02F9C7, 0x02F9C7, 0x0088DE},+{0x02F9C8, 0x02F9C8, 0x004635},+{0x02F9C9, 0x02F9C9, 0x0088FA},+{0x02F9CA, 0x02F9CA, 0x0034BB},+{0x02F9CB, 0x02F9CB, 0x0278AE},+{0x02F9CC, 0x02F9CC, 0x027966},+{0x02F9CD, 0x02F9CD, 0x0046BE},+{0x02F9CE, 0x02F9CE, 0x0046C7},+{0x02F9CF, 0x02F9CF, 0x008AA0},+{0x02F9D0, 0x02F9D0, 0x008AED},+{0x02F9D1, 0x02F9D1, 0x008B8A},+{0x02F9D2, 0x02F9D2, 0x008C55},+{0x02F9D3, 0x02F9D3, 0x027CA8},+{0x02F9D4, 0x02F9D4, 0x008CAB},+{0x02F9D5, 0x02F9D5, 0x008CC1},+{0x02F9D6, 0x02F9D6, 0x008D1B},+{0x02F9D7, 0x02F9D7, 0x008D77},+{0x02F9D8, 0x02F9D8, 0x027F2F},+{0x02F9D9, 0x02F9D9, 0x020804},+{0x02F9DA, 0x02F9DA, 0x008DCB},+{0x02F9DB, 0x02F9DB, 0x008DBC},+{0x02F9DC, 0x02F9DC, 0x008DF0},+{0x02F9DD, 0x02F9DD, 0x0208DE},+{0x02F9DE, 0x02F9DE, 0x008ED4},+{0x02F9DF, 0x02F9DF, 0x008F38},+{0x02F9E0, 0x02F9E0, 0x0285D2},+{0x02F9E1, 0x02F9E1, 0x0285ED},+{0x02F9E2, 0x02F9E2, 0x009094},+{0x02F9E3, 0x02F9E3, 0x0090F1},+{0x02F9E4, 0x02F9E4, 0x009111},+{0x02F9E5, 0x02F9E5, 0x02872E},+{0x02F9E6, 0x02F9E6, 0x00911B},+{0x02F9E7, 0x02F9E7, 0x009238},+{0x02F9E8, 0x02F9E8, 0x0092D7},+{0x02F9E9, 0x02F9E9, 0x0092D8},+{0x02F9EA, 0x02F9EA, 0x00927C},+{0x02F9EB, 0x02F9EB, 0x0093F9},+{0x02F9EC, 0x02F9EC, 0x009415},+{0x02F9ED, 0x02F9ED, 0x028BFA},+{0x02F9EE, 0x02F9EE, 0x00958B},+{0x02F9EF, 0x02F9EF, 0x004995},+{0x02F9F0, 0x02F9F0, 0x0095B7},+{0x02F9F1, 0x02F9F1, 0x028D77},+{0x02F9F2, 0x02F9F2, 0x0049E6},+{0x02F9F3, 0x02F9F3, 0x0096C3},+{0x02F9F4, 0x02F9F4, 0x005DB2},+{0x02F9F5, 0x02F9F5, 0x009723},+{0x02F9F6, 0x02F9F6, 0x029145},+{0x02F9F7, 0x02F9F7, 0x02921A},+{0x02F9F8, 0x02F9F8, 0x004A6E},+{0x02F9F9, 0x02F9F9, 0x004A76},+{0x02F9FA, 0x02F9FA, 0x0097E0},+{0x02F9FB, 0x02F9FB, 0x02940A},+{0x02F9FC, 0x02F9FC, 0x004AB2},+{0x02F9FD, 0x02F9FD, 0x029496},+{0x02F9FE, 0x02F9FF, 0x00980B},+{0x02FA00, 0x02FA00, 0x009829},+{0x02FA01, 0x02FA01, 0x0295B6},+{0x02FA02, 0x02FA02, 0x0098E2},+{0x02FA03, 0x02FA03, 0x004B33},+{0x02FA04, 0x02FA04, 0x009929},+{0x02FA05, 0x02FA05, 0x0099A7},+{0x02FA06, 0x02FA06, 0x0099C2},+{0x02FA07, 0x02FA07, 0x0099FE},+{0x02FA08, 0x02FA08, 0x004BCE},+{0x02FA09, 0x02FA09, 0x029B30},+{0x02FA0A, 0x02FA0A, 0x009B12},+{0x02FA0B, 0x02FA0B, 0x009C40},+{0x02FA0C, 0x02FA0C, 0x009CFD},+{0x02FA0D, 0x02FA0D, 0x004CCE},+{0x02FA0E, 0x02FA0E, 0x004CED},+{0x02FA0F, 0x02FA0F, 0x009D67},+{0x02FA10, 0x02FA10, 0x02A0CE},+{0x02FA11, 0x02FA11, 0x004CF8},+{0x02FA12, 0x02FA12, 0x02A105},+{0x02FA13, 0x02FA13, 0x02A20E},+{0x02FA14, 0x02FA14, 0x02A291},+{0x02FA15, 0x02FA15, 0x009EBB},+{0x02FA16, 0x02FA16, 0x004D56},+{0x02FA17, 0x02FA17, 0x009EF9},+{0x02FA18, 0x02FA18, 0x009EFE},+{0x02FA19, 0x02FA19, 0x009F05},+{0x02FA1A, 0x02FA1A, 0x009F0F},+{0x02FA1B, 0x02FA1B, 0x009F16},+{0x02FA1C, 0x02FA1C, 0x009F3B},+{0x02FA1D, 0x02FA1D, 0x02A600},};-const std::map<char32_t, char32_t> unicode_map_lowercase = {-{0x00041, 0x00061}, {0x00042, 0x00062}, {0x00043, 0x00063}, {0x00044, 0x00064}, {0x00045, 0x00065}, {0x00046, 0x00066},-{0x00047, 0x00067}, {0x00048, 0x00068}, {0x00049, 0x00069}, {0x0004A, 0x0006A}, {0x0004B, 0x0006B}, {0x0004C, 0x0006C},-{0x0004D, 0x0006D}, {0x0004E, 0x0006E}, {0x0004F, 0x0006F}, {0x00050, 0x00070}, {0x00051, 0x00071}, {0x00052, 0x00072},-{0x00053, 0x00073}, {0x00054, 0x00074}, {0x00055, 0x00075}, {0x00056, 0x00076}, {0x00057, 0x00077}, {0x00058, 0x00078},-{0x00059, 0x00079}, {0x0005A, 0x0007A}, {0x000C0, 0x000E0}, {0x000C1, 0x000E1}, {0x000C2, 0x000E2}, {0x000C3, 0x000E3},-{0x000C4, 0x000E4}, {0x000C5, 0x000E5}, {0x000C6, 0x000E6}, {0x000C7, 0x000E7}, {0x000C8, 0x000E8}, {0x000C9, 0x000E9},-{0x000CA, 0x000EA}, {0x000CB, 0x000EB}, {0x000CC, 0x000EC}, {0x000CD, 0x000ED}, {0x000CE, 0x000EE}, {0x000CF, 0x000EF},-{0x000D0, 0x000F0}, {0x000D1, 0x000F1}, {0x000D2, 0x000F2}, {0x000D3, 0x000F3}, {0x000D4, 0x000F4}, {0x000D5, 0x000F5},-{0x000D6, 0x000F6}, {0x000D8, 0x000F8}, {0x000D9, 0x000F9}, {0x000DA, 0x000FA}, {0x000DB, 0x000FB}, {0x000DC, 0x000FC},-{0x000DD, 0x000FD}, {0x000DE, 0x000FE}, {0x00100, 0x00101}, {0x00102, 0x00103}, {0x00104, 0x00105}, {0x00106, 0x00107},-{0x00108, 0x00109}, {0x0010A, 0x0010B}, {0x0010C, 0x0010D}, {0x0010E, 0x0010F}, {0x00110, 0x00111}, {0x00112, 0x00113},-{0x00114, 0x00115}, {0x00116, 0x00117}, {0x00118, 0x00119}, {0x0011A, 0x0011B}, {0x0011C, 0x0011D}, {0x0011E, 0x0011F},-{0x00120, 0x00121}, {0x00122, 0x00123}, {0x00124, 0x00125}, {0x00126, 0x00127}, {0x00128, 0x00129}, {0x0012A, 0x0012B},-{0x0012C, 0x0012D}, {0x0012E, 0x0012F}, {0x00130, 0x00069}, {0x00132, 0x00133}, {0x00134, 0x00135}, {0x00136, 0x00137},-{0x00139, 0x0013A}, {0x0013B, 0x0013C}, {0x0013D, 0x0013E}, {0x0013F, 0x00140}, {0x00141, 0x00142}, {0x00143, 0x00144},-{0x00145, 0x00146}, {0x00147, 0x00148}, {0x0014A, 0x0014B}, {0x0014C, 0x0014D}, {0x0014E, 0x0014F}, {0x00150, 0x00151},-{0x00152, 0x00153}, {0x00154, 0x00155}, {0x00156, 0x00157}, {0x00158, 0x00159}, {0x0015A, 0x0015B}, {0x0015C, 0x0015D},-{0x0015E, 0x0015F}, {0x00160, 0x00161}, {0x00162, 0x00163}, {0x00164, 0x00165}, {0x00166, 0x00167}, {0x00168, 0x00169},-{0x0016A, 0x0016B}, {0x0016C, 0x0016D}, {0x0016E, 0x0016F}, {0x00170, 0x00171}, {0x00172, 0x00173}, {0x00174, 0x00175},-{0x00176, 0x00177}, {0x00178, 0x000FF}, {0x00179, 0x0017A}, {0x0017B, 0x0017C}, {0x0017D, 0x0017E}, {0x00181, 0x00253},-{0x00182, 0x00183}, {0x00184, 0x00185}, {0x00186, 0x00254}, {0x00187, 0x00188}, {0x00189, 0x00256}, {0x0018A, 0x00257},-{0x0018B, 0x0018C}, {0x0018E, 0x001DD}, {0x0018F, 0x00259}, {0x00190, 0x0025B}, {0x00191, 0x00192}, {0x00193, 0x00260},-{0x00194, 0x00263}, {0x00196, 0x00269}, {0x00197, 0x00268}, {0x00198, 0x00199}, {0x0019C, 0x0026F}, {0x0019D, 0x00272},-{0x0019F, 0x00275}, {0x001A0, 0x001A1}, {0x001A2, 0x001A3}, {0x001A4, 0x001A5}, {0x001A6, 0x00280}, {0x001A7, 0x001A8},-{0x001A9, 0x00283}, {0x001AC, 0x001AD}, {0x001AE, 0x00288}, {0x001AF, 0x001B0}, {0x001B1, 0x0028A}, {0x001B2, 0x0028B},-{0x001B3, 0x001B4}, {0x001B5, 0x001B6}, {0x001B7, 0x00292}, {0x001B8, 0x001B9}, {0x001BC, 0x001BD}, {0x001C4, 0x001C6},-{0x001C5, 0x001C6}, {0x001C7, 0x001C9}, {0x001C8, 0x001C9}, {0x001CA, 0x001CC}, {0x001CB, 0x001CC}, {0x001CD, 0x001CE},-{0x001CF, 0x001D0}, {0x001D1, 0x001D2}, {0x001D3, 0x001D4}, {0x001D5, 0x001D6}, {0x001D7, 0x001D8}, {0x001D9, 0x001DA},-{0x001DB, 0x001DC}, {0x001DE, 0x001DF}, {0x001E0, 0x001E1}, {0x001E2, 0x001E3}, {0x001E4, 0x001E5}, {0x001E6, 0x001E7},-{0x001E8, 0x001E9}, {0x001EA, 0x001EB}, {0x001EC, 0x001ED}, {0x001EE, 0x001EF}, {0x001F1, 0x001F3}, {0x001F2, 0x001F3},-{0x001F4, 0x001F5}, {0x001F6, 0x00195}, {0x001F7, 0x001BF}, {0x001F8, 0x001F9}, {0x001FA, 0x001FB}, {0x001FC, 0x001FD},-{0x001FE, 0x001FF}, {0x00200, 0x00201}, {0x00202, 0x00203}, {0x00204, 0x00205}, {0x00206, 0x00207}, {0x00208, 0x00209},-{0x0020A, 0x0020B}, {0x0020C, 0x0020D}, {0x0020E, 0x0020F}, {0x00210, 0x00211}, {0x00212, 0x00213}, {0x00214, 0x00215},-{0x00216, 0x00217}, {0x00218, 0x00219}, {0x0021A, 0x0021B}, {0x0021C, 0x0021D}, {0x0021E, 0x0021F}, {0x00220, 0x0019E},-{0x00222, 0x00223}, {0x00224, 0x00225}, {0x00226, 0x00227}, {0x00228, 0x00229}, {0x0022A, 0x0022B}, {0x0022C, 0x0022D},-{0x0022E, 0x0022F}, {0x00230, 0x00231}, {0x00232, 0x00233}, {0x0023A, 0x02C65}, {0x0023B, 0x0023C}, {0x0023D, 0x0019A},-{0x0023E, 0x02C66}, {0x00241, 0x00242}, {0x00243, 0x00180}, {0x00244, 0x00289}, {0x00245, 0x0028C}, {0x00246, 0x00247},-{0x00248, 0x00249}, {0x0024A, 0x0024B}, {0x0024C, 0x0024D}, {0x0024E, 0x0024F}, {0x00370, 0x00371}, {0x00372, 0x00373},-{0x00376, 0x00377}, {0x0037F, 0x003F3}, {0x00386, 0x003AC}, {0x00388, 0x003AD}, {0x00389, 0x003AE}, {0x0038A, 0x003AF},-{0x0038C, 0x003CC}, {0x0038E, 0x003CD}, {0x0038F, 0x003CE}, {0x00391, 0x003B1}, {0x00392, 0x003B2}, {0x00393, 0x003B3},-{0x00394, 0x003B4}, {0x00395, 0x003B5}, {0x00396, 0x003B6}, {0x00397, 0x003B7}, {0x00398, 0x003B8}, {0x00399, 0x003B9},-{0x0039A, 0x003BA}, {0x0039B, 0x003BB}, {0x0039C, 0x003BC}, {0x0039D, 0x003BD}, {0x0039E, 0x003BE}, {0x0039F, 0x003BF},-{0x003A0, 0x003C0}, {0x003A1, 0x003C1}, {0x003A3, 0x003C3}, {0x003A4, 0x003C4}, {0x003A5, 0x003C5}, {0x003A6, 0x003C6},-{0x003A7, 0x003C7}, {0x003A8, 0x003C8}, {0x003A9, 0x003C9}, {0x003AA, 0x003CA}, {0x003AB, 0x003CB}, {0x003CF, 0x003D7},-{0x003D8, 0x003D9}, {0x003DA, 0x003DB}, {0x003DC, 0x003DD}, {0x003DE, 0x003DF}, {0x003E0, 0x003E1}, {0x003E2, 0x003E3},-{0x003E4, 0x003E5}, {0x003E6, 0x003E7}, {0x003E8, 0x003E9}, {0x003EA, 0x003EB}, {0x003EC, 0x003ED}, {0x003EE, 0x003EF},-{0x003F4, 0x003B8}, {0x003F7, 0x003F8}, {0x003F9, 0x003F2}, {0x003FA, 0x003FB}, {0x003FD, 0x0037B}, {0x003FE, 0x0037C},-{0x003FF, 0x0037D}, {0x00400, 0x00450}, {0x00401, 0x00451}, {0x00402, 0x00452}, {0x00403, 0x00453}, {0x00404, 0x00454},-{0x00405, 0x00455}, {0x00406, 0x00456}, {0x00407, 0x00457}, {0x00408, 0x00458}, {0x00409, 0x00459}, {0x0040A, 0x0045A},-{0x0040B, 0x0045B}, {0x0040C, 0x0045C}, {0x0040D, 0x0045D}, {0x0040E, 0x0045E}, {0x0040F, 0x0045F}, {0x00410, 0x00430},-{0x00411, 0x00431}, {0x00412, 0x00432}, {0x00413, 0x00433}, {0x00414, 0x00434}, {0x00415, 0x00435}, {0x00416, 0x00436},-{0x00417, 0x00437}, {0x00418, 0x00438}, {0x00419, 0x00439}, {0x0041A, 0x0043A}, {0x0041B, 0x0043B}, {0x0041C, 0x0043C},-{0x0041D, 0x0043D}, {0x0041E, 0x0043E}, {0x0041F, 0x0043F}, {0x00420, 0x00440}, {0x00421, 0x00441}, {0x00422, 0x00442},-{0x00423, 0x00443}, {0x00424, 0x00444}, {0x00425, 0x00445}, {0x00426, 0x00446}, {0x00427, 0x00447}, {0x00428, 0x00448},-{0x00429, 0x00449}, {0x0042A, 0x0044A}, {0x0042B, 0x0044B}, {0x0042C, 0x0044C}, {0x0042D, 0x0044D}, {0x0042E, 0x0044E},-{0x0042F, 0x0044F}, {0x00460, 0x00461}, {0x00462, 0x00463}, {0x00464, 0x00465}, {0x00466, 0x00467}, {0x00468, 0x00469},-{0x0046A, 0x0046B}, {0x0046C, 0x0046D}, {0x0046E, 0x0046F}, {0x00470, 0x00471}, {0x00472, 0x00473}, {0x00474, 0x00475},-{0x00476, 0x00477}, {0x00478, 0x00479}, {0x0047A, 0x0047B}, {0x0047C, 0x0047D}, {0x0047E, 0x0047F}, {0x00480, 0x00481},-{0x0048A, 0x0048B}, {0x0048C, 0x0048D}, {0x0048E, 0x0048F}, {0x00490, 0x00491}, {0x00492, 0x00493}, {0x00494, 0x00495},-{0x00496, 0x00497}, {0x00498, 0x00499}, {0x0049A, 0x0049B}, {0x0049C, 0x0049D}, {0x0049E, 0x0049F}, {0x004A0, 0x004A1},-{0x004A2, 0x004A3}, {0x004A4, 0x004A5}, {0x004A6, 0x004A7}, {0x004A8, 0x004A9}, {0x004AA, 0x004AB}, {0x004AC, 0x004AD},-{0x004AE, 0x004AF}, {0x004B0, 0x004B1}, {0x004B2, 0x004B3}, {0x004B4, 0x004B5}, {0x004B6, 0x004B7}, {0x004B8, 0x004B9},-{0x004BA, 0x004BB}, {0x004BC, 0x004BD}, {0x004BE, 0x004BF}, {0x004C0, 0x004CF}, {0x004C1, 0x004C2}, {0x004C3, 0x004C4},-{0x004C5, 0x004C6}, {0x004C7, 0x004C8}, {0x004C9, 0x004CA}, {0x004CB, 0x004CC}, {0x004CD, 0x004CE}, {0x004D0, 0x004D1},-{0x004D2, 0x004D3}, {0x004D4, 0x004D5}, {0x004D6, 0x004D7}, {0x004D8, 0x004D9}, {0x004DA, 0x004DB}, {0x004DC, 0x004DD},-{0x004DE, 0x004DF}, {0x004E0, 0x004E1}, {0x004E2, 0x004E3}, {0x004E4, 0x004E5}, {0x004E6, 0x004E7}, {0x004E8, 0x004E9},-{0x004EA, 0x004EB}, {0x004EC, 0x004ED}, {0x004EE, 0x004EF}, {0x004F0, 0x004F1}, {0x004F2, 0x004F3}, {0x004F4, 0x004F5},-{0x004F6, 0x004F7}, {0x004F8, 0x004F9}, {0x004FA, 0x004FB}, {0x004FC, 0x004FD}, {0x004FE, 0x004FF}, {0x00500, 0x00501},-{0x00502, 0x00503}, {0x00504, 0x00505}, {0x00506, 0x00507}, {0x00508, 0x00509}, {0x0050A, 0x0050B}, {0x0050C, 0x0050D},-{0x0050E, 0x0050F}, {0x00510, 0x00511}, {0x00512, 0x00513}, {0x00514, 0x00515}, {0x00516, 0x00517}, {0x00518, 0x00519},-{0x0051A, 0x0051B}, {0x0051C, 0x0051D}, {0x0051E, 0x0051F}, {0x00520, 0x00521}, {0x00522, 0x00523}, {0x00524, 0x00525},-{0x00526, 0x00527}, {0x00528, 0x00529}, {0x0052A, 0x0052B}, {0x0052C, 0x0052D}, {0x0052E, 0x0052F}, {0x00531, 0x00561},-{0x00532, 0x00562}, {0x00533, 0x00563}, {0x00534, 0x00564}, {0x00535, 0x00565}, {0x00536, 0x00566}, {0x00537, 0x00567},-{0x00538, 0x00568}, {0x00539, 0x00569}, {0x0053A, 0x0056A}, {0x0053B, 0x0056B}, {0x0053C, 0x0056C}, {0x0053D, 0x0056D},-{0x0053E, 0x0056E}, {0x0053F, 0x0056F}, {0x00540, 0x00570}, {0x00541, 0x00571}, {0x00542, 0x00572}, {0x00543, 0x00573},-{0x00544, 0x00574}, {0x00545, 0x00575}, {0x00546, 0x00576}, {0x00547, 0x00577}, {0x00548, 0x00578}, {0x00549, 0x00579},-{0x0054A, 0x0057A}, {0x0054B, 0x0057B}, {0x0054C, 0x0057C}, {0x0054D, 0x0057D}, {0x0054E, 0x0057E}, {0x0054F, 0x0057F},-{0x00550, 0x00580}, {0x00551, 0x00581}, {0x00552, 0x00582}, {0x00553, 0x00583}, {0x00554, 0x00584}, {0x00555, 0x00585},-{0x00556, 0x00586}, {0x010A0, 0x02D00}, {0x010A1, 0x02D01}, {0x010A2, 0x02D02}, {0x010A3, 0x02D03}, {0x010A4, 0x02D04},-{0x010A5, 0x02D05}, {0x010A6, 0x02D06}, {0x010A7, 0x02D07}, {0x010A8, 0x02D08}, {0x010A9, 0x02D09}, {0x010AA, 0x02D0A},-{0x010AB, 0x02D0B}, {0x010AC, 0x02D0C}, {0x010AD, 0x02D0D}, {0x010AE, 0x02D0E}, {0x010AF, 0x02D0F}, {0x010B0, 0x02D10},-{0x010B1, 0x02D11}, {0x010B2, 0x02D12}, {0x010B3, 0x02D13}, {0x010B4, 0x02D14}, {0x010B5, 0x02D15}, {0x010B6, 0x02D16},-{0x010B7, 0x02D17}, {0x010B8, 0x02D18}, {0x010B9, 0x02D19}, {0x010BA, 0x02D1A}, {0x010BB, 0x02D1B}, {0x010BC, 0x02D1C},-{0x010BD, 0x02D1D}, {0x010BE, 0x02D1E}, {0x010BF, 0x02D1F}, {0x010C0, 0x02D20}, {0x010C1, 0x02D21}, {0x010C2, 0x02D22},-{0x010C3, 0x02D23}, {0x010C4, 0x02D24}, {0x010C5, 0x02D25}, {0x010C7, 0x02D27}, {0x010CD, 0x02D2D}, {0x013A0, 0x0AB70},-{0x013A1, 0x0AB71}, {0x013A2, 0x0AB72}, {0x013A3, 0x0AB73}, {0x013A4, 0x0AB74}, {0x013A5, 0x0AB75}, {0x013A6, 0x0AB76},-{0x013A7, 0x0AB77}, {0x013A8, 0x0AB78}, {0x013A9, 0x0AB79}, {0x013AA, 0x0AB7A}, {0x013AB, 0x0AB7B}, {0x013AC, 0x0AB7C},-{0x013AD, 0x0AB7D}, {0x013AE, 0x0AB7E}, {0x013AF, 0x0AB7F}, {0x013B0, 0x0AB80}, {0x013B1, 0x0AB81}, {0x013B2, 0x0AB82},-{0x013B3, 0x0AB83}, {0x013B4, 0x0AB84}, {0x013B5, 0x0AB85}, {0x013B6, 0x0AB86}, {0x013B7, 0x0AB87}, {0x013B8, 0x0AB88},-{0x013B9, 0x0AB89}, {0x013BA, 0x0AB8A}, {0x013BB, 0x0AB8B}, {0x013BC, 0x0AB8C}, {0x013BD, 0x0AB8D}, {0x013BE, 0x0AB8E},-{0x013BF, 0x0AB8F}, {0x013C0, 0x0AB90}, {0x013C1, 0x0AB91}, {0x013C2, 0x0AB92}, {0x013C3, 0x0AB93}, {0x013C4, 0x0AB94},-{0x013C5, 0x0AB95}, {0x013C6, 0x0AB96}, {0x013C7, 0x0AB97}, {0x013C8, 0x0AB98}, {0x013C9, 0x0AB99}, {0x013CA, 0x0AB9A},-{0x013CB, 0x0AB9B}, {0x013CC, 0x0AB9C}, {0x013CD, 0x0AB9D}, {0x013CE, 0x0AB9E}, {0x013CF, 0x0AB9F}, {0x013D0, 0x0ABA0},-{0x013D1, 0x0ABA1}, {0x013D2, 0x0ABA2}, {0x013D3, 0x0ABA3}, {0x013D4, 0x0ABA4}, {0x013D5, 0x0ABA5}, {0x013D6, 0x0ABA6},-{0x013D7, 0x0ABA7}, {0x013D8, 0x0ABA8}, {0x013D9, 0x0ABA9}, {0x013DA, 0x0ABAA}, {0x013DB, 0x0ABAB}, {0x013DC, 0x0ABAC},-{0x013DD, 0x0ABAD}, {0x013DE, 0x0ABAE}, {0x013DF, 0x0ABAF}, {0x013E0, 0x0ABB0}, {0x013E1, 0x0ABB1}, {0x013E2, 0x0ABB2},-{0x013E3, 0x0ABB3}, {0x013E4, 0x0ABB4}, {0x013E5, 0x0ABB5}, {0x013E6, 0x0ABB6}, {0x013E7, 0x0ABB7}, {0x013E8, 0x0ABB8},-{0x013E9, 0x0ABB9}, {0x013EA, 0x0ABBA}, {0x013EB, 0x0ABBB}, {0x013EC, 0x0ABBC}, {0x013ED, 0x0ABBD}, {0x013EE, 0x0ABBE},-{0x013EF, 0x0ABBF}, {0x013F0, 0x013F8}, {0x013F1, 0x013F9}, {0x013F2, 0x013FA}, {0x013F3, 0x013FB}, {0x013F4, 0x013FC},-{0x013F5, 0x013FD}, {0x01C90, 0x010D0}, {0x01C91, 0x010D1}, {0x01C92, 0x010D2}, {0x01C93, 0x010D3}, {0x01C94, 0x010D4},-{0x01C95, 0x010D5}, {0x01C96, 0x010D6}, {0x01C97, 0x010D7}, {0x01C98, 0x010D8}, {0x01C99, 0x010D9}, {0x01C9A, 0x010DA},-{0x01C9B, 0x010DB}, {0x01C9C, 0x010DC}, {0x01C9D, 0x010DD}, {0x01C9E, 0x010DE}, {0x01C9F, 0x010DF}, {0x01CA0, 0x010E0},-{0x01CA1, 0x010E1}, {0x01CA2, 0x010E2}, {0x01CA3, 0x010E3}, {0x01CA4, 0x010E4}, {0x01CA5, 0x010E5}, {0x01CA6, 0x010E6},-{0x01CA7, 0x010E7}, {0x01CA8, 0x010E8}, {0x01CA9, 0x010E9}, {0x01CAA, 0x010EA}, {0x01CAB, 0x010EB}, {0x01CAC, 0x010EC},-{0x01CAD, 0x010ED}, {0x01CAE, 0x010EE}, {0x01CAF, 0x010EF}, {0x01CB0, 0x010F0}, {0x01CB1, 0x010F1}, {0x01CB2, 0x010F2},-{0x01CB3, 0x010F3}, {0x01CB4, 0x010F4}, {0x01CB5, 0x010F5}, {0x01CB6, 0x010F6}, {0x01CB7, 0x010F7}, {0x01CB8, 0x010F8},-{0x01CB9, 0x010F9}, {0x01CBA, 0x010FA}, {0x01CBD, 0x010FD}, {0x01CBE, 0x010FE}, {0x01CBF, 0x010FF}, {0x01E00, 0x01E01},-{0x01E02, 0x01E03}, {0x01E04, 0x01E05}, {0x01E06, 0x01E07}, {0x01E08, 0x01E09}, {0x01E0A, 0x01E0B}, {0x01E0C, 0x01E0D},-{0x01E0E, 0x01E0F}, {0x01E10, 0x01E11}, {0x01E12, 0x01E13}, {0x01E14, 0x01E15}, {0x01E16, 0x01E17}, {0x01E18, 0x01E19},-{0x01E1A, 0x01E1B}, {0x01E1C, 0x01E1D}, {0x01E1E, 0x01E1F}, {0x01E20, 0x01E21}, {0x01E22, 0x01E23}, {0x01E24, 0x01E25},-{0x01E26, 0x01E27}, {0x01E28, 0x01E29}, {0x01E2A, 0x01E2B}, {0x01E2C, 0x01E2D}, {0x01E2E, 0x01E2F}, {0x01E30, 0x01E31},-{0x01E32, 0x01E33}, {0x01E34, 0x01E35}, {0x01E36, 0x01E37}, {0x01E38, 0x01E39}, {0x01E3A, 0x01E3B}, {0x01E3C, 0x01E3D},-{0x01E3E, 0x01E3F}, {0x01E40, 0x01E41}, {0x01E42, 0x01E43}, {0x01E44, 0x01E45}, {0x01E46, 0x01E47}, {0x01E48, 0x01E49},-{0x01E4A, 0x01E4B}, {0x01E4C, 0x01E4D}, {0x01E4E, 0x01E4F}, {0x01E50, 0x01E51}, {0x01E52, 0x01E53}, {0x01E54, 0x01E55},-{0x01E56, 0x01E57}, {0x01E58, 0x01E59}, {0x01E5A, 0x01E5B}, {0x01E5C, 0x01E5D}, {0x01E5E, 0x01E5F}, {0x01E60, 0x01E61},-{0x01E62, 0x01E63}, {0x01E64, 0x01E65}, {0x01E66, 0x01E67}, {0x01E68, 0x01E69}, {0x01E6A, 0x01E6B}, {0x01E6C, 0x01E6D},-{0x01E6E, 0x01E6F}, {0x01E70, 0x01E71}, {0x01E72, 0x01E73}, {0x01E74, 0x01E75}, {0x01E76, 0x01E77}, {0x01E78, 0x01E79},-{0x01E7A, 0x01E7B}, {0x01E7C, 0x01E7D}, {0x01E7E, 0x01E7F}, {0x01E80, 0x01E81}, {0x01E82, 0x01E83}, {0x01E84, 0x01E85},-{0x01E86, 0x01E87}, {0x01E88, 0x01E89}, {0x01E8A, 0x01E8B}, {0x01E8C, 0x01E8D}, {0x01E8E, 0x01E8F}, {0x01E90, 0x01E91},-{0x01E92, 0x01E93}, {0x01E94, 0x01E95}, {0x01E9E, 0x000DF}, {0x01EA0, 0x01EA1}, {0x01EA2, 0x01EA3}, {0x01EA4, 0x01EA5},-{0x01EA6, 0x01EA7}, {0x01EA8, 0x01EA9}, {0x01EAA, 0x01EAB}, {0x01EAC, 0x01EAD}, {0x01EAE, 0x01EAF}, {0x01EB0, 0x01EB1},-{0x01EB2, 0x01EB3}, {0x01EB4, 0x01EB5}, {0x01EB6, 0x01EB7}, {0x01EB8, 0x01EB9}, {0x01EBA, 0x01EBB}, {0x01EBC, 0x01EBD},-{0x01EBE, 0x01EBF}, {0x01EC0, 0x01EC1}, {0x01EC2, 0x01EC3}, {0x01EC4, 0x01EC5}, {0x01EC6, 0x01EC7}, {0x01EC8, 0x01EC9},-{0x01ECA, 0x01ECB}, {0x01ECC, 0x01ECD}, {0x01ECE, 0x01ECF}, {0x01ED0, 0x01ED1}, {0x01ED2, 0x01ED3}, {0x01ED4, 0x01ED5},-{0x01ED6, 0x01ED7}, {0x01ED8, 0x01ED9}, {0x01EDA, 0x01EDB}, {0x01EDC, 0x01EDD}, {0x01EDE, 0x01EDF}, {0x01EE0, 0x01EE1},-{0x01EE2, 0x01EE3}, {0x01EE4, 0x01EE5}, {0x01EE6, 0x01EE7}, {0x01EE8, 0x01EE9}, {0x01EEA, 0x01EEB}, {0x01EEC, 0x01EED},-{0x01EEE, 0x01EEF}, {0x01EF0, 0x01EF1}, {0x01EF2, 0x01EF3}, {0x01EF4, 0x01EF5}, {0x01EF6, 0x01EF7}, {0x01EF8, 0x01EF9},-{0x01EFA, 0x01EFB}, {0x01EFC, 0x01EFD}, {0x01EFE, 0x01EFF}, {0x01F08, 0x01F00}, {0x01F09, 0x01F01}, {0x01F0A, 0x01F02},-{0x01F0B, 0x01F03}, {0x01F0C, 0x01F04}, {0x01F0D, 0x01F05}, {0x01F0E, 0x01F06}, {0x01F0F, 0x01F07}, {0x01F18, 0x01F10},-{0x01F19, 0x01F11}, {0x01F1A, 0x01F12}, {0x01F1B, 0x01F13}, {0x01F1C, 0x01F14}, {0x01F1D, 0x01F15}, {0x01F28, 0x01F20},-{0x01F29, 0x01F21}, {0x01F2A, 0x01F22}, {0x01F2B, 0x01F23}, {0x01F2C, 0x01F24}, {0x01F2D, 0x01F25}, {0x01F2E, 0x01F26},-{0x01F2F, 0x01F27}, {0x01F38, 0x01F30}, {0x01F39, 0x01F31}, {0x01F3A, 0x01F32}, {0x01F3B, 0x01F33}, {0x01F3C, 0x01F34},-{0x01F3D, 0x01F35}, {0x01F3E, 0x01F36}, {0x01F3F, 0x01F37}, {0x01F48, 0x01F40}, {0x01F49, 0x01F41}, {0x01F4A, 0x01F42},-{0x01F4B, 0x01F43}, {0x01F4C, 0x01F44}, {0x01F4D, 0x01F45}, {0x01F59, 0x01F51}, {0x01F5B, 0x01F53}, {0x01F5D, 0x01F55},-{0x01F5F, 0x01F57}, {0x01F68, 0x01F60}, {0x01F69, 0x01F61}, {0x01F6A, 0x01F62}, {0x01F6B, 0x01F63}, {0x01F6C, 0x01F64},-{0x01F6D, 0x01F65}, {0x01F6E, 0x01F66}, {0x01F6F, 0x01F67}, {0x01F88, 0x01F80}, {0x01F89, 0x01F81}, {0x01F8A, 0x01F82},-{0x01F8B, 0x01F83}, {0x01F8C, 0x01F84}, {0x01F8D, 0x01F85}, {0x01F8E, 0x01F86}, {0x01F8F, 0x01F87}, {0x01F98, 0x01F90},-{0x01F99, 0x01F91}, {0x01F9A, 0x01F92}, {0x01F9B, 0x01F93}, {0x01F9C, 0x01F94}, {0x01F9D, 0x01F95}, {0x01F9E, 0x01F96},-{0x01F9F, 0x01F97}, {0x01FA8, 0x01FA0}, {0x01FA9, 0x01FA1}, {0x01FAA, 0x01FA2}, {0x01FAB, 0x01FA3}, {0x01FAC, 0x01FA4},-{0x01FAD, 0x01FA5}, {0x01FAE, 0x01FA6}, {0x01FAF, 0x01FA7}, {0x01FB8, 0x01FB0}, {0x01FB9, 0x01FB1}, {0x01FBA, 0x01F70},-{0x01FBB, 0x01F71}, {0x01FBC, 0x01FB3}, {0x01FC8, 0x01F72}, {0x01FC9, 0x01F73}, {0x01FCA, 0x01F74}, {0x01FCB, 0x01F75},-{0x01FCC, 0x01FC3}, {0x01FD8, 0x01FD0}, {0x01FD9, 0x01FD1}, {0x01FDA, 0x01F76}, {0x01FDB, 0x01F77}, {0x01FE8, 0x01FE0},-{0x01FE9, 0x01FE1}, {0x01FEA, 0x01F7A}, {0x01FEB, 0x01F7B}, {0x01FEC, 0x01FE5}, {0x01FF8, 0x01F78}, {0x01FF9, 0x01F79},-{0x01FFA, 0x01F7C}, {0x01FFB, 0x01F7D}, {0x01FFC, 0x01FF3}, {0x02126, 0x003C9}, {0x0212A, 0x0006B}, {0x0212B, 0x000E5},-{0x02132, 0x0214E}, {0x02160, 0x02170}, {0x02161, 0x02171}, {0x02162, 0x02172}, {0x02163, 0x02173}, {0x02164, 0x02174},-{0x02165, 0x02175}, {0x02166, 0x02176}, {0x02167, 0x02177}, {0x02168, 0x02178}, {0x02169, 0x02179}, {0x0216A, 0x0217A},-{0x0216B, 0x0217B}, {0x0216C, 0x0217C}, {0x0216D, 0x0217D}, {0x0216E, 0x0217E}, {0x0216F, 0x0217F}, {0x02183, 0x02184},-{0x024B6, 0x024D0}, {0x024B7, 0x024D1}, {0x024B8, 0x024D2}, {0x024B9, 0x024D3}, {0x024BA, 0x024D4}, {0x024BB, 0x024D5},-{0x024BC, 0x024D6}, {0x024BD, 0x024D7}, {0x024BE, 0x024D8}, {0x024BF, 0x024D9}, {0x024C0, 0x024DA}, {0x024C1, 0x024DB},-{0x024C2, 0x024DC}, {0x024C3, 0x024DD}, {0x024C4, 0x024DE}, {0x024C5, 0x024DF}, {0x024C6, 0x024E0}, {0x024C7, 0x024E1},-{0x024C8, 0x024E2}, {0x024C9, 0x024E3}, {0x024CA, 0x024E4}, {0x024CB, 0x024E5}, {0x024CC, 0x024E6}, {0x024CD, 0x024E7},-{0x024CE, 0x024E8}, {0x024CF, 0x024E9}, {0x02C00, 0x02C30}, {0x02C01, 0x02C31}, {0x02C02, 0x02C32}, {0x02C03, 0x02C33},-{0x02C04, 0x02C34}, {0x02C05, 0x02C35}, {0x02C06, 0x02C36}, {0x02C07, 0x02C37}, {0x02C08, 0x02C38}, {0x02C09, 0x02C39},-{0x02C0A, 0x02C3A}, {0x02C0B, 0x02C3B}, {0x02C0C, 0x02C3C}, {0x02C0D, 0x02C3D}, {0x02C0E, 0x02C3E}, {0x02C0F, 0x02C3F},-{0x02C10, 0x02C40}, {0x02C11, 0x02C41}, {0x02C12, 0x02C42}, {0x02C13, 0x02C43}, {0x02C14, 0x02C44}, {0x02C15, 0x02C45},-{0x02C16, 0x02C46}, {0x02C17, 0x02C47}, {0x02C18, 0x02C48}, {0x02C19, 0x02C49}, {0x02C1A, 0x02C4A}, {0x02C1B, 0x02C4B},-{0x02C1C, 0x02C4C}, {0x02C1D, 0x02C4D}, {0x02C1E, 0x02C4E}, {0x02C1F, 0x02C4F}, {0x02C20, 0x02C50}, {0x02C21, 0x02C51},-{0x02C22, 0x02C52}, {0x02C23, 0x02C53}, {0x02C24, 0x02C54}, {0x02C25, 0x02C55}, {0x02C26, 0x02C56}, {0x02C27, 0x02C57},-{0x02C28, 0x02C58}, {0x02C29, 0x02C59}, {0x02C2A, 0x02C5A}, {0x02C2B, 0x02C5B}, {0x02C2C, 0x02C5C}, {0x02C2D, 0x02C5D},-{0x02C2E, 0x02C5E}, {0x02C2F, 0x02C5F}, {0x02C60, 0x02C61}, {0x02C62, 0x0026B}, {0x02C63, 0x01D7D}, {0x02C64, 0x0027D},-{0x02C67, 0x02C68}, {0x02C69, 0x02C6A}, {0x02C6B, 0x02C6C}, {0x02C6D, 0x00251}, {0x02C6E, 0x00271}, {0x02C6F, 0x00250},-{0x02C70, 0x00252}, {0x02C72, 0x02C73}, {0x02C75, 0x02C76}, {0x02C7E, 0x0023F}, {0x02C7F, 0x00240}, {0x02C80, 0x02C81},-{0x02C82, 0x02C83}, {0x02C84, 0x02C85}, {0x02C86, 0x02C87}, {0x02C88, 0x02C89}, {0x02C8A, 0x02C8B}, {0x02C8C, 0x02C8D},-{0x02C8E, 0x02C8F}, {0x02C90, 0x02C91}, {0x02C92, 0x02C93}, {0x02C94, 0x02C95}, {0x02C96, 0x02C97}, {0x02C98, 0x02C99},-{0x02C9A, 0x02C9B}, {0x02C9C, 0x02C9D}, {0x02C9E, 0x02C9F}, {0x02CA0, 0x02CA1}, {0x02CA2, 0x02CA3}, {0x02CA4, 0x02CA5},-{0x02CA6, 0x02CA7}, {0x02CA8, 0x02CA9}, {0x02CAA, 0x02CAB}, {0x02CAC, 0x02CAD}, {0x02CAE, 0x02CAF}, {0x02CB0, 0x02CB1},-{0x02CB2, 0x02CB3}, {0x02CB4, 0x02CB5}, {0x02CB6, 0x02CB7}, {0x02CB8, 0x02CB9}, {0x02CBA, 0x02CBB}, {0x02CBC, 0x02CBD},-{0x02CBE, 0x02CBF}, {0x02CC0, 0x02CC1}, {0x02CC2, 0x02CC3}, {0x02CC4, 0x02CC5}, {0x02CC6, 0x02CC7}, {0x02CC8, 0x02CC9},-{0x02CCA, 0x02CCB}, {0x02CCC, 0x02CCD}, {0x02CCE, 0x02CCF}, {0x02CD0, 0x02CD1}, {0x02CD2, 0x02CD3}, {0x02CD4, 0x02CD5},-{0x02CD6, 0x02CD7}, {0x02CD8, 0x02CD9}, {0x02CDA, 0x02CDB}, {0x02CDC, 0x02CDD}, {0x02CDE, 0x02CDF}, {0x02CE0, 0x02CE1},-{0x02CE2, 0x02CE3}, {0x02CEB, 0x02CEC}, {0x02CED, 0x02CEE}, {0x02CF2, 0x02CF3}, {0x0A640, 0x0A641}, {0x0A642, 0x0A643},-{0x0A644, 0x0A645}, {0x0A646, 0x0A647}, {0x0A648, 0x0A649}, {0x0A64A, 0x0A64B}, {0x0A64C, 0x0A64D}, {0x0A64E, 0x0A64F},-{0x0A650, 0x0A651}, {0x0A652, 0x0A653}, {0x0A654, 0x0A655}, {0x0A656, 0x0A657}, {0x0A658, 0x0A659}, {0x0A65A, 0x0A65B},-{0x0A65C, 0x0A65D}, {0x0A65E, 0x0A65F}, {0x0A660, 0x0A661}, {0x0A662, 0x0A663}, {0x0A664, 0x0A665}, {0x0A666, 0x0A667},-{0x0A668, 0x0A669}, {0x0A66A, 0x0A66B}, {0x0A66C, 0x0A66D}, {0x0A680, 0x0A681}, {0x0A682, 0x0A683}, {0x0A684, 0x0A685},-{0x0A686, 0x0A687}, {0x0A688, 0x0A689}, {0x0A68A, 0x0A68B}, {0x0A68C, 0x0A68D}, {0x0A68E, 0x0A68F}, {0x0A690, 0x0A691},-{0x0A692, 0x0A693}, {0x0A694, 0x0A695}, {0x0A696, 0x0A697}, {0x0A698, 0x0A699}, {0x0A69A, 0x0A69B}, {0x0A722, 0x0A723},-{0x0A724, 0x0A725}, {0x0A726, 0x0A727}, {0x0A728, 0x0A729}, {0x0A72A, 0x0A72B}, {0x0A72C, 0x0A72D}, {0x0A72E, 0x0A72F},-{0x0A732, 0x0A733}, {0x0A734, 0x0A735}, {0x0A736, 0x0A737}, {0x0A738, 0x0A739}, {0x0A73A, 0x0A73B}, {0x0A73C, 0x0A73D},-{0x0A73E, 0x0A73F}, {0x0A740, 0x0A741}, {0x0A742, 0x0A743}, {0x0A744, 0x0A745}, {0x0A746, 0x0A747}, {0x0A748, 0x0A749},-{0x0A74A, 0x0A74B}, {0x0A74C, 0x0A74D}, {0x0A74E, 0x0A74F}, {0x0A750, 0x0A751}, {0x0A752, 0x0A753}, {0x0A754, 0x0A755},-{0x0A756, 0x0A757}, {0x0A758, 0x0A759}, {0x0A75A, 0x0A75B}, {0x0A75C, 0x0A75D}, {0x0A75E, 0x0A75F}, {0x0A760, 0x0A761},-{0x0A762, 0x0A763}, {0x0A764, 0x0A765}, {0x0A766, 0x0A767}, {0x0A768, 0x0A769}, {0x0A76A, 0x0A76B}, {0x0A76C, 0x0A76D},-{0x0A76E, 0x0A76F}, {0x0A779, 0x0A77A}, {0x0A77B, 0x0A77C}, {0x0A77D, 0x01D79}, {0x0A77E, 0x0A77F}, {0x0A780, 0x0A781},-{0x0A782, 0x0A783}, {0x0A784, 0x0A785}, {0x0A786, 0x0A787}, {0x0A78B, 0x0A78C}, {0x0A78D, 0x00265}, {0x0A790, 0x0A791},-{0x0A792, 0x0A793}, {0x0A796, 0x0A797}, {0x0A798, 0x0A799}, {0x0A79A, 0x0A79B}, {0x0A79C, 0x0A79D}, {0x0A79E, 0x0A79F},-{0x0A7A0, 0x0A7A1}, {0x0A7A2, 0x0A7A3}, {0x0A7A4, 0x0A7A5}, {0x0A7A6, 0x0A7A7}, {0x0A7A8, 0x0A7A9}, {0x0A7AA, 0x00266},-{0x0A7AB, 0x0025C}, {0x0A7AC, 0x00261}, {0x0A7AD, 0x0026C}, {0x0A7AE, 0x0026A}, {0x0A7B0, 0x0029E}, {0x0A7B1, 0x00287},-{0x0A7B2, 0x0029D}, {0x0A7B3, 0x0AB53}, {0x0A7B4, 0x0A7B5}, {0x0A7B6, 0x0A7B7}, {0x0A7B8, 0x0A7B9}, {0x0A7BA, 0x0A7BB},-{0x0A7BC, 0x0A7BD}, {0x0A7BE, 0x0A7BF}, {0x0A7C0, 0x0A7C1}, {0x0A7C2, 0x0A7C3}, {0x0A7C4, 0x0A794}, {0x0A7C5, 0x00282},-{0x0A7C6, 0x01D8E}, {0x0A7C7, 0x0A7C8}, {0x0A7C9, 0x0A7CA}, {0x0A7D0, 0x0A7D1}, {0x0A7D6, 0x0A7D7}, {0x0A7D8, 0x0A7D9},-{0x0A7F5, 0x0A7F6}, {0x0FF21, 0x0FF41}, {0x0FF22, 0x0FF42}, {0x0FF23, 0x0FF43}, {0x0FF24, 0x0FF44}, {0x0FF25, 0x0FF45},-{0x0FF26, 0x0FF46}, {0x0FF27, 0x0FF47}, {0x0FF28, 0x0FF48}, {0x0FF29, 0x0FF49}, {0x0FF2A, 0x0FF4A}, {0x0FF2B, 0x0FF4B},-{0x0FF2C, 0x0FF4C}, {0x0FF2D, 0x0FF4D}, {0x0FF2E, 0x0FF4E}, {0x0FF2F, 0x0FF4F}, {0x0FF30, 0x0FF50}, {0x0FF31, 0x0FF51},-{0x0FF32, 0x0FF52}, {0x0FF33, 0x0FF53}, {0x0FF34, 0x0FF54}, {0x0FF35, 0x0FF55}, {0x0FF36, 0x0FF56}, {0x0FF37, 0x0FF57},-{0x0FF38, 0x0FF58}, {0x0FF39, 0x0FF59}, {0x0FF3A, 0x0FF5A}, {0x10400, 0x10428}, {0x10401, 0x10429}, {0x10402, 0x1042A},-{0x10403, 0x1042B}, {0x10404, 0x1042C}, {0x10405, 0x1042D}, {0x10406, 0x1042E}, {0x10407, 0x1042F}, {0x10408, 0x10430},-{0x10409, 0x10431}, {0x1040A, 0x10432}, {0x1040B, 0x10433}, {0x1040C, 0x10434}, {0x1040D, 0x10435}, {0x1040E, 0x10436},-{0x1040F, 0x10437}, {0x10410, 0x10438}, {0x10411, 0x10439}, {0x10412, 0x1043A}, {0x10413, 0x1043B}, {0x10414, 0x1043C},-{0x10415, 0x1043D}, {0x10416, 0x1043E}, {0x10417, 0x1043F}, {0x10418, 0x10440}, {0x10419, 0x10441}, {0x1041A, 0x10442},-{0x1041B, 0x10443}, {0x1041C, 0x10444}, {0x1041D, 0x10445}, {0x1041E, 0x10446}, {0x1041F, 0x10447}, {0x10420, 0x10448},-{0x10421, 0x10449}, {0x10422, 0x1044A}, {0x10423, 0x1044B}, {0x10424, 0x1044C}, {0x10425, 0x1044D}, {0x10426, 0x1044E},-{0x10427, 0x1044F}, {0x104B0, 0x104D8}, {0x104B1, 0x104D9}, {0x104B2, 0x104DA}, {0x104B3, 0x104DB}, {0x104B4, 0x104DC},-{0x104B5, 0x104DD}, {0x104B6, 0x104DE}, {0x104B7, 0x104DF}, {0x104B8, 0x104E0}, {0x104B9, 0x104E1}, {0x104BA, 0x104E2},-{0x104BB, 0x104E3}, {0x104BC, 0x104E4}, {0x104BD, 0x104E5}, {0x104BE, 0x104E6}, {0x104BF, 0x104E7}, {0x104C0, 0x104E8},-{0x104C1, 0x104E9}, {0x104C2, 0x104EA}, {0x104C3, 0x104EB}, {0x104C4, 0x104EC}, {0x104C5, 0x104ED}, {0x104C6, 0x104EE},-{0x104C7, 0x104EF}, {0x104C8, 0x104F0}, {0x104C9, 0x104F1}, {0x104CA, 0x104F2}, {0x104CB, 0x104F3}, {0x104CC, 0x104F4},-{0x104CD, 0x104F5}, {0x104CE, 0x104F6}, {0x104CF, 0x104F7}, {0x104D0, 0x104F8}, {0x104D1, 0x104F9}, {0x104D2, 0x104FA},-{0x104D3, 0x104FB}, {0x10570, 0x10597}, {0x10571, 0x10598}, {0x10572, 0x10599}, {0x10573, 0x1059A}, {0x10574, 0x1059B},-{0x10575, 0x1059C}, {0x10576, 0x1059D}, {0x10577, 0x1059E}, {0x10578, 0x1059F}, {0x10579, 0x105A0}, {0x1057A, 0x105A1},-{0x1057C, 0x105A3}, {0x1057D, 0x105A4}, {0x1057E, 0x105A5}, {0x1057F, 0x105A6}, {0x10580, 0x105A7}, {0x10581, 0x105A8},-{0x10582, 0x105A9}, {0x10583, 0x105AA}, {0x10584, 0x105AB}, {0x10585, 0x105AC}, {0x10586, 0x105AD}, {0x10587, 0x105AE},-{0x10588, 0x105AF}, {0x10589, 0x105B0}, {0x1058A, 0x105B1}, {0x1058C, 0x105B3}, {0x1058D, 0x105B4}, {0x1058E, 0x105B5},-{0x1058F, 0x105B6}, {0x10590, 0x105B7}, {0x10591, 0x105B8}, {0x10592, 0x105B9}, {0x10594, 0x105BB}, {0x10595, 0x105BC},-{0x10C80, 0x10CC0}, {0x10C81, 0x10CC1}, {0x10C82, 0x10CC2}, {0x10C83, 0x10CC3}, {0x10C84, 0x10CC4}, {0x10C85, 0x10CC5},-{0x10C86, 0x10CC6}, {0x10C87, 0x10CC7}, {0x10C88, 0x10CC8}, {0x10C89, 0x10CC9}, {0x10C8A, 0x10CCA}, {0x10C8B, 0x10CCB},-{0x10C8C, 0x10CCC}, {0x10C8D, 0x10CCD}, {0x10C8E, 0x10CCE}, {0x10C8F, 0x10CCF}, {0x10C90, 0x10CD0}, {0x10C91, 0x10CD1},-{0x10C92, 0x10CD2}, {0x10C93, 0x10CD3}, {0x10C94, 0x10CD4}, {0x10C95, 0x10CD5}, {0x10C96, 0x10CD6}, {0x10C97, 0x10CD7},-{0x10C98, 0x10CD8}, {0x10C99, 0x10CD9}, {0x10C9A, 0x10CDA}, {0x10C9B, 0x10CDB}, {0x10C9C, 0x10CDC}, {0x10C9D, 0x10CDD},-{0x10C9E, 0x10CDE}, {0x10C9F, 0x10CDF}, {0x10CA0, 0x10CE0}, {0x10CA1, 0x10CE1}, {0x10CA2, 0x10CE2}, {0x10CA3, 0x10CE3},-{0x10CA4, 0x10CE4}, {0x10CA5, 0x10CE5}, {0x10CA6, 0x10CE6}, {0x10CA7, 0x10CE7}, {0x10CA8, 0x10CE8}, {0x10CA9, 0x10CE9},-{0x10CAA, 0x10CEA}, {0x10CAB, 0x10CEB}, {0x10CAC, 0x10CEC}, {0x10CAD, 0x10CED}, {0x10CAE, 0x10CEE}, {0x10CAF, 0x10CEF},-{0x10CB0, 0x10CF0}, {0x10CB1, 0x10CF1}, {0x10CB2, 0x10CF2}, {0x118A0, 0x118C0}, {0x118A1, 0x118C1}, {0x118A2, 0x118C2},-{0x118A3, 0x118C3}, {0x118A4, 0x118C4}, {0x118A5, 0x118C5}, {0x118A6, 0x118C6}, {0x118A7, 0x118C7}, {0x118A8, 0x118C8},-{0x118A9, 0x118C9}, {0x118AA, 0x118CA}, {0x118AB, 0x118CB}, {0x118AC, 0x118CC}, {0x118AD, 0x118CD}, {0x118AE, 0x118CE},-{0x118AF, 0x118CF}, {0x118B0, 0x118D0}, {0x118B1, 0x118D1}, {0x118B2, 0x118D2}, {0x118B3, 0x118D3}, {0x118B4, 0x118D4},-{0x118B5, 0x118D5}, {0x118B6, 0x118D6}, {0x118B7, 0x118D7}, {0x118B8, 0x118D8}, {0x118B9, 0x118D9}, {0x118BA, 0x118DA},-{0x118BB, 0x118DB}, {0x118BC, 0x118DC}, {0x118BD, 0x118DD}, {0x118BE, 0x118DE}, {0x118BF, 0x118DF}, {0x16E40, 0x16E60},-{0x16E41, 0x16E61}, {0x16E42, 0x16E62}, {0x16E43, 0x16E63}, {0x16E44, 0x16E64}, {0x16E45, 0x16E65}, {0x16E46, 0x16E66},-{0x16E47, 0x16E67}, {0x16E48, 0x16E68}, {0x16E49, 0x16E69}, {0x16E4A, 0x16E6A}, {0x16E4B, 0x16E6B}, {0x16E4C, 0x16E6C},-{0x16E4D, 0x16E6D}, {0x16E4E, 0x16E6E}, {0x16E4F, 0x16E6F}, {0x16E50, 0x16E70}, {0x16E51, 0x16E71}, {0x16E52, 0x16E72},-{0x16E53, 0x16E73}, {0x16E54, 0x16E74}, {0x16E55, 0x16E75}, {0x16E56, 0x16E76}, {0x16E57, 0x16E77}, {0x16E58, 0x16E78},-{0x16E59, 0x16E79}, {0x16E5A, 0x16E7A}, {0x16E5B, 0x16E7B}, {0x16E5C, 0x16E7C}, {0x16E5D, 0x16E7D}, {0x16E5E, 0x16E7E},-{0x16E5F, 0x16E7F}, {0x1E900, 0x1E922}, {0x1E901, 0x1E923}, {0x1E902, 0x1E924}, {0x1E903, 0x1E925}, {0x1E904, 0x1E926},-{0x1E905, 0x1E927}, {0x1E906, 0x1E928}, {0x1E907, 0x1E929}, {0x1E908, 0x1E92A}, {0x1E909, 0x1E92B}, {0x1E90A, 0x1E92C},-{0x1E90B, 0x1E92D}, {0x1E90C, 0x1E92E}, {0x1E90D, 0x1E92F}, {0x1E90E, 0x1E930}, {0x1E90F, 0x1E931}, {0x1E910, 0x1E932},-{0x1E911, 0x1E933}, {0x1E912, 0x1E934}, {0x1E913, 0x1E935}, {0x1E914, 0x1E936}, {0x1E915, 0x1E937}, {0x1E916, 0x1E938},-{0x1E917, 0x1E939}, {0x1E918, 0x1E93A}, {0x1E919, 0x1E93B}, {0x1E91A, 0x1E93C}, {0x1E91B, 0x1E93D}, {0x1E91C, 0x1E93E},-{0x1E91D, 0x1E93F}, {0x1E91E, 0x1E940}, {0x1E91F, 0x1E941}, {0x1E920, 0x1E942}, {0x1E921, 0x1E943},-};diff --git a/llama.cpp/unicode-data.h b/llama.cpp/unicode-data.hindex 3cf84117..e27fe177 100644--- a/llama.cpp/unicode-data.h+++ b/llama.cpp/unicode-data.h@@ -1,16 +1,20 @@#pragma once#include <cstdint>-#include <map>-#include <utility>#include <vector>+#include <unordered_map>+#include <unordered_set>-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_number;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_letter;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_whitespace;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_mark;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;-extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;-extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;-extern const std::map<char32_t, char32_t> unicode_map_lowercase;+struct range_nfd {+ uint32_t first;+ uint32_t last;+ uint32_t nfd;+};++static const uint32_t MAX_CODEPOINTS = 0x110000;++extern const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;+extern const std::unordered_set<uint32_t> unicode_set_whitespace;+extern const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase;+extern const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase;+extern const std::vector<range_nfd> unicode_ranges_nfd;diff --git a/llama.cpp/unicode.cpp b/llama.cpp/unicode.cppindex 955c5696..056a4c74 100644--- a/llama.cpp/unicode.cpp+++ b/llama.cpp/unicode.cpp@@ -1,4 +1,4 @@-#include "unicode.h"+#include "unicode.h"#include "unicode-data.h"#include <cassert>@@ -9,6 +9,7 @@#include <stdexcept>#include <string>#include <unordered_map>+#include <unordered_set>#include <utility>#include <vector>#include <locale>@@ -108,57 +109,49 @@ static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset)// return result;//}-static std::unordered_map<uint32_t, int> unicode_cpt_type_map() {- std::unordered_map<uint32_t, int> cpt_types;- for (auto p : unicode_ranges_number) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_NUMBER;- }- }- for (auto p : unicode_ranges_letter) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_LETTER;- }- }- for (auto p : unicode_ranges_whitespace) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_WHITESPACE;+static std::vector<codepoint_flags> unicode_cpt_flags_array() {+ std::vector<codepoint_flags> cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED);++ assert (unicode_ranges_flags.front().first == 0);+ assert (unicode_ranges_flags.back().first == MAX_CODEPOINTS);+ for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {+ const auto range_ini = unicode_ranges_flags[i-1]; // codepoint_ini, flags+ const auto range_end = unicode_ranges_flags[i]; // codepoint_end, flags+ for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {+ cpt_flags[cpt] = range_ini.second;}}- for (auto p : unicode_ranges_accent_mark) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_ACCENT_MARK;- }++ for (auto cpt : unicode_set_whitespace) {+ cpt_flags[cpt].is_whitespace = true;}- for (auto p : unicode_ranges_punctuation) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_PUNCTUATION;- }++ for (auto p : unicode_map_lowercase) {+ cpt_flags[p.second].is_lowercase = true;}- for (auto p : unicode_ranges_symbol) {- for (auto i = p.first; i <= p.second; ++i) {- cpt_types[i] = CODEPOINT_TYPE_SYMBOL;- }++ for (auto p : unicode_map_uppercase) {+ cpt_flags[p.second].is_uppercase = true;}- for (auto p : unicode_ranges_control) {- for (auto i = p.first; i <= p.second; ++ i) {- cpt_types[i] = CODEPOINT_TYPE_CONTROL;- }++ for (auto &range : unicode_ranges_nfd) { // start, last, nfd+ cpt_flags[range.nfd].is_nfd = true;}- return cpt_types;++ return cpt_flags;}static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {std::unordered_map<uint8_t, std::string> map;- for (int ch = u'!'; ch <= u'~'; ++ch) {+ for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'assert(0 <= ch && ch < 256);map[ch] = unicode_cpt_to_utf8(ch);}- for (int ch = u'¡'; ch <= u'¬'; ++ch) {+ for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'assert(0 <= ch && ch < 256);map[ch] = unicode_cpt_to_utf8(ch);}- for (int ch = u'®'; ch <= u'ÿ'; ++ch) {+ for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'assert(0 <= ch && ch < 256);map[ch] = unicode_cpt_to_utf8(ch);}@@ -174,15 +167,15 @@ static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {std::unordered_map<std::string, uint8_t> map;- for (int ch = u'!'; ch <= u'~'; ++ch) {+ for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'assert(0 <= ch && ch < 256);map[unicode_cpt_to_utf8(ch)] = ch;}- for (int ch = u'¡'; ch <= u'¬'; ++ch) {+ for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'assert(0 <= ch && ch < 256);map[unicode_cpt_to_utf8(ch)] = ch;}- for (int ch = u'®'; ch <= u'ÿ'; ++ch) {+ for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'assert(0 <= ch && ch < 256);map[unicode_cpt_to_utf8(ch)] = ch;}@@ -224,138 +217,255 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & tstd::vector<size_t> bpe_offsets; // store the offset of each wordbpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size- size_t start = 0;-const auto cpts = unicode_cpts_from_utf8(text);+ size_t start = 0;for (auto offset : offsets) {- std::string token;+ const size_t offset_ini = start;+ const size_t offset_end = start + offset;+ assert(offset_end <= cpts.size());+ start = offset_end;++ auto _get_cpt = [&] (const size_t pos) -> char32_t {+ return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : 0;+ };++ auto _get_flags = [&] (const size_t pos) -> codepoint_flags {+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);+ return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : undef;+ };++ size_t _prev_end = offset_ini;+ auto _add_token = [&] (const size_t end) -> size_t {+ assert(_prev_end <= end && end <= offset_end);+ size_t len = end - _prev_end;+ if (len > 0) {+ bpe_offsets.push_back(len);+ }+ _prev_end = end;+ //if (len > 0) {+ // std::string s = "";+ // for(size_t p = end-len; p < end; p++)+ // s += unicode_cpt_to_utf8(cpts[p]);+ // printf(">>> '%s'\n", s.c_str());+ //}+ return len;+ };++ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {+ const char32_t cpt = _get_cpt(pos);+ const auto flags = _get_flags(pos);++ // regex: 's|'t|'re|'ve|'m|'ll|'d+ if (cpt == '\'' && pos+1 < offset_end) {+ char32_t cpt_next = _get_cpt(pos+1);+ if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {+ pos += _add_token(pos+2);+ continue;+ }+ if (pos+2 < offset_end) {+ char32_t cpt_next_next = _get_cpt(pos+2);+ if ((cpt_next == 'r' && cpt_next_next == 'e') ||+ (cpt_next == 'v' && cpt_next_next == 'e') ||+ (cpt_next == 'l' && cpt_next_next == 'l')) {+ pos += _add_token(pos+3);+ continue;+ }+ }+ }++ auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);+ // regex: <space>?\p{L}++ if (flags2.is_letter) {+ pos += (cpt == ' ');+ while (flags2.is_letter) {+ flags2 = _get_flags(++pos);+ }+ _add_token(pos);+ continue;+ }+ // regex: <space>?\p{N}++ if (flags2.is_number) {+ pos += (cpt == ' ');+ while (flags2.is_number) {+ flags2 = _get_flags(++pos);+ }+ _add_token(pos);+ continue;+ }+ // regex: <space>?[^\s\p{L}\p{N}]++ if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) {+ pos += (cpt == ' ');+ while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) {+ flags2 = _get_flags(++pos);+ }+ _add_token(pos);+ continue;+ }++ size_t num_whitespaces = 0;+ while (_get_flags(pos+num_whitespaces).is_whitespace) {+ num_whitespaces++;+ }- bool collecting_numeric = false;- bool collecting_letter = false;- bool collecting_special = false;- bool collecting_whitespace_lookahead = false;- bool collecting = false;+ // regex: \s+(?!\S)+ if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) {+ pos += num_whitespaces - 1;+ _add_token(pos);+ continue;+ }- std::vector<std::string> text_utf;- text_utf.reserve(offset);+ // regex: \s++ if (num_whitespaces > 0) {+ pos += num_whitespaces;+ _add_token(pos);+ continue;+ }- for (size_t i = start; i < start + offset; ++i) {- text_utf.emplace_back(unicode_cpt_to_utf8(cpts[i]));+ // no matches+ _add_token(++pos);}+ }- for (int i = 0; i < (int)text_utf.size(); i++) {- const std::string & utf_char = text_utf[i];- bool split_condition = false;- int bytes_remain = text_utf.size() - i;+ return bpe_offsets;+}++// LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"+static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) {+ std::vector<size_t> bpe_offsets; // store the offset of each word+ bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size- // forward backward lookups- const std::string & utf_char_next = (i + 1 < (int)text_utf.size()) ? text_utf[i + 1] : "";- const std::string & utf_char_next_next = (i + 2 < (int)text_utf.size()) ? text_utf[i + 2] : "";+ const auto cpts = unicode_cpts_from_utf8(text);- // handling contractions- if (!split_condition && bytes_remain >= 2) {- // 's|'t|'m|'d- if (utf_char == "\'" && (utf_char_next == "s" || utf_char_next == "t" || utf_char_next == "m" || utf_char_next == "d")) {- split_condition = true;+ size_t start = 0;+ for (auto offset : offsets) {+ const size_t offset_ini = start;+ const size_t offset_end = start + offset;+ assert(offset_end <= cpts.size());+ start = offset_end;++ auto _get_cpt = [&] (const size_t pos) -> char32_t {+ return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : 0;+ };++ auto _get_flags = [&] (const size_t pos) -> codepoint_flags {+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);+ return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : undef;+ };++ size_t _prev_end = offset_ini;+ auto _add_token = [&] (const size_t end) -> size_t {+ assert(_prev_end <= end && end <= offset_end);+ size_t len = end - _prev_end;+ if (len > 0) {+ bpe_offsets.push_back(len);+ }+ _prev_end = end;+ //if (len > 0) {+ // std::string s = "";+ // for(size_t p = end-len; p < end; p++)+ // s += unicode_cpt_to_utf8(cpts[p]);+ // printf(">>> '%s'\n", s.c_str());+ //}+ return len;+ };++ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {+ const char32_t cpt = _get_cpt(pos);+ const auto flags = _get_flags(pos);++ // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive+ if (cpt == '\'' && pos+1 < offset_end) {+ char32_t cpt_next = unicode_tolower(_get_cpt(pos+1));+ if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {+ pos += _add_token(pos+2);+ continue;}- if (split_condition) {- if (token.size()) {- bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());+ if (pos+2 < offset_end) {+ char32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));+ if ((cpt_next == 'r' && cpt_next_next == 'e') ||+ (cpt_next == 'v' && cpt_next_next == 'e') ||+ (cpt_next == 'l' && cpt_next_next == 'l')) {+ pos += _add_token(pos+3);+ continue;}- token = utf_char + utf_char_next;- bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());- token = "";- i++;- continue;}}- if (!split_condition && bytes_remain >= 3) {- // 're|'ve|'ll- if (utf_char == "\'" && (- (utf_char_next == "r" && utf_char_next_next == "e") ||- (utf_char_next == "v" && utf_char_next_next == "e") ||- (utf_char_next == "l" && utf_char_next_next == "l"))- ) {- split_condition = true;- }- if (split_condition) {- // current token + next token can be defined- if (token.size()) {- bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());- }- token = utf_char;- token += utf_char_next;- token += utf_char_next_next;- bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());- token = "";- i += 2;+ // regex: [^\r\n\p{L}\p{N}]?\p{L}+ //####FIXME: the first \p{L} is correct?+ if (!(cpt == '\r' || cpt == '\n' || /*flags.is_letter |*/ flags.is_number)) {+ if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters+ pos++;+ while (_get_flags(pos).is_letter) {+ pos++;+ }+ _add_token(pos);continue;}}- if (!split_condition && !collecting) {- if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER)) {- collecting_letter = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_NUMBER || (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_NUMBER)) {- collecting_numeric = true;- collecting = true;- }- else if (- ((unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_NUMBER) && (unicode_cpt_type(utf_char) != CODEPOINT_TYPE_WHITESPACE)) ||- (token.empty() && utf_char == " " && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_LETTER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_NUMBER && unicode_cpt_type(utf_char_next) != CODEPOINT_TYPE_WHITESPACE)- ) {- collecting_special = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE && unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_WHITESPACE) {- collecting_whitespace_lookahead = true;- collecting = true;- }- else if (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE) {- split_condition = true;+ // regex: \p{N}{1,3}+ if (flags.is_number) {+ size_t ini = pos;+ while (_get_flags(pos).is_number) {+ if (++pos - ini >= 3 ) {+ _add_token(pos);+ ini = pos;+ }}+ _add_token(pos);+ continue;}- else if (!split_condition && collecting) {- if (collecting_letter && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_LETTER) {- split_condition = true;- }- else if (collecting_numeric && unicode_cpt_type(utf_char) != CODEPOINT_TYPE_NUMBER) {- split_condition = true;++ // regex: <space>?[^\s\p{L}\p{N}]+[\r\n]*+ auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);+ if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) {+ pos += (cpt == ' ');+ while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) {+ flags2 = _get_flags(++pos);}- else if (collecting_special && (unicode_cpt_type(utf_char) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_NUMBER || unicode_cpt_type(utf_char) == CODEPOINT_TYPE_WHITESPACE)) {- split_condition = true;+ char32_t cpt2 = _get_cpt(pos);+ while (cpt2 == '\r' || cpt2 == '\n') {+ cpt2 = _get_cpt(++pos);}- else if (collecting_whitespace_lookahead && (unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_LETTER || unicode_cpt_type(utf_char_next) == CODEPOINT_TYPE_NUMBER)) {- split_condition = true;+ _add_token(pos);+ continue;+ }++ size_t num_whitespaces = 0;+ size_t last_end_r_or_n = 0;+ while (_get_flags(pos+num_whitespaces).is_whitespace) {+ char32_t cpt2 = _get_cpt(pos+num_whitespaces);+ if (cpt2 == '\r' || cpt2 == '\n') {+ last_end_r_or_n = pos + num_whitespaces + 1;}+ num_whitespaces++;}- if (utf_char_next == "") {- split_condition = true; // final- token += utf_char;+ // regex: \s*[\r\n]++ if (last_end_r_or_n > 0) {+ pos = last_end_r_or_n;+ _add_token(pos);+ continue;}- if (split_condition) {- if (token.size()) {- bpe_offsets.emplace_back(unicode_cpts_from_utf8(token).size());- }- token = utf_char;- collecting = false;- collecting_letter = false;- collecting_numeric = false;- collecting_special = false;- collecting_whitespace_lookahead = false;+ // regex: \s+(?!\S)+ if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) {+ pos += num_whitespaces - 1;+ _add_token(pos);+ continue;}- else {- token += utf_char;++ // regex: \s++ if (num_whitespaces > 0) {+ pos += num_whitespaces;+ _add_token(pos);+ continue;}- }- start += offset;+ // no matches+ _add_token(++pos);+ }}return bpe_offsets;@@ -424,14 +534,14 @@ static std::vector<size_t> unicode_regex_split_stl(const std::string & text, constatic std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {std::vector<size_t> bpe_offsets;- (void)(text);- (void)(regex_expr);- (void)(offsets);- // TODO: this implementation is actually wrong, uncomment and run:- // make -j && ./bin/test-tokenizer-0 ../models/ggml-vocab-gpt-2.gguf- //if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {- // bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);- //}+ if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {+ bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);+ } else if (+ regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||+ regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {++ bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);+ }return bpe_offsets;}@@ -470,15 +580,14 @@ std::string unicode_cpt_to_utf8(uint32_t cp) {}std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {- std::vector<uint32_t> result;- result.reserve(cpts.size());+ auto comp = [] (const uint32_t cpt, const range_nfd & range) {+ return cpt < range.first;+ };+ std::vector<uint32_t> result(cpts.size());for (size_t i = 0; i < cpts.size(); ++i) {- auto it = unicode_map_nfd.find(cpts[i]);- if (it == unicode_map_nfd.end()) {- result.push_back(cpts[i]);- } else {- result.push_back(it->second);- }+ const uint32_t cpt = cpts[i];+ auto it = std::upper_bound(unicode_ranges_nfd.cbegin(), unicode_ranges_nfd.cend(), cpt, comp) - 1;+ result[i] = (it->first <= cpt && cpt <= it->last) ? it->nfd : cpt;}return result;}@@ -492,18 +601,19 @@ std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {return result;}-int unicode_cpt_type(uint32_t cp) {- static std::unordered_map<uint32_t, int> cpt_types = unicode_cpt_type_map();- const auto it = cpt_types.find(cp);- return it == cpt_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second;+codepoint_flags unicode_cpt_flags(const uint32_t cp) {+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);+ static const auto cpt_flags = unicode_cpt_flags_array();+ return cp < cpt_flags.size() ? cpt_flags[cp] : undef;}-int unicode_cpt_type(const std::string & utf8) {- if (utf8.length() == 0) {- return CODEPOINT_TYPE_UNIDENTIFIED;+codepoint_flags unicode_cpt_flags(const std::string & utf8) {+ static const codepoint_flags undef(codepoint_flags::UNDEFINED);+ if (utf8.empty()) {+ return undef; // undefined}size_t offset = 0;- return unicode_cpt_type(unicode_cpt_from_utf8(utf8, offset));+ return unicode_cpt_flags(unicode_cpt_from_utf8(utf8, offset));}std::string unicode_byte_to_utf8(uint8_t byte) {@@ -524,21 +634,21 @@ char32_t unicode_tolower(char32_t cp) {std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {// unicode categoriesstatic const std::map<std::string, int> k_ucat_enum = {- { "\\p{N}", CODEPOINT_TYPE_NUMBER },- { "\\p{L}", CODEPOINT_TYPE_LETTER },- { "\\p{P}", CODEPOINT_TYPE_PUNCTUATION },+ { "\\p{N}", codepoint_flags::NUMBER },+ { "\\p{L}", codepoint_flags::LETTER },+ { "\\p{P}", codepoint_flags::PUNCTUATION },};static const std::map<int, int> k_ucat_cpt = {- { CODEPOINT_TYPE_NUMBER, 0xD1 },- { CODEPOINT_TYPE_LETTER, 0xD2 },- { CODEPOINT_TYPE_PUNCTUATION, 0xD3 },+ { codepoint_flags::NUMBER, 0xD1 },+ { codepoint_flags::LETTER, 0xD2 },+ { codepoint_flags::PUNCTUATION, 0xD3 },};static const std::map<int, std::string> k_ucat_map = {- { CODEPOINT_TYPE_NUMBER, "\x30-\x39" }, // 0-9- { CODEPOINT_TYPE_LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z- { CODEPOINT_TYPE_PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}+ { codepoint_flags::NUMBER, "\x30-\x39" }, // 0-9+ { codepoint_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z+ { codepoint_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\}};// compute collapsed codepoints only if needed by at least one regex@@ -569,10 +679,10 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const stdcontinue;}- const int cpt_type = unicode_cpt_type(cpts[i]);+ const int cpt_flag = unicode_cpt_flags(cpts[i]).category_flag();- if (k_ucat_cpt.find(cpt_type) != k_ucat_cpt.end()) {- text_collapsed[i] = k_ucat_cpt.at(cpt_type);+ if (k_ucat_cpt.find(cpt_flag) != k_ucat_cpt.end()) {+ text_collapsed[i] = k_ucat_cpt.at(cpt_flag);} else {text_collapsed[i] = (char) 0xD0; // fallback}diff --git a/llama.cpp/unicode.h b/llama.cpp/unicode.hindex e9026dc8..7513be4a 100644--- a/llama.cpp/unicode.h+++ b/llama.cpp/unicode.h@@ -4,22 +4,56 @@#include <string>#include <vector>-#define CODEPOINT_TYPE_UNIDENTIFIED 0-#define CODEPOINT_TYPE_NUMBER 1-#define CODEPOINT_TYPE_LETTER 2-#define CODEPOINT_TYPE_WHITESPACE 3-#define CODEPOINT_TYPE_ACCENT_MARK 4-#define CODEPOINT_TYPE_PUNCTUATION 5-#define CODEPOINT_TYPE_SYMBOL 6-#define CODEPOINT_TYPE_CONTROL 7+struct codepoint_flags {+ enum {+ UNDEFINED = 0x0001,+ NUMBER = 0x0002, // regex: \p{N}+ LETTER = 0x0004, // regex: \p{L}+ SEPARATOR = 0x0008, // regex: \p{Z}+ ACCENT_MARK = 0x0010, // regex: \p{M}+ PUNCTUATION = 0x0020, // regex: \p{P}+ SYMBOL = 0x0040, // regex: \p{S}+ CONTROL = 0x0080, // regex: \p{C}+ MASK_CATEGORIES = 0x00FF,+ };++ // codepoint type+ uint16_t is_undefined : 1;+ uint16_t is_number : 1; // regex: \p{N}+ uint16_t is_letter : 1; // regex: \p{L}+ uint16_t is_separator : 1; // regex: \p{Z}+ uint16_t is_accent_mark : 1; // regex: \p{M}+ uint16_t is_punctuation : 1; // regex: \p{P}+ uint16_t is_symbol : 1; // regex: \p{S}+ uint16_t is_control : 1; // regex: \p{C}+ // helper flags+ uint16_t is_whitespace : 1; // regex: \s+ uint16_t is_lowercase : 1;+ uint16_t is_uppercase : 1;+ uint16_t is_nfd : 1;++ // decode from uint16+ inline codepoint_flags(const uint16_t flags=0) {+ *reinterpret_cast<uint16_t*>(this) = flags;+ }++ inline uint16_t as_uint() const {+ return *reinterpret_cast<const uint16_t*>(this);+ }++ inline uint16_t category_flag() const {+ return this->as_uint() & MASK_CATEGORIES;+ }+};+std::string unicode_cpt_to_utf8(uint32_t cp);std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);-int unicode_cpt_type(uint32_t cp);-int unicode_cpt_type(const std::string & utf8);+codepoint_flags unicode_cpt_flags(const uint32_t cp);+codepoint_flags unicode_cpt_flags(const std::string & utf8);std::string unicode_byte_to_utf8(uint8_t byte);uint8_t unicode_utf8_to_byte(const std::string & utf8);diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 352f2ee4..120c3f2d 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -57,16 +57,24 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");#define NVCC_LIBS BLAS_ONLY("-lcublas"), "-lcuda"-#define NVCC_FLAGS \- "--shared", "--forward-unknown-to-host-compiler", "--compiler-options", \- (!IsWindows() ? (!IsAarch64() ? "-fPIC -O3 -march=native -mtune=native" \- : "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \- : "/nologo /EHsc /O2 /GR /MT"), \- "-DNDEBUG", "-DGGML_BUILD=1", "-DGGML_SHARED=1", "-DGGML_CUDA_MMV_Y=1", \- "-DGGML_MULTIPLATFORM", "-DGGML_CUDA_DMMV_X=32", "-DK_QUANTS_PER_ITERATION=2", \- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+#define COMMON_FLAGS \+ /* "-DNDEBUG", */ "-DGGML_BUILD=1", "-DGGML_SHARED=1", "-DGGML_MULTIPLATFORM", \+ "-DGGML_CUDA_DMMV_X=32", "-DK_QUANTS_PER_ITERATION=2", \+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", "-DGGML_CUDA_MMV_Y=1", \(FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DGGML_USE_CUBLAS")+#define NVCC_FLAGS \+ "-std=c++11", "-O3", "--shared", "--use_fast_math", "--forward-unknown-to-host-compiler", \+ "--compiler-options", \+ (!IsWindows() \+ ? (!IsAarch64() \+ ? "-fPIC -O3 -march=native -mtune=native -std=c++11 -Wno-unused-function " \+ "-Wno-unused-result -Wno-return-type -Wno-pedantic" \+ : "-fPIC -O3 -march=native -mtune=native -std=c++11 -Wno-unused-function " \+ "-Wno-unused-result -Wno-return-type -Wno-pedantic -ffixed-x28") \+ : "/nologo /EHsc /O2 /GR /MT"), \+ COMMON_FLAGS+static const struct Source {const char *zip;const char *name;@@ -538,22 +546,20 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char(char *)clangxx,"-O3","-shared",- "-DNDEBUG","-x","hip","--hip-link","-std=gnu++14","-fuse-ld=lld",- "-DGGML_BUILD=1",- "-DGGML_SHARED=1","-DGGML_USE_HIPBLAS",+ "-Wno-return-type",+ "-Wno-unused-result",+ "-Wno-unused-function",+ "-Wno-expansion-to-defined",(char *)offload_arch,- "-DGGML_CUDA_MMV_Y=1",- "-DGGML_MULTIPLATFORM",- "-DGGML_CUDA_DMMV_X=32","-Wno-ignored-attributes","-D_CRT_SECURE_NO_WARNINGS",- "-DK_QUANTS_PER_ITERATION=2",+ COMMON_FLAGS,"-o",(char *)tmpdso,(char *)src,@@ -563,7 +569,6 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char"-amdgpu-function-calls=false","-mllvm","-amdgpu-early-inline-all=true",- FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DIGNORE","-isystem",gc(xasprintf("%s/include", hip_path)),BLAS_ONLY("-l"),@@ -592,22 +597,16 @@ static bool compile_amd_unix(const char *dso, const char *src, const char *tmpds"-O3","-fPIC","-shared",- "-DNDEBUG",offload_arch,"-march=native","-mtune=native",- "-DGGML_BUILD=1",- "-DGGML_SHARED=1",+ "-DGGML_USE_HIPBLAS","-Wno-return-type","-Wno-unused-result",- "-DGGML_USE_HIPBLAS",- "-DGGML_CUDA_MMV_Y=1",- "-DGGML_MULTIPLATFORM",- "-DGGML_CUDA_DMMV_X=32",+ "-Wno-unused-function",+ "-Wno-expansion-to-defined",ARMS_ONLY("-ffixed-x28"),- "-DK_QUANTS_PER_ITERATION=2",- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128",- FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DIGNORE",+ COMMON_FLAGS,"-o",(char *)tmpdso,(char *)src,@@ -757,7 +756,7 @@ static bool import_cuda_impl(void) {bool needs_rebuild = FLAG_recompile;for (int i = 0; i < sizeof(srcs) / sizeof(*srcs); ++i) {llamafile_get_app_dir(src, sizeof(src));- if (!i && mkdir(src, 0755) && errno != EEXIST) {+ if (!i && makedirs(src, 0755)) {perror(src);return false;}diff --git a/llamafile/get_app_dir.c b/llamafile/get_app_dir.cindex cb95afd6..387194e1 100644--- a/llamafile/get_app_dir.c+++ b/llamafile/get_app_dir.c@@ -16,8 +16,9 @@// limitations under the License.#include "llamafile.h"+#include "version.h"+#include <stdio.h>#include <stdlib.h>-#include <string.h>static const char *llamafile_get_home_dir(void) {const char *homedir;@@ -30,6 +31,6 @@ static const char *llamafile_get_home_dir(void) {* Returns path of directory for app-specific files.*/void llamafile_get_app_dir(char *path, size_t size) {- strlcpy(path, llamafile_get_home_dir(), size);- strlcat(path, "/.llamafile/", size);+ snprintf(path, size, "%s/.llamafile/v/%d.%d.%d/", llamafile_get_home_dir(), LLAMAFILE_MAJOR,+ LLAMAFILE_MINOR, LLAMAFILE_PATCH);}diff --git a/llamafile/metal.c b/llamafile/metal.cindex 2cdd3f92..816b3841 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -96,7 +96,7 @@ static bool BuildMetal(const char *dso) {bool needs_rebuild = false;for (int i = 0; i < sizeof(srcs) / sizeof(*srcs); ++i) {llamafile_get_app_dir(src, PATH_MAX);- if (!i && mkdir(src, 0755) && errno != EEXIST) {+ if (!i && makedirs(src, 0755)) {perror(src);return false;}diff --git a/llamafile/rollup.py b/llamafile/rollup.pynew file mode 100755index 00000000..43ed1e00--- /dev/null+++ b/llamafile/rollup.py@@ -0,0 +1,62 @@+#!/usr/bin/env python++import os+import re+import sys++fils = sys.argv[1:]+hdrs = set()+sysi = set()++for path in fils:+ with open(path) as f:+ for line in f:+ line = line.rstrip('\n')+ m = re.search(r'# *include *"([^"]*)"', line)+ if m:+ hdr = os.path.basename(m.group(1))+ hdrs.add(hdr)+ m = re.search(r'# *include *<([^>]*)>', line)+ if m:+ hdr = m.group(1)+ if ('cu' not in hdr and+ 'hip' not in hdr and+ not os.path.exists(os.path.join('/usr/local/cuda/include', hdr))):+ sysi.add(hdr)++srcs = [f for f in fils if f not in hdrs]+done = set()++def visit(path):+ print()+ print('////////////////////////////////////////////////////////////////////////////////')+ print('//')+ print('// ROLLUP %s' % (path))+ print('//')+ print('////////////////////////////////////////////////////////////////////////////////')+ print()+ with open(path) as f:+ for line in f:+ line = line.rstrip('\n')+ m = re.search(r'# *include *"([^"]*)"', line)+ if m:+ hdr = os.path.basename(m.group(1))+ if hdr in fils:+ if hdr not in done:+ done.add(hdr)+ visit(hdr)+ continue+ m = re.search(r'# *include *<([^>]*)>', line)+ if m:+ hdr = m.group(1)+ if hdr in sysi:+ continue+ if '#pragma once' in line:+ continue+ print(line)++for path in sorted(sysi):+ print('#include <%s>' % (path))+print()+for path in srcs:+ visit(path)diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.hindex c728c8cd..43021159 100644--- a/llamafile/tinyblas_cpu.h+++ b/llamafile/tinyblas_cpu.h@@ -16,13 +16,15 @@// limitations under the License.//-// _ _ ___ _ _ ___-// | |_(_)_ _ _ _| _ ) | /_\ / __|-// | _| | ' \ || | _ \ |__ / _ \\__ \.-// \__|_|_||_\_, |___/____/_/ \_\___/-// |__///-// BASIC LINEAR ALGEBRA SUBPROGRAMS+// ██████╗ ██╗ █████╗ ██████╗+// ██████╗██╗██╗ ██╗██═██╗██╔══██╗██║ ██╔══██╗██╔═══╝+// ╚═██╔═╝██║███▄██║██ ██║██████╔╝██║ ███████║██████╗+// ██║ ██║██▀███║╚███╔╝██╔══██╗██║ ██╔══██║╔═══██║+// ██║ ██║██║ ██║ ███║ ██████╔╝████╗██║ ██║██████║+// ╚═╝ ╚═╝╚═╝ ╚═╝ ╚══╝ ╚═════╝ ╚═══╝╚═╝ ╚═╝╚═════╝+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS////// This file implements multithreaded CPU matrix multiplication for thediff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.incindex 560771ea..bdc9ece7 100644--- a/llamafile/tinyblas_cpu_sgemm.inc+++ b/llamafile/tinyblas_cpu_sgemm.inc@@ -85,7 +85,7 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constif (k % 32)return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {- tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, TC> tb{+ tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;