Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Synchronize with llama.cpp 2024-01-26
Justine Tunneyc008e436aef4
diff --git a/Makefile b/Makefileindex 69543a1a..72212ee2 100644--- a/Makefile+++ b/Makefile@@ -23,17 +23,20 @@ o/$(MODE)/: o/$(MODE)/llama.cpp o/$(MODE)/llamafile.PHONY: installinstall: llamafile/zipalign.1 \llama.cpp/main/main.1 \+ llama.cpp/imatrix/imatrix.1 \llama.cpp/quantize/quantize.1 \llama.cpp/perplexity/perplexity.1 \llama.cpp/llava/llava-quantize.1 \o/$(MODE)/llamafile/zipalign \o/$(MODE)/llama.cpp/main/main \+ o/$(MODE)/llama.cpp/imatrix/imatrix \o/$(MODE)/llama.cpp/quantize/quantize \o/$(MODE)/llama.cpp/perplexity/perplexity \o/$(MODE)/llama.cpp/llava/llava-quantizemkdir -p $(PREFIX)/bin$(INSTALL) o/$(MODE)/llamafile/zipalign $(PREFIX)/bin/zipalign$(INSTALL) o/$(MODE)/llama.cpp/main/main $(PREFIX)/bin/llamafile+ $(INSTALL) o/$(MODE)/llama.cpp/imatrix/imatrix $(PREFIX)/bin/llamafile-imatrix$(INSTALL) o/$(MODE)/llama.cpp/quantize/quantize $(PREFIX)/bin/llamafile-quantize$(INSTALL) build/llamafile-convert $(PREFIX)/bin/llamafile-convert$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity@@ -41,6 +44,7 @@ install: llamafile/zipalign.1 \mkdir -p $(PREFIX)/share/man/man1$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1$(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1+ $(INSTALL) -m 0644 llama.cpp/imatrix/imatrix.1 $(PREFIX)/share/man/man1/llamafile-imatrix.1$(INSTALL) -m 0644 llama.cpp/quantize/quantize.1 $(PREFIX)/share/man/man1/llamafile-quantize.1$(INSTALL) -m 0644 llama.cpp/perplexity/perplexity.1 $(PREFIX)/share/man/man1/llamafile-perplexity.1$(INSTALL) -m 0644 llama.cpp/llava/llava-quantize.1 $(PREFIX)/share/man/man1/llava-quantize.1diff --git a/README.md b/README.mdindex 053887fb..75246c1d 100644--- a/README.md+++ b/README.md@@ -38,7 +38,7 @@ chmod +x llava-v1.5-7b-q4.llamafile5. Run the llamafile. e.g.:```sh-./llava-v1.5-7b-q4.llamafile+./llava-v1.5-7b-q4.llamafile -ngl 9999```6. Your browser should open automatically and display a chat interface.@@ -174,19 +174,19 @@ try out llamafile with different kinds of LLMs.Here is an example for the Mistral command-line llamafile:```sh-./mistral-7b-instruct-v0.2.Q5_K_M.llamafile --temp 0.7 -p '[INST]Write a story about llamas[/INST]'+./mistral-7b-instruct-v0.2.Q5_K_M.llamafile -ngl 9999 --temp 0.7 -p '[INST]Write a story about llamas[/INST]'```And here is an example for WizardCoder-Python command-line llamafile:```sh-./wizardcoder-python-13b.llamafile --temp 0 -e -r '```\n' -p '```c\nvoid *memcpy_sse2(char *dst, const char *src, size_t size) {\n'+./wizardcoder-python-13b.llamafile -ngl 9999 --temp 0 -e -r '```\n' -p '```c\nvoid *memcpy_sse2(char *dst, const char *src, size_t size) {\n'```And here's an example for the LLaVA command-line llamafile:```sh-./llava-v1.5-7b-q4.llamafile --temp 0.2 --image lemurs.jpg -e -p '### User: What do you see?\n### Assistant:'+./llava-v1.5-7b-q4.llamafile -ngl 9999 --temp 0.2 --image lemurs.jpg -e -p '### User: What do you see?\n### Assistant:'```As before, macOS, Linux, and BSD users will need to use the "chmod"@@ -341,7 +341,7 @@ On Apple Silicon running MacOS, your Metal GPU should just work if theXcode Command Line Tools are installed.On Windows, GPU should just work so long as (1) you're using our release-binaries, and (2) you pass the `-ngl 35` flag. If you only have the+binaries, and (2) you pass the `-ngl 9999` flag. If you only have thegraphics card drivers installed, then llamafile will use tinyBLAS as itsmath kernel library, which goes slower for batch processing tasks, e.g.summarization. In order to get full performance, NVIDIA GPU owners need@@ -355,8 +355,8 @@ and running your llamafiles inside of WSL. Using WSL has the addedbenefit of letting you run llamafiles greater than 4GB on Windows.On Linux, Nvidia cuBLAS GPU support will be compiled on the fly if (1)-you have the `cc` compiler installed, (2) you pass the `-ngl 35` flag to-enable GPU, and (3) the CUDA developer toolkit is installed on your+you have the `cc` compiler installed, (2) you pass the `-ngl 9999` flag+to enable GPU, and (3) the CUDA developer toolkit is installed on yourmachine and the `nvcc` compiler is on your path.If you have both an AMD GPU *and* an NVIDIA GPU in your machine, then@@ -385,7 +385,7 @@ llama.cpp command line interface, utilizing WizardCoder-Python-13Bweights:```sh-llamafile \+llamafile -ngl 9999 \-m wizardcoder-python-13b-v1.0.Q8_0.gguf \--temp 0 -r '}\n' -r '```\n' \-e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'@@ -395,7 +395,7 @@ Here's a similar example that instead utilizes Mistral-7B-Instructweights for prose composition:```sh-llamafile \+llamafile -ngl 9999 \-m mistral-7b-instruct-v0.1.Q4_K_M.gguf \-p '[INST]Write a story about llamas[/INST]'```@@ -424,24 +424,24 @@ Here's an example of how you can use llamafile to summarize HTML URLs:-dump https://www.poetryfoundation.org/poems/48860/the-raven |sed 's/ */ /g'echo '[/INST]'-) | llamafile \+) | llamafile -ngl 9999 \-m mistral-7b-instruct-v0.2.Q5_K_M.gguf \-f /dev/stdin \-c 0 \--temp 0 \-n 500 \- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/null```Here's how you can use llamafile to describe a jpg/png/gif/bmp image:```sh-llamafile --temp 0 \+llamafile -ngl 9999 --temp 0 \--image ~/Pictures/lemurs.jpg \-m llava-v1.5-7b-Q4_K.gguf \--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \-e -p '### User: What do you see?\n### Assistant: ' \- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/null```It's possible to use BNF grammar to enforce the output is predictable@@ -451,13 +451,13 @@ standard output either `"yes\n"` or `"no\n"`. Another example is if youwanted to write a script to rename all your image files, you could say:```sh-llamafile --temp 0 \+llamafile -ngl 9999 --temp 0 \--image lemurs.jpg \-m llava-v1.5-7b-Q4_K.gguf \--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \--grammar 'root ::= [a-z]+ (" " [a-z]+)+' \-e -p '### User: What do you see?\n### Assistant: ' \- --silent-prompt 2>/dev/null |+ --no-display-prompt 2>/dev/null |sed -e's/ /_/g' -e's/$/.jpg/'a_baby_monkey_on_the_back_of_a_mother.jpg```@@ -467,7 +467,7 @@ example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp'srecently-added support for image inputs.```sh-llamafile \+llamafile -ngl 9999 \-m llava-v1.5-7b-Q8_0.gguf \--mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \--host 0.0.0.0@@ -497,6 +497,8 @@ llava-v1.5-7b-Q8_0.ggufllava-v1.5-7b-mmproj-Q8_0.gguf--host0.0.0.0+-ngl+9999...```diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex 79557dc5..619072ca 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -21,6 +21,7 @@ o/$(MODE)/llama.cpp/llama.cpp.a: $(LLAMA_CPP_OBJS)include llama.cpp/llava/BUILD.mkinclude llama.cpp/server/BUILD.mkinclude llama.cpp/main/BUILD.mk+include llama.cpp/imatrix/BUILD.mkinclude llama.cpp/quantize/BUILD.mkinclude llama.cpp/perplexity/BUILD.mkdiff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex 873a10e0..98ca1794 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- 4f56458d34cb13dcbf69aca650e9bf77d5497e6f- 2024-01-10+ a1d6df129bcd3d42cda38c09217d8d4ec4ea3bdd+ 2024-01-26LOCAL MODIFICATIONSdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 8026a8fe..75a3c180 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -147,9 +147,6 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {const std::string arg_prefix = "--";llama_sampling_params & sparams = params.sparams;- assert(FLAG_gpu == LLAMAFILE_GPU_ERROR);- FLAG_gpu = LLAMAFILE_GPU_AUTO;-for (int i = 1; i < argc; i++) {arg = argv[i];if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {@@ -182,6 +179,24 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {if (params.n_threads_batch <= 0) {params.n_threads_batch = std::thread::hardware_concurrency();}+ } else if (arg == "-td" || arg == "--threads-draft") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.n_threads_draft = std::stoi(argv[i]);+ if (params.n_threads_draft <= 0) {+ params.n_threads_draft = std::thread::hardware_concurrency();+ }+ } else if (arg == "-tbd" || arg == "--threads-batch-draft") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.n_threads_batch_draft = std::stoi(argv[i]);+ if (params.n_threads_batch_draft <= 0) {+ params.n_threads_batch_draft = std::thread::hardware_concurrency();+ }} else if (arg == "-p" || arg == "--prompt") {if (++i >= argc) {invalid_param = true;@@ -200,6 +215,23 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {params.prompt_cache_all = true;} else if (arg == "--prompt-cache-ro") {params.prompt_cache_ro = true;+ } else if (arg == "-bf" || arg == "--binary-file") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ std::ifstream file(argv[i], std::ios::binary);+ if (!file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ break;+ }+ // store the external file name in params+ params.prompt_file = argv[i];+ std::ostringstream ss;+ ss << file.rdbuf();+ params.prompt = ss.str();+ fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), argv[i]);} else if (arg == "-f" || arg == "--file") {if (++i >= argc) {invalid_param = true;@@ -607,8 +639,10 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {const std::regex regex{R"([,/]+)"};std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};std::vector<std::string> split_arg{it, {}};- GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES);-+ if (split_arg.size() >= LLAMA_MAX_DEVICES) {+ invalid_param = true;+ break;+ }for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) {if (i < split_arg.size()) {params.tensor_split[i] = std::stof(split_arg[i]);@@ -616,17 +650,14 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {params.tensor_split[i] = 0.0f;}}- } else if (arg == "--no-mul-mat-q" || arg == "-nommq") {- passed_gpu_flags = true;- params.mul_mat_q = false;} else if (arg == "--no-mmap") {params.use_mmap = false;} else if (arg == "--numa") {params.numa = true;} else if (arg == "--verbose-prompt") {params.verbose_prompt = true;- } else if (arg == "--silent-prompt") {- params.silent_prompt = true;+ } else if (arg == "--no-display-prompt" || arg == "--silent-prompt") {+ params.display_prompt = false;} else if (arg == "-r" || arg == "--reverse-prompt") {if (++i >= argc) {invalid_param = true;@@ -643,6 +674,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {if (params.logdir.back() != DIRECTORY_SEPARATOR) {params.logdir += DIRECTORY_SEPARATOR;}+ } else if (arg == "--save-all-logits" || arg == "--kl-divergence-base") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.logits_file = argv[i];} else if (arg == "--perplexity" || arg == "--all-logits") {params.logits_all = true;} else if (arg == "--ppl-stride") {@@ -651,6 +688,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {break;}params.ppl_stride = std::stoi(argv[i]);+ } else if (arg == "-ptc" || arg == "--print-token-count") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.n_print = std::stoi(argv[i]);} else if (arg == "--ppl-output-type") {if (++i >= argc) {invalid_param = true;@@ -665,6 +708,24 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {break;}params.hellaswag_tasks = std::stoi(argv[i]);+ } else if (arg == "--winogrande") {+ params.winogrande = true;+ } else if (arg == "--winogrande-tasks") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.winogrande_tasks = std::stoi(argv[i]);+ } else if (arg == "--multiple-choice") {+ params.multiple_choice = true;+ } else if (arg == "--multiple-choice-tasks") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ params.multiple_choice_tasks = std::stoi(argv[i]);+ } else if (arg == "--kl-divergence") {+ params.kl_divergence = true;} else if (arg == "--ignore-eos") {params.ignore_eos = true;} else if (arg == "--no-penalize-nl") {@@ -691,6 +752,10 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {} else if (arg == "-h" || arg == "--help") {return false;+ } else if (arg == "--version") {+ fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);+ fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);+ exit(0);} else if (arg == "--random-prompt") {params.random_prompt = true;} else if (arg == "--in-prefix-bos") {@@ -819,6 +884,7 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {}params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);+ // tinylog(__func__, ": parsed configuration flags\n", NULL);return true;}@@ -831,7 +897,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf("\n");printf("options:\n");printf(" -h, --help show this help message and exit\n");- printf(" --version show version and build info\n");+ printf(" --version show version and build info\n");printf(" -i, --interactive run in interactive mode\n");printf(" --interactive-first run in interactive mode and wait for input right away\n");printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n");@@ -845,6 +911,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" -t N, --threads N number of threads to use during generation (default: %d)\n", params.n_threads);printf(" -tb N, --threads-batch N\n");printf(" number of threads to use during batch and prompt processing (default: same as --threads)\n");+ printf(" -td N, --threads-draft N");+ printf(" number of threads to use during generation (default: same as --threads)");+ printf(" -tbd N, --threads-batch-draft N\n");+ printf(" number of threads to use during batch and prompt processing (default: same as --threads-draft)\n");printf(" -p PROMPT, --prompt PROMPT\n");printf(" prompt to start generation with (default: empty)\n");printf(" -e, --escape process prompt escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n");@@ -858,6 +928,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --in-suffix STRING string to suffix after user inputs with (default: empty)\n");printf(" -f FNAME, --file FNAME\n");printf(" prompt file to start generation.\n");+ printf(" -bf FNAME, --binary-file FNAME\n");+ printf(" binary file containing multiple choice tasks.\n");printf(" -n N, --n-predict N number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)\n", params.n_predict);printf(" -c N, --ctx-size N size of the prompt context (default: %d, 0 = loaded from model)\n", params.n_ctx);printf(" -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);@@ -904,6 +976,11 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --logits-all return logits for all tokens in the batch (default: disabled)\n");printf(" --hellaswag compute HellaSwag score over random tasks from datafile supplied with -f\n");printf(" --hellaswag-tasks N number of tasks to use when computing the HellaSwag score (default: %zu)\n", params.hellaswag_tasks);+ printf(" --winogrande compute Winogrande score over random tasks from datafile supplied with -f\n");+ printf(" --winogrande-tasks N number of tasks to use when computing the Winogrande score (default: %zu)\n", params.winogrande_tasks);+ printf(" --multiple-choice compute multiple choice score over random tasks from datafile supplied with -f\n");+ printf(" --multiple-choice-tasks N number of tasks to use when computing the multiple choice score (default: %zu)\n", params.winogrande_tasks);+ printf(" --kl-divergence computes KL-divergence to logits provided via --kl-divergence-base");printf(" --keep N number of tokens to keep from the initial prompt (default: %d, -1 = all)\n", params.n_keep);printf(" --draft N number of tokens to draft for speculative decoding (default: %d)\n", params.n_draft);printf(" --chunks N max number of chunks to process (default: %d, -1 = all)\n", params.n_chunks);@@ -928,20 +1005,22 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" number of layers to store in VRAM\n");printf(" -ngld N, --n-gpu-layers-draft N\n");printf(" number of layers to store in VRAM for the draft model\n");- printf(" -ts SPLIT --tensor-split SPLIT\n");- printf(" how to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1\n");- printf(" -mg i, --main-gpu i the GPU to use for scratch and small tensors\n");-#ifdef GGML_USE_CUBLAS- printf(" -nommq, --no-mul-mat-q\n");- printf(" use " GGML_CUBLAS_NAME " instead of custom mul_mat_q " GGML_CUDA_NAME " kernels.\n");- printf(" Not recommended since this is both slower and uses more VRAM.\n");-#endif // GGML_USE_CUBLAS+ printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");+ printf(" how to split the model across multiple GPUs, one of:\n");+ printf(" - none: use one GPU only\n");+ printf(" - layer (default): split layers and KV across GPUs\n");+ printf(" - row: split rows across GPUs\n");+ printf(" -ts SPLIT, --tensor-split SPLIT\n");+ printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");+ printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");+ printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu);#endif+ printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false");+ printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false");printf(" -gan N, --grp-attn-n N\n");printf(" group-attention factor (default: %d)\n", params.grp_attn_n);printf(" -gaw N, --grp-attn-w N\n");printf(" group-attention width (default: %.1f)\n", (double)params.grp_attn_w);- printf(" --verbose-prompt print prompt before generation\n");printf(" -dkvc, --dump-kv-cache\n");printf(" verbose print of the KV cache\n");printf(" -nkvo, --no-kv-offload\n");@@ -963,6 +1042,8 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {printf(" --override-kv KEY=TYPE:VALUE\n");printf(" advanced option to override model metadata by key. may be specified multiple times.\n");printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");+ printf(" -ptc N, --print-token-count N\n");+ printf(" print token count every N tokens (default: %d)\n", params.n_print);printf("\n");#ifndef LOG_DISABLE_LOGSlog_print_usage();@@ -1052,6 +1133,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &mparams.n_gpu_layers = params.n_gpu_layers;}mparams.main_gpu = params.main_gpu;+ mparams.split_mode = params.split_mode;mparams.tensor_split = params.tensor_split;mparams.use_mmap = params.use_mmap;mparams.use_mlock = params.use_mlock;@@ -1066,6 +1148,9 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &}static ggml_type kv_cache_type_from_str(const std::string & s) {+ if (s == "f32") {+ return GGML_TYPE_F32;+ }if (s == "f16") {return GGML_TYPE_F16;}@@ -1279,7 +1364,74 @@ bool llama_should_add_bos_token(const llama_model * model) {// returns true if successful, false otherwisebool create_directory_with_parents(const std::string & path) {- return !makedirs(path.c_str(), 0755);+#ifdef _WIN32+ std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;+ std::wstring wpath = converter.from_bytes(path);++ // if the path already exists, check whether it's a directory+ const DWORD attributes = GetFileAttributesW(wpath.c_str());+ if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {+ return true;+ }++ size_t pos_slash = 0;++ // process path from front to back, procedurally creating directories+ while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {+ const std::wstring subpath = wpath.substr(0, pos_slash);+ const wchar_t * test = subpath.c_str();++ const bool success = CreateDirectoryW(test, NULL);+ if (!success) {+ const DWORD error = GetLastError();++ // if the path already exists, ensure that it's a directory+ if (error == ERROR_ALREADY_EXISTS) {+ const DWORD attributes = GetFileAttributesW(subpath.c_str());+ if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {+ return false;+ }+ } else {+ return false;+ }+ }++ pos_slash += 1;+ }++ return true;+#else+ // if the path already exists, check whether it's a directory+ struct stat info;+ if (stat(path.c_str(), &info) == 0) {+ return S_ISDIR(info.st_mode);+ }++ size_t pos_slash = 1; // skip leading slashes for directory creation++ // process path from front to back, procedurally creating directories+ while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {+ const std::string subpath = path.substr(0, pos_slash);+ struct stat info;++ // if the path already exists, ensure that it's a directory+ if (stat(subpath.c_str(), &info) == 0) {+ if (!S_ISDIR(info.st_mode)) {+ return false;+ }+ } else {+ // create parent directories+ const int ret = mkdir(subpath.c_str(), 0755);+ if (ret != 0) {+ return false;+ }+ }++ pos_slash += 1;+ }++ return true;+#endif // _WIN32}void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector<float> & data) {@@ -1510,6 +1662,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p);fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false");+ fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false");}//diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 493986e0..12803e1e 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -58,7 +58,9 @@ struct gpt_params {uint32_t seed = -1; // RNG seedint32_t n_threads = pick_default_thread_count();+ int32_t n_threads_draft = -1;int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)+ int32_t n_threads_batch_draft = -1;int32_t n_predict = -1; // new tokens to predictint32_t n_ctx = 512; // context sizeint32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)@@ -71,11 +73,13 @@ struct gpt_params {float p_split = 0.1f; // speculative decoding split probabilityint32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)+ llama_split_mode split_mode = LLAMA_SPLIT_LAYER; // how to split the model across GPUsint32_t main_gpu = 0; // the GPU that is used for scratch and small tensorsfloat tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUsint32_t n_beams = 0; // if non-zero then use beam search of given width.int32_t grp_attn_n = 1; // group-attention factorint32_t grp_attn_w = 512; // group-attention width+ int32_t n_print = -1; // print token count every n tokens (-1 = disabled)float rope_freq_base = 0.0f; // RoPE base frequencyfloat rope_freq_scale = 0.0f; // RoPE frequency scaling factorfloat yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor@@ -99,6 +103,7 @@ struct gpt_params {std::string input_suffix = ""; // string to suffix user inputs withstd::vector<std::string> antiprompt; // string upon seeing which more user input is promptedstd::string logdir = ""; // directory in which to save YAML log files+ std::string logits_file = ""; // file for saving *all* logitsstd::vector<llama_model_kv_override> kv_overrides;@@ -113,6 +118,14 @@ struct gpt_params {bool hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in promptsize_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag score+ bool winogrande = false; // compute Winogrande score over random tasks from datafile supplied in prompt+ size_t winogrande_tasks= 0; // number of tasks to use when computing the Winogrande score. If 0, all tasks will be computed++ bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in prompt+ size_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed++ bool kl_divergence = false; // compute KL-divergence+bool mul_mat_q = true; // if true, use mul_mat_q kernels instead of cuBLASbool random_prompt = false; // do not randomize prompt if none providedbool use_color = false; // use color to distinguish generations and inputs@@ -136,7 +149,7 @@ struct gpt_params {bool use_mlock = false; // use mlock to keep model in memorybool numa = false; // attempt optimizations that help on some NUMA systemsbool verbose_prompt = false; // print prompt tokens before generation- bool silent_prompt = false; // don't print prompt to stdout+ bool display_prompt = true; // print prompt before generationbool infill = false; // use infill modebool dump_kv_cache = false; // dump the KV cache contents for debugging purposesbool no_kv_offload = false; // disable KV offloading@@ -256,4 +269,3 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80);// Dump the KV cache view showing individual sequences in each cell (long output).void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40);-diff --git a/llama.cpp/ggml-alloc.c b/llama.cpp/ggml-alloc.cindex de4c982e..b036d975 100644--- a/llama.cpp/ggml-alloc.c+++ b/llama.cpp/ggml-alloc.c@@ -104,8 +104,6 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {}}- AT_PRINTF("block %d\n", best_fit_block);-if (best_fit_block == -1) {// the last block is our last resortstruct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1];@@ -113,12 +111,13 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {if (block->size >= size) {best_fit_block = alloc->n_free_blocks - 1;} else {- fprintf(stderr, "%s: not enough space in the buffer (needed %zu, largest block available %zu)\n",- __func__, size, max_avail);+ fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, largest block available %zu)\n",+ __func__, tensor->name, size, max_avail);GGML_ASSERT(!"not enough space in the buffer");return;}}+struct free_block * block = &alloc->free_blocks[best_fit_block];void * addr = block->addr;block->addr = (char*)block->addr + size;@@ -131,6 +130,8 @@ void ggml_tallocr_alloc(ggml_tallocr_t alloc, struct ggml_tensor * tensor) {}}+ AT_PRINTF("block %d, addr %p\n", best_fit_block, addr);+tensor->data = addr;tensor->buffer = alloc->buffer;if (!alloc->measure) {@@ -231,6 +232,7 @@ void ggml_tallocr_reset(ggml_tallocr_t alloc) {alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows} else {alloc->free_blocks[0].size = ggml_backend_buffer_get_size(alloc->buffer) - align_offset;+ ggml_backend_buffer_reset(alloc->buffer);}}@@ -265,9 +267,9 @@ ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment) {return alloc;}-ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) {+ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft) {// create a backend buffer to get the correct tensor allocation sizes- ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, 1);+ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, 1);// TODO: move alloc initialization to a common ggml_tallocr_new_impl functionggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);@@ -277,13 +279,22 @@ ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backereturn alloc;}-ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) {- ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, size);+ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend) {+ return ggml_tallocr_new_measure_from_buft(ggml_backend_get_default_buffer_type(backend));+}++ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size) {+ // create a backend buffer to get the correct tensor allocation sizes+ ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(buffer);alloc->buffer_owned = true;return alloc;}+ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size) {+ return ggml_tallocr_new_from_buft(ggml_backend_get_default_buffer_type(backend), size);+}+ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer) {ggml_tallocr_t alloc = (ggml_tallocr_t)malloc(sizeof(struct ggml_tallocr));@@ -326,7 +337,9 @@ bool ggml_tallocr_is_measure(ggml_tallocr_t alloc) {}size_t ggml_tallocr_max_size(ggml_tallocr_t alloc) {- return alloc->max_size;+ // FIXME: changes in the tensor sizes compared to the measure graph may cause allocations to fail+ // to avoid this, we add a 10% margin to the buffer size+ return alloc->max_size + alloc->max_size/10;}// graph allocator@@ -781,10 +794,21 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conteif (nbytes == 0) {// all the tensors in the context are already allocated+#ifndef NDEBUG+ fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__);+#endifreturn NULL;}ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, nbytes);+ if (buffer == NULL) {+ // failed to allocate buffer+#ifndef NDEBUG+ fprintf(stderr, "%s: failed to allocate buffer\n", __func__);+#endif+ return NULL;+ }+ggml_tallocr_t tallocr = ggml_tallocr_new_from_buffer(buffer);for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {diff --git a/llama.cpp/ggml-alloc.h b/llama.cpp/ggml-alloc.hindex 64a41246..4e599752 100644--- a/llama.cpp/ggml-alloc.h+++ b/llama.cpp/ggml-alloc.h@@ -52,8 +52,10 @@ typedef struct ggml_tallocr * ggml_tallocr_t;GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment);GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment);-GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer);+GGML_API ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size);GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer+GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer);+GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft);GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend);GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc);diff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hindex b5c0fc90..3329168f 100644--- a/llama.cpp/ggml-backend-impl.h+++ b/llama.cpp/ggml-backend-impl.h@@ -18,9 +18,10 @@ extern "C" {typedef void * ggml_backend_buffer_type_context_t;struct ggml_backend_buffer_type_i {+ const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft);ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment- size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding+ size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including paddingbool (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend// check if tensor data is in host memory// should be equivalent to supports_backend(buft, ggml_backend_cpu_init())@@ -36,16 +37,15 @@ extern "C" {typedef void * ggml_backend_buffer_context_t;struct ggml_backend_buffer_i {- void (*GGML_CALL free_buffer) (ggml_backend_buffer_t buffer);- //void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras- void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer);- void (*GGML_CALL init_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);- void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);- void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);- // (optional) copy tensor between different buffer-type, allow for single-copy tranfers- void (*GGML_CALL cpy_tensor_from)(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*GGML_CALL cpy_tensor_to) (ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value);+ const char * (*GGML_CALL get_name) (ggml_backend_buffer_t buffer);+ void (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);+ void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer);+ void (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ bool (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer+ void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value);+ void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras};struct ggml_backend_buffer {@@ -53,14 +53,17 @@ extern "C" {ggml_backend_buffer_type_t buft;ggml_backend_buffer_context_t context;size_t size;+ enum ggml_backend_buffer_usage usage;};- ggml_backend_buffer_t ggml_backend_buffer_init(+ GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(ggml_backend_buffer_type_t buft,struct ggml_backend_buffer_i iface,ggml_backend_buffer_context_t context,- size_t size) GGML_CALL;+ size_t size);+ // do not use directly, use ggml_backend_tensor_copy instead+ bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);//// Backend@@ -76,22 +79,20 @@ extern "C" {// buffer allocationggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);- // (optional) asynchroneous tensor data access+ // (optional) asynchronous tensor data accessvoid (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst);- // (optional) asynchroneous tensor copy- void (*GGML_CALL cpy_tensor_from_async)(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*GGML_CALL cpy_tensor_to_async) (ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);-+ // (optional) complete all pending operationsvoid (*GGML_CALL synchronize)(ggml_backend_t backend);// compute graph with a plan- ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);void (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);- // compute graph without a plan+ // compute graph without a plan (async)bool (*GGML_CALL graph_compute)(ggml_backend_t backend, struct ggml_cgraph * cgraph);// check if the backend supports an operation@@ -104,14 +105,13 @@ extern "C" {ggml_backend_context_t context;};-//// Backend registry//typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);- void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);+ GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);//// GGML Backend API@@ -121,10 +121,13 @@ extern "C" {//struct ggml_backend_api {+ bool *FLAG_log_disable;void (*GGML_CALL exit)(int);void (*GGML_CALL free)(void *);void *(*GGML_CALL malloc)(size_t);+ void (*GGML_CALL ggml_backend_register)(const char *, ggml_backend_init_fn, ggml_backend_buffer_type_t, void *);ggml_backend_buffer_t (*GGML_CALL ggml_backend_buffer_init)(ggml_backend_buffer_type_t, struct ggml_backend_buffer_i, ggml_backend_buffer_context_t, size_t);+ bool (*GGML_CALL ggml_backend_buffer_is_host)(ggml_backend_buffer_t);ggml_backend_buffer_t (*GGML_CALL ggml_backend_cpu_buffer_from_ptr)(void *, size_t);ggml_backend_buffer_type_t (*GGML_CALL ggml_backend_cpu_buffer_type)(void);size_t (*GGML_CALL ggml_backend_buft_get_alloc_size)(ggml_backend_buffer_type_t, struct ggml_tensor *);diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex ac89c1dc..0c9d0c03 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -2,11 +2,6 @@// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#include "ggml-backend-impl.h"-#include "ggml-alloc.h"-#include "llama.cpp/ggml-metal.h"-#include "ggml-impl.h"-#include "llama.cpp/ggml-cuda.h"-#include "libc/thread/tls.h"#include <assert.h>#include <limits.h>@@ -16,12 +11,22 @@#include <string.h>#include <stdlib.h>+#include "ggml-alloc.h"+#include "ggml-cuda.h"+#include "ggml-impl.h"+#include "ggml-metal.h"+#include "llamafile/log.h"+#define MAX(a, b) ((a) > (b) ? (a) : (b))// backend buffer type+const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {+ return buft->iface.get_name(buft);+}+GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {return buft->iface.alloc_buffer(buft, size);}@@ -33,7 +38,9 @@ size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {GGML_CALL size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) {// get_alloc_size is optional, defaults to ggml_nbytesif (buft->iface.get_alloc_size) {- return buft->iface.get_alloc_size(buft, tensor);+ size_t size = buft->iface.get_alloc_size(buft, tensor);+ assert(size >= ggml_nbytes(tensor));+ return size;}return ggml_nbytes(tensor);}@@ -65,11 +72,16 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(/* .buft = */ buft,/* .context = */ context,/* .size = */ size,+ /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY};return buffer;}+const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name(buffer);+}+void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {if (buffer == NULL) {return;@@ -101,25 +113,43 @@ GGML_CALL void ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, str}size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer) {- return ggml_backend_buft_get_alignment(ggml_backend_buffer_type(buffer));+ return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));}size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {- return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type(buffer), tensor);+ return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);}void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {buffer->iface.clear(buffer, value);}-bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {- return ggml_backend_buft_is_host(ggml_backend_buffer_type(buffer));+GGML_CALL bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {+ return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));+}++void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {+ buffer->usage = usage;}-ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer) {+ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {return buffer->buft;}+void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {+ if (buffer->iface.reset) {+ buffer->iface.reset(buffer);+ }+}++bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {+ ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;+ if (dst_buf->iface.cpy_tensor) {+ return src->buffer->iface.cpy_tensor(dst_buf, src, dst);+ }+ return false;+}+// backendconst char * ggml_backend_name(ggml_backend_t backend) {@@ -153,30 +183,42 @@ void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor *GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- backend->iface.set_tensor_async(backend, tensor, data, offset, size);+ if (backend->iface.set_tensor_async == NULL) {+ ggml_backend_tensor_set(tensor, data, offset, size);+ } else {+ backend->iface.set_tensor_async(backend, tensor, data, offset, size);+ }}void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- backend->iface.get_tensor_async(backend, tensor, data, offset, size);+ if (backend->iface.get_tensor_async == NULL) {+ ggml_backend_tensor_get(tensor, data, offset, size);+ } else {+ backend->iface.get_tensor_async(backend, tensor, data, offset, size);+ }}GGML_CALL void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;+GGML_ASSERT(tensor->data != NULL && "tensor not allocated");- GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");+ GGML_ASSERT(buf != NULL && "tensor buffer not set");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- tensor->buffer->iface.set_tensor(tensor->buffer, tensor, data, offset, size);+ tensor->buffer->iface.set_tensor(buf, tensor, data, offset, size);}GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;+GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(tensor->buffer != NULL && "tensor buffer not set");GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- tensor->buffer->iface.get_tensor(tensor->buffer, tensor, data, offset, size);+ tensor->buffer->iface.get_tensor(buf, tensor, data, offset, size);}void ggml_backend_synchronize(ggml_backend_t backend) {@@ -197,19 +239,10 @@ void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plavoid ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {backend->iface.graph_plan_compute(backend, plan);-- // TODO: optional sync- ggml_backend_synchronize(backend);}bool ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {- if (!backend->iface.graph_compute(backend, cgraph)) {- return false;- }-- // TODO: optional sync- ggml_backend_synchronize(backend);- return true;+ return backend->iface.graph_compute(backend, cgraph);}bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {@@ -234,28 +267,20 @@ static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml}void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {- //printf("src: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", src->name, (int)src->ne[0], (int)src->ne[1], (int)src->ne[2], (int)src->ne[3], (int)src->nb[0], (int)src->nb[1], (int)src->nb[2], (int)src->nb[3]);- //printf("dst: %s ne: [%d %d %d %d] nb: [%d %d %d %d]\n", dst->name, (int)dst->ne[0], (int)dst->ne[1], (int)dst->ne[2], (int)dst->ne[3], (int)dst->nb[0], (int)dst->nb[1], (int)dst->nb[2], (int)dst->nb[3]);GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");- // fprintf(stderr, "cpy tensor %s from %s to %s (%lu bytes)\n", src->name, ggml_backend_name(src->backend), ggml_backend_name(dst->backend), ggml_nbytes(src));-if (src == dst) {return;}- // TODO: allow backends to support copy to/from same backend-- if (dst->buffer->iface.cpy_tensor_from != NULL) {- dst->buffer->iface.cpy_tensor_from(dst->buffer, src, dst);- } else if (src->buffer->iface.cpy_tensor_to != NULL) {- src->buffer->iface.cpy_tensor_to(src->buffer, src, dst);- } else {- // shouldn't be hit when copying from/to CPU- #ifndef NDEBUG- fprintf(stderr, "ggml_backend_tensor_copy: neither cpy_tensor_from nor cpy_tensor_to "- "are implemented for %s and %s, falling back to get/set\n", src->name, dst->name);- #endif+ if (ggml_backend_buffer_is_host(src->buffer)) {+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));+ } else if (ggml_backend_buffer_is_host(dst->buffer)) {+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));+ } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {+#ifndef NDEBUG+ fprintf(stderr, "%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));+#endifsize_t nbytes = ggml_nbytes(src);void * data = malloc(nbytes);ggml_backend_tensor_get(src, data, 0, nbytes);@@ -264,6 +289,31 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst}}+void ggml_backend_tensor_copy_async(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {+ GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");++ if (src == dst) {+ return;+ }++ if (ggml_backend_buft_supports_backend(src->buffer->buft, backend) && ggml_backend_buft_supports_backend(dst->buffer->buft, backend)) {+ if (backend->iface.cpy_tensor_async != NULL) {+ if (backend->iface.cpy_tensor_async(backend, src, dst)) {+ return;+ }+ }+ }++ size_t nbytes = ggml_nbytes(src);+ if (ggml_backend_buffer_is_host(src->buffer)) {+ ggml_backend_tensor_set_async(backend, dst, src->data, 0, nbytes);+ }+ else {+ ggml_backend_tensor_copy(src, dst);+ }+}++// backend registry#define GGML_MAX_BACKENDS_REG 16@@ -280,13 +330,6 @@ static size_t ggml_backend_registry_count = 0;GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data);-GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {- return ggml_backend_metal_init();-- GGML_UNUSED(params);- GGML_UNUSED(user_data);-}-GGML_CALL static void ggml_backend_registry_init(void) {static bool initialized = false;@@ -298,18 +341,16 @@ GGML_CALL static void ggml_backend_registry_init(void) {ggml_backend_register("CPU", ggml_backend_reg_cpu_init, ggml_backend_cpu_buffer_type(), NULL);- if (ggml_metal_supported()) {- ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);- }-// add forward decls here to avoid including the backend headers- extern int ggml_backend_cuda_reg_devices(void);- if (!ggml_metal_supported() && ggml_cublas_loaded()) {- ggml_backend_cuda_reg_devices();- }+ extern GGML_CALL void ggml_backend_cuda_reg_devices(void);+ ggml_backend_cuda_reg_devices();++ extern GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data);+ extern GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);+ ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);}-void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {+GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG);size_t id = ggml_backend_registry_count;@@ -324,7 +365,7 @@ void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggmlsnprintf(ggml_backend_registry[id].name, sizeof(ggml_backend_registry[id].name), "%s", name);#ifndef NDEBUG- fprintf(stderr, "%s: registered backend %s\n", __func__, name);+ tinylogf("%s: registered backend %s\n", __func__, name);#endifggml_backend_registry_count++;@@ -404,6 +445,12 @@ ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size) {// backend CPU+GGML_CALL static const char * ggml_backend_cpu_buffer_name(ggml_backend_buffer_t buffer) {+ return "CPU";++ GGML_UNUSED(buffer);+}+GGML_CALL static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {return (void *)buffer->context;}@@ -424,14 +471,12 @@ GGML_CALL static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t bGGML_UNUSED(buffer);}-GGML_CALL static void ggml_backend_cpu_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {- ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));-- GGML_UNUSED(buffer);-}--GGML_CALL static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {- ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));+GGML_CALL static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {+ if (ggml_backend_buffer_is_host(src->buffer)) {+ memcpy(dst->data, src->data, ggml_nbytes(src));+ return true;+ }+ return false;GGML_UNUSED(buffer);}@@ -441,30 +486,38 @@ GGML_CALL static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer}static struct ggml_backend_buffer_i cpu_backend_buffer_i = {+ /* .get_name = */ ggml_backend_cpu_buffer_name,/* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,/* .get_base = */ ggml_backend_cpu_buffer_get_base,/* .init_tensor = */ NULL, // no initialization required/* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,/* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,- /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,- /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,+ /* .reset = */ NULL,};// for buffers from ptr, free is not calledstatic struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {+ /* .get_name = */ ggml_backend_cpu_buffer_name,/* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed/* .get_base = */ ggml_backend_cpu_buffer_get_base,/* .init_tensor = */ NULL, // no initialization required/* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,/* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,- /* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,- /* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,+ /* .reset = */ NULL,};static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512+GGML_CALL static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {+ return "CPU";++ GGML_UNUSED(buft);+}+GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {size += TENSOR_ALIGNMENT; // malloc may return an address that is not alignedvoid * data = malloc(size); // TODO: maybe use GGML_ALIGNED_MALLOC?@@ -495,6 +548,7 @@ GGML_CALL static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_tGGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {/* .iface = */ {+ /* .get_name = */ ggml_backend_cpu_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes@@ -513,11 +567,23 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {#include <hbwmalloc.h>-static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {+GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {+ return "CPU_HBM";++ GGML_UNUSED(buft);+}++GGML_CALL static const char * ggml_backend_cpu_hbm_buffer_get_name(ggml_backend_buffer_t buf) {+ return "CPU_HBM";++ GGML_UNUSED(buf);+}++GGML_CALL static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {hbw_free(buffer->context);}-static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+GGML_CALL static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {//void * ptr = hbw_malloc(size);void * ptr;int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);@@ -526,17 +592,18 @@ static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_return NULL;}- // FIXME: this is a hack to avoid having to implement a new buffer typeggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);buffer->buft = buft;+ buffer->iface.get_name = ggml_backend_cpu_hbm_buffer_get_name;buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer;return buffer;}-ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type() {+ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {/* .iface = */ {+ /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes@@ -580,7 +647,7 @@ struct ggml_backend_plan_cpu {struct ggml_cgraph cgraph;};-GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) {struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;struct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu));@@ -631,6 +698,8 @@ GGML_CALL static bool ggml_backend_cpu_graph_compute(ggml_backend_t backend, strGGML_CALL static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {switch (op->op) {+ case GGML_OP_CPY:+ return op->type != GGML_TYPE_IQ2_XXS && op->type != GGML_TYPE_IQ2_XS; // missing type_traits.from_floatcase GGML_OP_MUL_MAT:return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type;default:@@ -646,8 +715,7 @@ static struct ggml_backend_i cpu_backend_i = {/* .get_default_buffer_type = */ ggml_backend_cpu_get_default_buffer_type,/* .set_tensor_async = */ NULL,/* .get_tensor_async = */ NULL,- /* .cpy_tensor_from_async = */ NULL,- /* .cpy_tensor_to_async = */ NULL,+ /* .cpy_tensor_async = */ NULL,/* .synchronize = */ NULL,/* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create,/* .graph_plan_free = */ ggml_backend_cpu_graph_plan_free,@@ -673,7 +741,7 @@ ggml_backend_t ggml_backend_cpu_init(void) {}GGML_CALL bool ggml_backend_is_cpu(ggml_backend_t backend) {- return backend->iface.get_name == ggml_backend_cpu_name;+ return backend && backend->iface.get_name == ggml_backend_cpu_name;}void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {@@ -697,7 +765,7 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, v// scheduler-#define GGML_MAX_BACKENDS 4+#define GGML_MAX_BACKENDS 16#define GGML_MAX_SPLITS 256#define GGML_MAX_SPLIT_INPUTS 16@@ -707,21 +775,29 @@ struct ggml_backend_sched_split {int i_end;struct ggml_tensor * inputs[GGML_MAX_SPLIT_INPUTS];int n_inputs;+ // graph view of this splitstruct ggml_cgraph graph;};struct ggml_backend_sched {+ bool is_reset; // true if the scheduler has been reset since the last graph split+int n_backends;ggml_backend_t backends[GGML_MAX_BACKENDS];+ ggml_backend_buffer_type_t bufts[GGML_MAX_BACKENDS];ggml_tallocr_t tallocs[GGML_MAX_BACKENDS];ggml_gallocr_t galloc;+ // hash keys of the nodes in the graphstruct ggml_hash_set hash_set;- ggml_tallocr_t * node_talloc; // [hash_set.size]- struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // [hash_set.size][GGML_MAX_BACKENDS]+ // hash values (arrays of [hash_set.size])+ ggml_tallocr_t * node_talloc; // tallocr assigned to each node (indirectly this is the backend)+ struct ggml_tensor * (* node_copies)[GGML_MAX_BACKENDS]; // copies of each node for each destination backend+ // copy of the graph with modified inputsstruct ggml_cgraph * graph;+struct ggml_backend_sched_split splits[GGML_MAX_SPLITS];int n_splits;@@ -734,6 +810,9 @@ struct ggml_backend_sched {__attribute__((aligned(GGML_MEM_ALIGN)))#endifchar context_buffer[GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS*sizeof(struct ggml_tensor) + sizeof(struct ggml_cgraph)];++ ggml_backend_sched_eval_callback callback_eval;+ void * callback_eval_user_data;};#define hash_id(node) ggml_hash_find_or_insert(sched->hash_set, node)@@ -762,14 +841,22 @@ static int sched_allocr_prio(ggml_backend_sched_t sched, ggml_tallocr_t allocr)return INT_MAX;}-static ggml_backend_t get_buffer_backend(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {+static ggml_tallocr_t sched_allocr_from_buffer(ggml_backend_sched_t sched, ggml_backend_buffer_t buffer) {if (buffer == NULL) {return NULL;}++ // check if this is already allocate in a allocr buffer (from user manual allocations)+ for (int i = 0; i < sched->n_backends; i++) {+ if (ggml_tallocr_get_buffer(sched->tallocs[i]) == buffer) {+ return sched->tallocs[i];+ }+ }+// find highest prio backend that supports the buffer typefor (int i = 0; i < sched->n_backends; i++) {if (ggml_backend_buft_supports_backend(buffer->buft, sched->backends[i])) {- return sched->backends[i];+ return sched->tallocs[i];}}GGML_ASSERT(false && "tensor buffer type not supported by any backend");@@ -779,7 +866,6 @@ static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_tallocif (allocr == NULL) {return NULL;}- // find highest prio backend that supports the buffer typefor (int i = 0; i < sched->n_backends; i++) {if (sched->tallocs[i] == allocr) {return sched->backends[i];@@ -789,7 +875,7 @@ static ggml_backend_t get_allocr_backend(ggml_backend_sched_t sched, ggml_talloc}#if 0-static char causes[GGML_DEFAULT_GRAPH_SIZE*8 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug, remove+static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS][128]; // debug only#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)#define GET_CAUSE(node) causes[hash_id(node)]#else@@ -798,45 +884,37 @@ static char causes[GGML_DEFAULT_GRAPH_SIZE*8 + GGML_MAX_SPLITS*GGML_MAX_SPLIT_IN#endif// returns the backend that should be used for the node based on the current locations-static ggml_backend_t sched_backend_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) {- // if the dst tensor is already allocated in a buffer, we must assume that it is critical to keep it there- // ie. kv cache updates- // note that this doesn't allow fallback to CPU. need to add output tensors to the splits to copy the data back to the original backend.+static ggml_tallocr_t sched_allocr_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * node) {+ // assign pre-allocated nodes to their backend// dst- ggml_backend_t cur_backend = get_buffer_backend(sched, node->buffer);- if (cur_backend != NULL) {+ ggml_tallocr_t cur_allocr = sched_allocr_from_buffer(sched, node->buffer);+ if (cur_allocr != NULL) {SET_CAUSE(node, "1.dst");- return cur_backend;+ return cur_allocr;}-// view_src- if (node->view_src != NULL && get_buffer_backend(sched, node->view_src->buffer) != NULL) {- SET_CAUSE(node, "1.vsrc");- return get_buffer_backend(sched, node->view_src->buffer);+ if (node->view_src != NULL) {+ cur_allocr = sched_allocr_from_buffer(sched, node->view_src->buffer);+ if (cur_allocr != NULL) {+ SET_CAUSE(node, "1.vsrc");+ return cur_allocr;+ }}-- // src- int cur_prio = INT_MAX;- size_t cur_size = 0;-+ // assign nodes that use weights to the backend of the weightsfor (int i = 0; i < GGML_MAX_SRC; i++) {const struct ggml_tensor * src = node->src[i];if (src == NULL) {break;}- ggml_backend_t src_backend = get_buffer_backend(sched, src->buffer);- if (src_backend != NULL) {- int src_prio = sched_backend_prio(sched, src_backend);- size_t src_size = ggml_nbytes(src);- if (src_prio < cur_prio && src_size >= cur_size) {- cur_prio = src_prio;- cur_size = src_size;- cur_backend = src_backend;- SET_CAUSE(node, "1.src%d", i);- }+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {+ ggml_tallocr_t src_allocr = sched_allocr_from_buffer(sched, src->buffer);+ // operations with weights are always run on the same backend as the weights+ SET_CAUSE(node, "1.wgt%d", i);+ return src_allocr;}}- return cur_backend;++ return NULL;}static char * fmt_size(size_t size) {@@ -869,7 +947,7 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra}ggml_tallocr_t node_allocr = node_allocr(node);ggml_backend_t node_backend = node_allocr ? get_allocr_backend(sched, node_allocr) : NULL; // FIXME:- fprintf(stderr, "node #%3d (%10.10s): %20.20s (%4.4s) [%4.4s %8.8s]:", i, ggml_op_name(node->op), node->name,+ fprintf(stderr, "node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, ggml_op_name(node->op), node->name,fmt_size(ggml_nbytes(node)), node_allocr ? ggml_backend_name(node_backend) : "NULL", GET_CAUSE(node));for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];@@ -878,7 +956,7 @@ static void sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgra}ggml_tallocr_t src_allocr = node_allocr(src);ggml_backend_t src_backend = src_allocr ? get_allocr_backend(sched, src_allocr) : NULL;- fprintf(stderr, " %20.20s (%4.4s) [%4.4s %8.8s]", src->name,+ fprintf(stderr, " %20.20s (%5.5s) [%5.5s %8.8s]", src->name,fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));}fprintf(stderr, "\n");@@ -894,15 +972,17 @@ static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, coreturn dup;}++//#define DEBUG_PASS1+//#define DEBUG_PASS2+//#define DEBUG_PASS3+//#define DEBUG_PASS4+// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend-// TODO: merge passesstatic void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {- // reset state- size_t hash_size = sched->hash_set.size;- memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size);- memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size);- memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size);+ // reset splitssched->n_splits = 0;+ sched->is_reset = false;struct ggml_init_params params = {/* .mem_size = */ sizeof(sched->context_buffer),@@ -910,26 +990,22 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g/* .no_alloc = */ true};- if (sched->ctx != NULL) {- ggml_free(sched->ctx);- }+ ggml_free(sched->ctx);sched->ctx = ggml_init(params);+ if (sched->ctx == NULL) {+ fprintf(stderr, "%s: failed to initialize context\n", __func__);+ GGML_ASSERT(false);+ }- // pass 1: assign backends to ops with allocated inputs+ // pass 1: assign backends to ops with pre-allocated inputsfor (int i = 0; i < graph->n_leafs; i++) {struct ggml_tensor * leaf = graph->leafs[i];if (node_allocr(leaf) != NULL) {// do not overwrite user assignmentscontinue;}- ggml_backend_t leaf_backend = get_buffer_backend(sched, leaf->buffer);- if (leaf_backend == NULL && leaf->view_src != NULL) {- leaf_backend = get_buffer_backend(sched, leaf->view_src->buffer);- }- if (leaf_backend != NULL) {- node_allocr(leaf) = ggml_backend_sched_get_tallocr(sched, leaf_backend);- }+ node_allocr(leaf) = sched_allocr_from_cur(sched, leaf);}for (int i = 0; i < graph->n_nodes; i++) {@@ -938,50 +1014,120 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g// do not overwrite user assignmentscontinue;}- ggml_backend_t node_backend = sched_backend_from_cur(sched, node);- if (node_backend != NULL) {- node_allocr(node) = ggml_backend_sched_get_tallocr(sched, node_backend);+ node_allocr(node) = sched_allocr_from_cur(sched, node);+ // src+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ if (node_allocr(src) == NULL) {+ node_allocr(src) = sched_allocr_from_cur(sched, src);+ }}}- //printf("PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#ifdef DEBUG_PASS1+ fprintf(stderr, "PASS 1 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#endif- // pass 2: assign backends to ops from current assignments- // TODO:- // - reuse sched_backend_from_cur- for (int i = 0; i < graph->n_nodes; i++) {- struct ggml_tensor * node = graph->nodes[i];- ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr == NULL) {- int cur_prio = INT_MAX;- size_t cur_size = 0;- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * src = node->src[j];- if (src == NULL) {- break;+ // pass 2: expand current backend assignments+ // assign the same backend to adjacent nodes+ // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)+ // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops++ // pass 2.1 expand gpu up+ {+ ggml_tallocr_t cur_allocr = NULL;+ for (int i = graph->n_nodes - 1; i >= 0; i--) {+ struct ggml_tensor * node = graph->nodes[i];+ if (ggml_is_view_op(node->op)) {+ continue;+ }+ ggml_tallocr_t node_allocr = node_allocr(node);+ if (node_allocr != NULL) {+ if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) {+ // skip cpu (lowest prio backend)+ cur_allocr = NULL;+ } else {+ cur_allocr = node_allocr;}- ggml_tallocr_t src_allocr = node_allocr(src);- if (src_allocr != NULL) {- int src_prio = sched_allocr_prio(sched, src_allocr);- size_t src_size = ggml_nbytes(src);- if (src_prio < cur_prio && src_size >= cur_size) {- cur_prio = src_prio;- cur_size = src_size;- node_allocr = src_allocr;- SET_CAUSE(node, "2.src%d", j);- }+ } else {+ node_allocr(node) = cur_allocr;+ SET_CAUSE(node, "2.1");+ }+ }+ }++ // pass 2.2 expand gpu down+ {+ ggml_tallocr_t cur_allocr = NULL;+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ if (ggml_is_view_op(node->op)) {+ continue;+ }+ ggml_tallocr_t node_allocr = node_allocr(node);+ if (node_allocr != NULL) {+ if (sched_allocr_prio(sched, node_allocr) == sched->n_backends - 1) {+ // skip cpu (lowest prio backend)+ cur_allocr = NULL;+ } else {+ cur_allocr = node_allocr;}+ } else {+ node_allocr(node) = cur_allocr;+ SET_CAUSE(node, "2.2");}+ }+ }++ // pass 2.3 expand rest up+ {+ ggml_tallocr_t cur_allocr = NULL;+ for (int i = graph->n_nodes - 1; i >= 0; i--) {+ struct ggml_tensor * node = graph->nodes[i];+ if (ggml_is_view_op(node->op)) {+ continue;+ }+ ggml_tallocr_t node_allocr = node_allocr(node);if (node_allocr != NULL) {- node_allocr(node) = node_allocr;+ cur_allocr = node_allocr;+ } else {+ node_allocr(node) = cur_allocr;+ SET_CAUSE(node, "2.3");}}}- //printf("PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);- // pass 3: assign backends to remaining src from dst (should only be leafs)+ // pass 2.4 expand rest down+ {+ ggml_tallocr_t cur_allocr = NULL;+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ if (ggml_is_view_op(node->op)) {+ continue;+ }+ ggml_tallocr_t node_allocr = node_allocr(node);+ if (node_allocr != NULL) {+ cur_allocr = node_allocr;+ } else {+ node_allocr(node) = cur_allocr;+ SET_CAUSE(node, "2.4");+ }+ }+ }+#ifdef DEBUG_PASS2+ fprintf(stderr, "PASS 2 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#endif++ // pass 3: assign backends to remaining src from dst and view_srcfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];- ggml_tallocr_t node_allocr = node_allocr(node);+ ggml_tallocr_t cur_allocr = node_allocr(node);+ if (node->view_src != NULL && cur_allocr == NULL) {+ cur_allocr = node_allocr(node) = node_allocr(node->view_src);+ SET_CAUSE(node, "3.vsrc");+ }for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {@@ -989,81 +1135,113 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * g}ggml_tallocr_t src_allocr = node_allocr(src);if (src_allocr == NULL) {- node_allocr(src) = node_allocr;+ if (src->view_src != NULL) {+ // views are always on the same backend as the source+ node_allocr(src) = node_allocr(src->view_src);+ SET_CAUSE(src, "3.vsrc");+ } else {+ node_allocr(src) = cur_allocr;+ SET_CAUSE(src, "3.cur");+ }}}}- //printf("PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#ifdef DEBUG_PASS3+ fprintf(stderr, "PASS 3 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#endif// pass 4: split graph, find tensors that need to be copied- // TODO:- // - when switching from a less preferred backend to a more preferred backend, check if it is possible to move the switch to an earlier point for the same cost- // find first backend- int cur_split = 0;- for (int i = 0; i < graph->n_nodes; i++) {- struct ggml_tensor * node = graph->nodes[i];- if (node->view_src == NULL) {- sched->splits[0].tallocr = node_allocr(node);- break;- }- }- sched->splits[0].i_start = 0;- sched->splits[0].n_inputs = 0;- memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK- ggml_tallocr_t cur_allocr = sched->splits[0].tallocr;- size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr);- for (int i = 0; i < graph->n_nodes; i++) {- struct ggml_tensor * node = graph->nodes[i];-- if (ggml_is_view_op(node->op)) {- continue;+ {+ int cur_split = 0;+ // find the backend of the first split, skipping view ops+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];+ if (!ggml_is_view_op(node->op)) {+ sched->splits[0].tallocr = node_allocr(node);+ break;+ }}+ sched->splits[0].i_start = 0;+ sched->splits[0].n_inputs = 0;+ memset(sched->splits[0].inputs, 0, sizeof(sched->splits[0].inputs)); //HACK+ ggml_tallocr_t cur_allocr = sched->splits[0].tallocr;+ size_t cur_backend_id = sched_allocr_prio(sched, cur_allocr);+ for (int i = 0; i < graph->n_nodes; i++) {+ struct ggml_tensor * node = graph->nodes[i];++ if (ggml_is_view_op(node->op)) {+ continue;+ }- ggml_tallocr_t node_allocr = node_allocr(node);+ ggml_tallocr_t node_allocr = node_allocr(node);- if (node_allocr != cur_allocr) {- sched->splits[cur_split].i_end = i;- cur_split++;- GGML_ASSERT(cur_split < GGML_MAX_SPLITS);- sched->splits[cur_split].tallocr = node_allocr;- sched->splits[cur_split].i_start = i;- sched->splits[cur_split].n_inputs = 0;- memset(sched->splits[cur_split].inputs, 0, sizeof(sched->splits[cur_split].inputs)); //HACK- cur_allocr = node_allocr;- cur_backend_id = sched_allocr_prio(sched, cur_allocr);- }+ GGML_ASSERT(node_allocr != NULL); // all nodes should be assigned by now- // find inputs that are not on the same backend- for (int j = 0; j < GGML_MAX_SRC; j++) {- struct ggml_tensor * src = node->src[j];- if (src == NULL) {- break;+ if (node_allocr != cur_allocr) {+ sched->splits[cur_split].i_end = i;+ cur_split++;+ GGML_ASSERT(cur_split < GGML_MAX_SPLITS);+ sched->splits[cur_split].tallocr = node_allocr;+ sched->splits[cur_split].i_start = i;+ sched->splits[cur_split].n_inputs = 0;+ cur_allocr = node_allocr;+ cur_backend_id = sched_allocr_prio(sched, cur_allocr);}- ggml_tallocr_t src_allocr = node_allocr(src);- if (src_allocr != node_allocr) {- int n_inputs = sched->splits[cur_split].n_inputs++;- GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);- sched->splits[cur_split].inputs[n_inputs] = (struct ggml_tensor *)src;-- // create copies- size_t id = hash_id(src);- if (sched->node_copies[id][cur_backend_id] == NULL) {- struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);- sched->node_copies[id][cur_backend_id] = tensor_copy;- node_allocr(tensor_copy) = cur_allocr;- ggml_backend_t backend = get_allocr_backend(sched, cur_allocr);- ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name);++ // find inputs that are not on the same backend+ for (int j = 0; j < GGML_MAX_SRC; j++) {+ struct ggml_tensor * src = node->src[j];+ if (src == NULL) {+ break;+ }+ ggml_tallocr_t src_allocr = node_allocr(src);+ GGML_ASSERT(src_allocr != NULL); // all inputs should be assigned by now+ if (src_allocr != node_allocr) {+ // create a copy of the input in the split's backend+ size_t id = hash_id(src);+ if (sched->node_copies[id][cur_backend_id] == NULL) {+ ggml_backend_t backend = get_allocr_backend(sched, cur_allocr);+ struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);+ ggml_format_name(tensor_copy, "%s#%s", ggml_backend_name(backend), src->name);++ sched->node_copies[id][cur_backend_id] = tensor_copy;+ node_allocr(tensor_copy) = cur_allocr;+ SET_CAUSE(tensor_copy, "4.cpy");++ int n_inputs = sched->splits[cur_split].n_inputs++;+ GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);+ sched->splits[cur_split].inputs[n_inputs] = src;+ }+ node->src[j] = sched->node_copies[id][cur_backend_id];++#if 0+ // check if the input is already in the split+ bool found = false;+ for (int k = 0; k < sched->splits[cur_split].n_inputs; k++) {+ if (sched->splits[cur_split].inputs[k] == src) {+ found = true;+ break;+ }+ }++ if (!found) {+ int n_inputs = sched->splits[cur_split].n_inputs++;+ //printf("split %d input %d: %s (%s)\n", cur_split, n_inputs, src->name, ggml_backend_name(get_allocr_backend(sched, src_allocr)));+ GGML_ASSERT(n_inputs < GGML_MAX_SPLIT_INPUTS);+ sched->splits[cur_split].inputs[n_inputs] = src;+ }+#endif}- node->src[j] = sched->node_copies[id][cur_backend_id];}}+ sched->splits[cur_split].i_end = graph->n_nodes;+ sched->n_splits = cur_split + 1;}- sched->splits[cur_split].i_end = graph->n_nodes;- sched->n_splits = cur_split + 1;-- //fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph); fflush(stdout);+#ifdef DEBUG_PASS4+ fprintf(stderr, "PASS 4 ASSIGNMENTS\n"); sched_print_assignments(sched, graph);+#endif-#if 1+#ifndef NDEBUG// sanity check: all sources should have the same backend as the nodefor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -1071,6 +1249,11 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gif (node_allocr == NULL) {fprintf(stderr, "!!!!!!! %s has no backend\n", node->name);}+ if (node->view_src != NULL && node_allocr != node_allocr(node->view_src)) {+ fprintf(stderr, "!!!!!!! %s has backend %s, view_src %s has backend %s\n",+ node->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL",+ node->view_src->name, node_allocr(node->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(node->view_src))) : "NULL");+ }for (int j = 0; j < GGML_MAX_SRC; j++) {struct ggml_tensor * src = node->src[j];if (src == NULL) {@@ -1082,8 +1265,14 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gnode->name, node_allocr ? ggml_backend_name(get_allocr_backend(sched, node_allocr)) : "NULL",j, src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL");}+ if (src->view_src != NULL && src_allocr != node_allocr(src->view_src)) {+ fprintf(stderr, "!!!!!!! [src] %s has backend %s, view_src %s has backend %s\n",+ src->name, src_allocr ? ggml_backend_name(get_allocr_backend(sched, src_allocr)) : "NULL",+ src->view_src->name, node_allocr(src->view_src) ? ggml_backend_name(get_allocr_backend(sched, node_allocr(src->view_src))) : "NULL");+ }}}+ fflush(stderr);#endif// create copies of the graph for each split@@ -1097,6 +1286,8 @@ static void sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * gfor (int j = 0; j < split->n_inputs; j++) {struct ggml_tensor * input = split->inputs[j];struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_allocr_prio(sched, split->tallocr)];+ // add a dependency to the input source so that it is not freed before the copy is done+ GGML_ASSERT(input_cpy->src[0] == NULL || input_cpy->src[0] == input);input_cpy->src[0] = input;graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;}@@ -1131,24 +1322,16 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {uint64_t copy_start_us = ggml_time_us();for (int j = 0; j < split->n_inputs; j++) {struct ggml_tensor * input = split->inputs[j];- struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][sched_backend_prio(sched, split_backend)];- if (input->buffer == NULL) {- if (input->view_src == NULL) {- fprintf(stderr, "input %s has no buffer and no view_src\n", input->name);- exit(1);- }- // FIXME: may need to use the sched buffer instead- ggml_backend_view_init(input->view_src->buffer, input);- }- if (input_cpy->buffer == NULL) {- fprintf(stderr, "input_cpy %s has no buffer\n", input_cpy->name);- exit(1);- }- //GGML_ASSERT(input->buffer->backend != input_cpy->buffer->backend);- //GGML_ASSERT(input_cpy->buffer->backend == split_backend);- ggml_backend_tensor_copy(input, input_cpy);+ struct ggml_tensor * input_cpy = sched->node_copies[hash_id(input)][split_backend_id];++ GGML_ASSERT(input->buffer != NULL);+ GGML_ASSERT(input_cpy->buffer != NULL);++ // TODO: avoid this copy if it was already copied in a previous split, and the input didn't change+ // this is important to avoid copying constants such as KQ_mask and inp_pos multiple times+ ggml_backend_tensor_copy_async(split_backend, input, input_cpy);}- // ggml_backend_synchronize(split_backend);+ //ggml_backend_synchronize(split_backend); // necessary to measure copy timeint64_t copy_end_us = ggml_time_us();copy_us[split_backend_id] += copy_end_us - copy_start_us;@@ -1158,9 +1341,38 @@ static void sched_compute_splits(ggml_backend_sched_t sched) {ggml_graph_dump_dot(split->graph, NULL, split_filename);#endif+uint64_t compute_start_us = ggml_time_us();- ggml_backend_graph_compute(split_backend, &split->graph);- // ggml_backend_synchronize(split_backend);+ if (!sched->callback_eval) {+ ggml_backend_graph_compute(split_backend, &split->graph);+ //ggml_backend_synchronize(split_backend); // necessary to measure compute time+ } else {+ // similar to ggml_backend_compare_graph_backend+ for (int j0 = 0; j0 < split->graph.n_nodes; j0++) {+ struct ggml_tensor * t = split->graph.nodes[j0];++ // check if the user needs data from this node+ bool need = sched->callback_eval(t, true, sched->callback_eval_user_data);++ int j1 = j0;++ // determine the range [j0, j1] of nodes that can be computed together+ while (!need && j1 < split->graph.n_nodes - 1) {+ t = split->graph.nodes[++j1];+ need = sched->callback_eval(t, true, sched->callback_eval_user_data);+ }++ struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);++ ggml_backend_graph_compute(split_backend, &gv);++ if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {+ break;+ }++ j0 = j1;+ }+ }uint64_t compute_end_us = ggml_time_us();compute_us[split_backend_id] += compute_end_us - compute_start_us;}@@ -1180,26 +1392,41 @@ static void sched_reset(ggml_backend_sched_t sched) {for (int i = 0; i < sched->n_backends; i++) {ggml_tallocr_reset(sched->tallocs[i]);}+ // reset state for the next run+ size_t hash_size = sched->hash_set.size;+ memset(sched->hash_set.keys, 0, sizeof(sched->hash_set.keys[0]) * hash_size);+ memset(sched->node_talloc, 0, sizeof(sched->node_talloc[0]) * hash_size);+ memset(sched->node_copies, 0, sizeof(sched->node_copies[0]) * hash_size);++ sched->is_reset = true;}-ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends) {+ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size) {+ GGML_ASSERT(n_backends > 0);GGML_ASSERT(n_backends <= GGML_MAX_BACKENDS);- struct ggml_backend_sched * sched = malloc(sizeof(struct ggml_backend_sched));- memset(sched, 0, sizeof(struct ggml_backend_sched));+ struct ggml_backend_sched * sched = calloc(sizeof(struct ggml_backend_sched), 1);++ // initialize hash table+ sched->hash_set = ggml_hash_set_new(graph_size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);+ sched->node_talloc = calloc(sizeof(sched->node_talloc[0]) * sched->hash_set.size, 1);+ sched->node_copies = calloc(sizeof(sched->node_copies[0]) * sched->hash_set.size, 1);sched->n_backends = n_backends;for (int i = 0; i < n_backends; i++) {sched->backends[i] = backends[i];+ sched->bufts[i] = bufts ? bufts[i] : ggml_backend_get_default_buffer_type(backends[i]);}sched->galloc = ggml_gallocr_new();// init measure allocs for each backendfor (int i = 0; i < n_backends; i++) {- sched->tallocs[i] = ggml_tallocr_new_measure_from_backend(backends[i]);+ sched->tallocs[i] = ggml_tallocr_new_measure_from_buft(sched->bufts[i]);}+ sched_reset(sched);+return sched;}@@ -1211,6 +1438,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {ggml_tallocr_free(sched->tallocs[i]);}ggml_gallocr_free(sched->galloc);+ ggml_free(sched->ctx);free(sched->hash_set.keys);free(sched->node_talloc);free(sched->node_copies);@@ -1218,12 +1446,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {}void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {- // initialize hash tables- size_t hash_size = measure_graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS;- sched->hash_set.size = hash_size;- sched->hash_set.keys = malloc(sizeof(sched->hash_set.keys[0]) * hash_size);- sched->node_talloc = malloc(sizeof(sched->node_talloc[0]) * hash_size);- sched->node_copies = malloc(sizeof(sched->node_copies[0]) * hash_size);+ GGML_ASSERT(ggml_tallocr_is_measure(sched->tallocs[0])); // can only be initialized oncesched_split_graph(sched, measure_graph);sched_alloc_splits(sched);@@ -1232,28 +1455,47 @@ void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgrfor (int i = 0; i < sched->n_backends; i++) {size_t size = ggml_tallocr_max_size(sched->tallocs[i]);ggml_tallocr_free(sched->tallocs[i]);- sched->tallocs[i] = ggml_tallocr_new_from_backend(sched->backends[i], size);+ sched->tallocs[i] = ggml_tallocr_new_from_buft(sched->bufts[i], size);}sched_reset(sched);}void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {- GGML_ASSERT(sched->hash_set.size >= graph->visited_hash_table.size + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);+ GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + GGML_MAX_SPLITS*GGML_MAX_SPLIT_INPUTS);++ if (!sched->is_reset) {+ sched_reset(sched);+ }sched_split_graph(sched, graph);sched_alloc_splits(sched);sched_compute_splits(sched);+}++void ggml_backend_sched_reset(ggml_backend_sched_t sched) {sched_reset(sched);}++void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {+ sched->callback_eval = callback;+ sched->callback_eval_user_data = user_data;+}++int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {+ return sched->n_splits;+}+ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend) {int backend_index = sched_backend_prio(sched, backend);+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);return sched->tallocs[backend_index];}ggml_backend_buffer_t ggml_backend_sched_get_buffer(ggml_backend_sched_t sched, ggml_backend_t backend) {int backend_index = sched_backend_prio(sched, backend);+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);return ggml_tallocr_get_buffer(sched->tallocs[backend_index]);}@@ -1263,10 +1505,19 @@ void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggmlnode_allocr(node) = sched->tallocs[backend_index];}+ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {+ ggml_tallocr_t allocr = node_allocr(node);+ if (allocr == NULL) {+ return NULL;+ }+ return get_allocr_backend(sched, allocr);+}+// utils+void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {GGML_ASSERT(tensor->buffer == NULL);- //GGML_ASSERT(tensor->data == NULL); // views of pre-allocted tensors may have the data set, but still need to be initialized+ //GGML_ASSERT(tensor->data == NULL); // views of pre-allocated tensors may have the data set in ggml_new_tensor, but still need to be initialized by the backendGGML_ASSERT(tensor->view_src != NULL);GGML_ASSERT(tensor->view_src->buffer != NULL);GGML_ASSERT(tensor->view_src->data != NULL);@@ -1332,6 +1583,7 @@ static void graph_init_tensor(struct ggml_hash_set hash_set, struct ggml_tensorstruct ggml_tensor * dst = node_copies[id];if (dst->view_src != NULL) {+ graph_init_tensor(hash_set, node_copies, node_init, src->view_src);ggml_backend_view_init(dst->view_src->buffer, dst);}else {@@ -1365,6 +1617,21 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, sstruct ggml_context * ctx_allocated = ggml_init(params);struct ggml_context * ctx_unallocated = ggml_init(params);+ if (ctx_allocated == NULL || ctx_unallocated == NULL) {+ fprintf(stderr, "failed to allocate context for graph copy\n");+ free(hash_set.keys);+ free(node_copies);+ free(node_init);+ ggml_free(ctx_allocated);+ ggml_free(ctx_unallocated);+ return (struct ggml_backend_graph_copy) {+ /* .buffer = */ NULL,+ /* .ctx_allocated = */ NULL,+ /* .ctx_unallocated = */ NULL,+ /* .graph = */ NULL,+ };+ }+// dup nodesfor (int i = 0; i < graph->n_nodes; i++) {struct ggml_tensor * node = graph->nodes[i];@@ -1373,6 +1640,20 @@ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, s// allocate nodesggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);+ if (buffer == NULL) {+ fprintf(stderr, "failed to allocate buffer for graph copy\n");+ free(hash_set.keys);+ free(node_copies);+ free(node_init);+ ggml_free(ctx_allocated);+ ggml_free(ctx_unallocated);+ return (struct ggml_backend_graph_copy) {+ /* .buffer = */ NULL,+ /* .ctx_allocated = */ NULL,+ /* .ctx_unallocated = */ NULL,+ /* .graph = */ NULL,+ };+ }//printf("copy buffer size: %zu MB\n", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);@@ -1409,8 +1690,12 @@ void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {ggml_free(copy.ctx_unallocated);}-void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) {+bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data) {struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);+ if (copy.buffer == NULL) {+ return false;+ }+struct ggml_cgraph * g1 = graph;struct ggml_cgraph * g2 = copy.graph;@@ -1440,6 +1725,8 @@ void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t}ggml_backend_graph_copy_free(copy);++ return true;}GGML_CALL static void system_exit(int rc) {@@ -1455,10 +1742,13 @@ GGML_CALL static void *system_malloc(size_t n) {}static const struct ggml_backend_api kGgmlBackendApi = {+ &FLAG_log_disable,system_exit,system_free,system_malloc,+ ggml_backend_register,ggml_backend_buffer_init,+ ggml_backend_buffer_is_host,ggml_backend_cpu_buffer_from_ptr,ggml_backend_cpu_buffer_type,ggml_backend_buft_get_alloc_size,diff --git a/llama.cpp/ggml-backend.h b/llama.cpp/ggml-backend.hindex be6fdd4f..2ca1654e 100644--- a/llama.cpp/ggml-backend.h+++ b/llama.cpp/ggml-backend.h@@ -1,5 +1,3 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once#include "ggml.h"@@ -19,22 +17,31 @@ extern "C" {//// buffer type- GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) GGML_CALL;- GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);- GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor) GGML_CALL;- GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);- GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);+ GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);+ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);+ GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);+ GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);+ GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);+ GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);// buffer- GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);- GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);- GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);- GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) GGML_CALL;- GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);- GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);- GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);- GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);- GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer);+ enum ggml_backend_buffer_usage {+ GGML_BACKEND_BUFFER_USAGE_ANY = 0,+ GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,+ };++ GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);+ GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);+ GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);+ GGML_API GGML_CALL bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);+ GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);//// Backend@@ -51,8 +58,8 @@ extern "C" {GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);- GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) GGML_CALL;- GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) GGML_CALL;+ GGML_API GGML_CALL void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);GGML_API void ggml_backend_synchronize(ggml_backend_t backend);@@ -73,13 +80,13 @@ extern "C" {GGML_API ggml_backend_t ggml_backend_cpu_init(void);- GGML_API bool ggml_backend_is_cpu(ggml_backend_t backend) GGML_CALL;- GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);+ GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);+ GGML_API void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);// Create a backend buffer from an existing pointer- GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) GGML_CALL;+ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);- GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) GGML_CALL;+ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);#ifdef GGML_USE_CPU_HBMGGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);@@ -141,24 +148,36 @@ extern "C" {struct ggml_backend_sched;typedef struct ggml_backend_sched * ggml_backend_sched_t;- // Initialize a backend scheduler- GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, int n_backends);-- GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);+ // when ask == true, the scheduler wants to know if the user wants to observe this node+ // this allows the scheduler to batch nodes together in order to evaluate them in a single call+ //+ // when ask == false, the scheduler is passing the node tensor to the user for observation+ // if the user returns false, the scheduler will cancel the graph compute+ //+ typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);+ // Initialize a backend scheduler+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size);+ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);// Initialize backend buffers from a measure graph- GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);+ GGML_API void ggml_backend_sched_init_measure(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);+ // Get the number of splits of the last graph+ GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);GGML_API ggml_tallocr_t ggml_backend_sched_get_tallocr(ggml_backend_sched_t sched, ggml_backend_t backend);GGML_API ggml_backend_buffer_t ggml_backend_sched_get_buffer (ggml_backend_sched_t sched, ggml_backend_t backend);- GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);+ GGML_API void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);+ GGML_API ggml_backend_t ggml_backend_sched_get_node_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);++ // Allocate and compute graph on the backend scheduler+ GGML_API void ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);- // Allocate a graph on the backend scheduler- GGML_API void ggml_backend_sched_graph_compute(- ggml_backend_sched_t sched,- struct ggml_cgraph * graph);+ // Reset all assignments and allocators - must be called before using the sched allocators to allocate inputs+ GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);+ // Set a callback to be called for each resulting node during graph compute+ GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);//// Utils@@ -178,7 +197,7 @@ extern "C" {typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);// Compare the output of two backends- GGML_API void ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);+ GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);// Tensor initializationGGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex fbd0653b..f9e7d0bf 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -1,5 +1,5 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#include <algorithm>#include <assert.h>@@ -11,12 +11,19 @@#include <limits>#include <stdint.h>#include <stdio.h>+#include <string>#include <vector>+#include <map>+#include <array>#ifdef __i386__#error "you need to use a 64-bit compiler for llamafile"#endif+// stringize macro for converting __CUDA_ARCH_LIST__ (list of integers) to string+#define STRINGIZE_IMPL(...) #__VA_ARGS__+#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)+#if defined(GGML_USE_TINYBLAS) && defined(GGML_USE_HIPBLAS)#include <hip/hip_runtime.h>#include <hipblas/hipblas.h>@@ -83,6 +90,7 @@#define cudaMemcpyKind hipMemcpyKind#define cudaMemset hipMemset#define cudaMemsetAsync hipMemsetAsync+#define cudaMemGetInfo hipMemGetInfo#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize#define cudaSetDevice hipSetDevice#define cudaStreamCreateWithFlags hipStreamCreateWithFlags@@ -94,13 +102,6 @@#define cudaSuccess hipSuccess#define __trap abort#define CUBLAS_STATUS_SUCCESS TINYBLAS_STATUS_SUCCESS-#define CUBLAS_STATUS_NOT_INITIALIZED TINYBLAS_STATUS_NOT_INITIALIZED-#define CUBLAS_STATUS_ALLOC_FAILED TINYBLAS_STATUS_ALLOC_FAILED-#define CUBLAS_STATUS_INVALID_VALUE TINYBLAS_STATUS_INVALID_VALUE-#define CUBLAS_STATUS_ARCH_MISMATCH TINYBLAS_STATUS_ARCH_MISMATCH-#define CUBLAS_STATUS_MAPPING_ERROR TINYBLAS_STATUS_MAPPING_ERROR-#define CUBLAS_STATUS_EXECUTION_FAILED TINYBLAS_STATUS_EXECUTION_FAILED-#define CUBLAS_STATUS_INTERNAL_ERROR TINYBLAS_STATUS_INTERNAL_ERROR#define CUBLAS_STATUS_NOT_SUPPORTED TINYBLAS_STATUS_NOT_SUPPORTED#include "tinyblas.cu"#define cublasGetStatusString tinyblasGetStatusString@@ -150,6 +151,7 @@#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT#define CUBLAS_OP_N HIPBLAS_OP_N#define CUBLAS_OP_T HIPBLAS_OP_T+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS#define CUBLAS_TF32_TENSOR_OP_MATH 0#define CUDA_R_16F HIPBLAS_R_16F#define CUDA_R_32F HIPBLAS_R_32F@@ -201,6 +203,7 @@#define cudaMemcpyKind hipMemcpyKind#define cudaMemset hipMemset#define cudaMemsetAsync hipMemsetAsync+#define cudaMemGetInfo hipMemGetInfo#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize#define cudaSetDevice hipSetDevice#define cudaStreamCreateWithFlags hipStreamCreateWithFlags@@ -220,8 +223,6 @@#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED-#define CUDART_VERSION 12000-#else#include <cuda_runtime.h>#include <cuda.h>@@ -238,6 +239,7 @@#endif // defined(GGML_USE_HIPBLAS)+// ggml-cuda need half type so keep ggml headers include at last#include "ggml-cuda.h"#include "ggml.h"#include "ggml-backend-impl.h"@@ -252,9 +254,10 @@#define CUBLAS_HANDLE(id) g_cublas_handles[id]#endif-static bool FLAG_log_disable;static const struct ggml_backend_api *g_backend;#define exit g_backend->exit+#define FLAG_log_disable (*g_backend->FLAG_log_disable)+#define ggml_backend_register g_backend->ggml_backend_register#define ggml_is_quantized g_backend->ggml_is_quantized#define ggml_type_size g_backend->ggml_type_size#define ggml_blck_size g_backend->ggml_blck_size@@ -275,11 +278,24 @@ static const struct ggml_backend_api *g_backend;#define ggml_backend_buft_alloc_buffer g_backend->ggml_backend_buft_alloc_buffer#define ggml_backend_buft_get_alloc_size g_backend->ggml_backend_buft_get_alloc_size#define ggml_backend_cpu_buffer_type g_backend->ggml_backend_cpu_buffer_type+#define ggml_backend_buffer_init g_backend->ggml_backend_buffer_init+#define ggml_backend_cpu_buffer_from_ptr g_backend->ggml_backend_cpu_buffer_from_ptrGGML_CALL void ggml_cuda_link(const struct ggml_backend_api *backend_api) {g_backend = backend_api;+ if (!FLAG_log_disable) {+ fprintf(stderr, "%s: welcome to " GGML_CUDA_NAME " SDK with "+#ifdef GGML_USE_TINYBLAS+ "tinyBLAS"+#else+ GGML_CUBLAS_NAME+#endif+ "\n", __func__);+ }}+#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)+#define CC_PASCAL 600#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products#define CC_VOLTA 700@@ -379,7 +395,7 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");-//[[noreturn]]+[[noreturn]]static void ggml_cuda_error(const char * stmt, const char * func, const char * file, const int line, const char * msg) {int id = -1; // in case cudaGetDevice failscudaGetDevice(&id);@@ -650,6 +666,15 @@ typedef struct {} block_iq2_xxs;static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");+#define QR2_XS 8+#define QI2_XS (QK_K / (4*QR2_XS))+typedef struct {+ half d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");+#define WARP_SIZE 32#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses@@ -674,6 +699,8 @@ static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint1#define CUDA_ACC_BLOCK_SIZE 256#define CUDA_IM2COL_BLOCK_SIZE 256+#define CUDA_Q8_0_NE_ALIGN 2048+// dmmv = dequantize_mul_mat_vec#ifndef GGML_CUDA_DMMV_X#define GGML_CUDA_DMMV_X 32@@ -717,7 +744,7 @@ static void ggml_cuda_set_device(const int device) {static int g_device_count = -1;static int g_main_device = 0;-static float g_tensor_split[GGML_CUDA_MAX_DEVICES] = {0};+static std::array<float, GGML_CUDA_MAX_DEVICES> g_default_tensor_split = {};struct cuda_device_capabilities {int cc; // compute capability@@ -728,22 +755,33 @@ struct cuda_device_capabilities {static cuda_device_capabilities g_device_caps[GGML_CUDA_MAX_DEVICES] = { {0, 0, false, 0} };-static void * g_scratch_buffer = nullptr;-static size_t g_scratch_size = 0; // disabled by default-static size_t g_scratch_offset = 0;-#ifndef GGML_USE_TINYBLASstatic cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};#endif-//[[noreturn]]-static __device__ void bad_arch() {- printf("ERROR: ggml-cuda was compiled without support for the current GPU architecture.\n");+[[noreturn]]+static __device__ void no_device_code(+ const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {++#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+ printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",+ file_name, line, function_name, arch);+ (void) arch_list;+#else+ printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",+ file_name, line, function_name, arch, arch_list);+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)__trap();- (void) bad_arch; // suppress unused function warning+ (void) no_device_code; // suppress unused function warning}+#ifdef __CUDA_ARCH__+#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))+#else+#define NO_DEVICE_CODE GGML_ASSERT(false && "NO_DEVICE_CODE not valid in host code.")+#endif // __CUDA_ARCH__+static __device__ __forceinline__ float warp_reduce_sum(float x) {#pragma unrollfor (int mask = 16; mask > 0; mask >>= 1) {@@ -762,16 +800,16 @@ static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {}static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {-#if __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))- (void) a;- bad_arch();-#else+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL#pragma unrollfor (int mask = 16; mask > 0; mask >>= 1) {a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, mask, 32));}return a;-#endif // __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+#else+ (void) a;+ NO_DEVICE_CODE;+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL}static __device__ __forceinline__ float warp_reduce_max(float x) {@@ -783,16 +821,16 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {}static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {-#if __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))- (void) x;- bad_arch();-#else+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX#pragma unrollfor (int mask = 16; mask > 0; mask >>= 1) {x = __hmax2(x, __shfl_xor_sync(0xffffffff, x, mask, 32));}return x;-#endif // __CUDA_ARCH__ < CC_PASCAL || (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))+#else+ (void) x;+ NO_DEVICE_CODE;+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX}static __device__ __forceinline__ float op_repeat(const float a, const float b) {@@ -1260,6 +1298,61 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in#endif // GGML_CUDA_F16}+template<typename dst_t>+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {++ const int i = blockIdx.x;++ // assume 32 threads+ const int tid = threadIdx.x;+ const int il = tid/8;+ const int ir = tid%8;+ const int ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }++ dst_t * y = yy + 256*i + 32*ir + 4*il;++ const block_q4_0 * x = (const block_q4_0 *)vx + ib;+ const float d = __half2float(x->d);+ const float dm = -8*d;++ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d * (q[l] & 0xF) + dm;+ y[l+16] = d * (q[l] >> 4) + dm;+ }+}++template<typename dst_t>+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {++ const int i = blockIdx.x;++ // assume 32 threads+ const int tid = threadIdx.x;+ const int il = tid/8;+ const int ir = tid%8;+ const int ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }++ dst_t * y = yy + 256*i + 32*ir + 4*il;++ const block_q4_1 * x = (const block_q4_1 *)vx + ib;+ const float2 d = __half22float2(x->dm);++ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = d.x * (q[l] & 0xF) + d.y;+ y[l+16] = d.x * (q[l] >> 4) + d.y;+ }+}+//================================== k-quantstemplate<typename dst_t>@@ -1494,7 +1587,7 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t#endif}-static const __device__ uint64_t kgrid_iq2xxs[256] = {+static const __device__ uint64_t iq2xxs_grid[256] = {0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,@@ -1561,6 +1654,137 @@ static const __device__ uint64_t kgrid_iq2xxs[256] = {0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,};+static const __device__ uint64_t iq2xs_grid[512] = {+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+};+static const __device__ uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -1605,7 +1829,7 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dsdst_t * y = yy + i*QK_K + 32*ib + 8*il;const uint16_t * q2 = x[i].qs + 4*ib;const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid = (const uint8_t *)(kgrid_iq2xxs + aux8[il]);+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);const uint32_t aux32 = q2[2] | (q2[3] << 16);const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];@@ -1616,6 +1840,28 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds}+template<typename dst_t>+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {++ const int i = blockIdx.x;+ const block_iq2_xs * x = (const block_iq2_xs *) vx;++ const int tid = threadIdx.x;+#if QK_K == 256+ const int il = tid/8; // 0...3+ const int ib = tid%8; // 0...7+ dst_t * y = yy + i*QK_K + 32*ib + 8*il;+ const uint16_t * q2 = x[i].qs + 4*ib;+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));+ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];+ for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+#else+ assert(false);+#endif++}+static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows) {static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");@@ -2331,6 +2577,45 @@ static __global__ void convert_unary(const void * __restrict__ vx, dst_t * __resy[i] = x[i];}+template <bool need_check>+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int k) {+#if __CUDA_ARCH__ >= CC_PASCAL+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;++ const int i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;+ const int * x0 = ((int *) vx) + blockIdx.x * nint;+ half2 * y2 = (half2 *) (y + i0);++ __shared__ int vals[nint];++#pragma unroll+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {+ break;+ }++ const int ix = ix0 + threadIdx.x;+ vals[ix] = x0[ix];+ }++#pragma unroll+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {+ return;+ }++ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);+ const half d = *b0;+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];++ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));+ }+#else+ (void) vx; (void) y; (void) k;+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= CC_PASCAL+}+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q@@ -2358,7 +2643,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_imp// second part effectively subtracts 8 from each quant valuereturn d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2395,7 +2680,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_imp// scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding itreturn sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2430,7 +2715,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_imp// second part effectively subtracts 16 from each quant valuereturn d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2475,7 +2760,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impreturn sumi*d5d8 + m5s8 / (QI5_1 / vdr);#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2496,7 +2781,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impreturn d8_0*d8_1 * sumi;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2526,7 +2811,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_imp// scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding itreturn sumi*d8d8 + m8s8 / (QI8_1 / vdr);#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2561,7 +2846,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(return dm2f.x*sumf_d - dm2f.y*sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2598,7 +2883,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2638,7 +2923,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(return d3 * sumf;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2663,7 +2948,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(return d3*d8 * sumi;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2696,7 +2981,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2729,7 +3014,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2769,7 +3054,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(return dm5f.x*sumf_d - dm5f.y*sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2802,7 +3087,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2832,7 +3117,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(return d*sumf;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2863,7 +3148,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(return d6 * sumf_d;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -3729,7 +4014,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(return dall * sumf_d - dmin * sumf_m;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A#endif@@ -3912,7 +4197,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(return d * sumf_d;#else- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= MIN_CC_DP4A#endif@@ -4162,7 +4447,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(uint32_t aux32 = q2[2] | (q2[3] << 16);int sumi = 0;for (int l = 0; l < 4; ++l) {- const uint8_t * grid = (const uint8_t *)(kgrid_iq2xxs + aux8[l]);+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);const uint8_t signs = ksigns_iq2xs[aux32 & 127];for (int j = 0; j < 8; ++j) {sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);@@ -4170,7 +4455,7 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(q8 += 8;aux32 >>= 7;}- const float d = (float)bq2->d * (0.5f + aux32) * (float)bq8_1[ib32].ds.x * 0.25f;+ const float d = (float)bq2->d * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.25f;return d * sumi;#else// iqs is 0...15@@ -4178,10 +4463,10 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(const int il = iqs%2;const uint16_t * q2 = bq2->qs + 4*ib32;const uint8_t * aux8 = (const uint8_t *)q2;- const uint8_t * grid1 = (const uint8_t *)(kgrid_iq2xxs + aux8[2*il+0]);- const uint8_t * grid2 = (const uint8_t *)(kgrid_iq2xxs + aux8[2*il+1]);+ const uint8_t * grid1 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq2xxs_grid + aux8[2*il+1]);const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * (float)bq8_1[ib32].ds.x * 0.25f;+ const float d = (float)bq2->d * (0.5f + (aux32 >> 28)) * __low2float(bq8_1[ib32].ds) * 0.25f;const uint8_t signs1 = ksigns_iq2xs[(aux32 >> 14*il) & 127];const uint8_t signs2 = ksigns_iq2xs[(aux32 >> (14*il + 7)) & 127];const int8_t * q8 = bq8_1[ib32].qs + 16*il;@@ -4198,6 +4483,42 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(#endif}+static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+#if QK_K == 256+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;++ const int ib32 = iqs;+ const uint16_t * q2 = bq2->qs + 4*ib32;+ const int8_t * q8 = bq8_1[ib32].qs;+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;+ const uint8_t ls2 = bq2->scales[ib32] >> 4;+ int sumi1 = 0;+ for (int l = 0; l < 2; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ int sumi2 = 0;+ for (int l = 2; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ const float d = (float)bq2->d * __low2float(bq8_1[ib32].ds) * 0.25f;+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);+#else+ assert(false);+ return 0.f;+#endif+}+template <int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>static __device__ __forceinline__ void mul_mat_q(@@ -4371,7 +4692,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_0_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4440,7 +4761,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_1_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4507,7 +4828,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_0_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4574,7 +4895,7 @@ mul_mat_q5_1((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_1_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4641,7 +4962,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q8_0_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4708,7 +5029,7 @@ mul_mat_q2_K((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q2_K_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4777,7 +5098,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q3_K_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4846,7 +5167,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_K_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4913,7 +5234,7 @@ mul_mat_q5_K((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_K_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4982,7 +5303,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q6_K_q8_1_mul_mat;- bad_arch();+ NO_DEVICE_CODE;#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -5003,10 +5324,10 @@ static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void *const block_q_t * x = (const block_q_t *) vx;const block_q8_1 * y = (const block_q8_1 *) vy;- for (int i = 0; i < blocks_per_row; i += blocks_per_warp) {- const int ibx = row*blocks_per_row + i + threadIdx.x / (qi/vdr); // x block index+ for (int i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {+ const int ibx = row*blocks_per_row + i; // x block index- const int iby = (i + threadIdx.x / (qi/vdr)) * (qk/QK8_1); // y block index that aligns with ibx+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibxconst int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int@@ -5581,7 +5902,7 @@ static __global__ void diag_mask_inf_f32(const float * x, float * dst, const inttemplate <bool vals_smem, int ncols_template, int block_size_template, bool need_check>static __global__ void soft_max_f16(const float * x, const float * y, float * dst, const int ncols_par, const int nrows_y, const float scale) {-#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAXconst int ncols_data = ncols_template == 0 ? ncols_par : ncols_template;const int ncols_smem = GGML_PAD(ncols_data, 2*WARP_SIZE)/2;@@ -5705,8 +6026,8 @@ static __global__ void soft_max_f16(const float * x, const float * y, float * ds}#else(void) x; (void) y; (void) dst; (void) ncols_par; (void) nrows_y; (void) scale;- bad_arch();-#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL+ NO_DEVICE_CODE;+#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_PASCAL && CUDART_VERSION >= CUDART_HMAX}template <bool vals_smem, int ncols_template, int block_size_template>@@ -6149,6 +6470,17 @@ static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restridequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);}+static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {+ const bool need_check = false;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ } else {+ const bool need_check = true;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ }+}+template<typename dst_t>static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {const int nb = k / QK_K;@@ -6169,6 +6501,20 @@ static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int k, cu#endif}+template<typename dst_t>+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);+}++template<typename dst_t>+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);+}+template<typename dst_t>static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {const int nb = k / QK_K;@@ -6201,6 +6547,12 @@ static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int k,dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);}+template<typename dst_t>+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);+}+template <typename src_t, typename dst_t>static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int k, cudaStream_t stream) {const int num_blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;@@ -6208,16 +6560,21 @@ static void convert_unary_cuda(const void * __restrict__ vx, dst_t * __restrict_}static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ int id;switch (type) {case GGML_TYPE_Q4_0:- return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:- return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ return dequantize_row_q4_1_cuda;case GGML_TYPE_Q5_0:return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;case GGML_TYPE_Q5_1:return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;case GGML_TYPE_Q8_0:+ CUDA_CHECK(cudaGetDevice(&id));+ if (g_device_caps[id].cc >= CC_PASCAL) {+ return dequantize_block_q8_0_f16_cuda;+ }return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;case GGML_TYPE_Q2_K:return dequantize_row_q2_K_cuda;@@ -6231,6 +6588,8 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_q6_K_cuda;case GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;case GGML_TYPE_F32:return convert_unary_cuda<float>;default:@@ -6241,9 +6600,9 @@ static to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {switch (type) {case GGML_TYPE_Q4_0:- return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ return dequantize_row_q4_0_cuda;case GGML_TYPE_Q4_1:- return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ return dequantize_row_q4_1_cuda;case GGML_TYPE_Q5_0:return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;case GGML_TYPE_Q5_1:@@ -6262,6 +6621,8 @@ static to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {return dequantize_row_q6_K_cuda;case GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;case GGML_TYPE_F16:return convert_unary_cuda<half>;default:@@ -6465,6 +6826,15 @@ static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, floa<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);}+static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, cudaStream_t stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;+ const dim3 block_nums(block_num_y, 1, 1);+ const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);+ mul_mat_vec_q<QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>+ <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows);+}+static void ggml_mul_mat_q4_0_q8_1_cuda(const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x,const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {@@ -7362,7 +7732,7 @@ static void * ggml_cuda_pool_malloc_vmm(int device, size_t size, size_t * actualg_cuda_pool_used[device] += size;#ifdef DEBUG_CUDA_MALLOC- fprintf(stderr, "cuda pool[%d]: allocated %llu bytes at %llx [%s]\n", id, (unsigned long long) size, ptr);+ printf("cuda pool[%d]: allocated %llu bytes at %llx [%s]\n", id, (unsigned long long) size, ptr);#endifreturn ptr;@@ -7372,7 +7742,7 @@ static void ggml_cuda_pool_free_vmm(int device, void * ptr, size_t size) {scoped_spin_lock lock(g_cuda_pool_lock);#ifdef DEBUG_CUDA_MALLOC- fprintf(stderr, "cuda pool[%d]: freed %llu bytes at %llx\n", id, (unsigned long long) size, ptr);+ printf("cuda pool[%d]: freed %llu bytes at %llx\n", id, (unsigned long long) size, ptr);#endifg_cuda_pool_used[device] -= size;@@ -7438,13 +7808,12 @@ struct cuda_pool_alloc {static bool g_cublas_loaded = false;-GGML_CALL bool ggml_cublas_loaded() {+GGML_CALL bool ggml_cublas_loaded(void) {return g_cublas_loaded;}-GGML_CALL void ggml_init_cublas(bool log_disable) {+GGML_CALL void ggml_init_cublas() {static bool initialized = false;- FLAG_log_disable = log_disable;if (!initialized) {@@ -7505,12 +7874,12 @@ GGML_CALL void ggml_init_cublas(bool log_disable) {cudaDeviceProp prop;CUDA_CHECK(cudaGetDeviceProperties(&prop, id));- if (!FLAG_log_disable) {+ if (!FLAG_log_disable)fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");- }- g_tensor_split[id] = total_vram;+ g_default_tensor_split[id] = total_vram;total_vram += prop.totalGlobalMem;+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)g_device_caps[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;#else@@ -7519,7 +7888,7 @@ GGML_CALL void ggml_init_cublas(bool log_disable) {g_device_caps[id].smpb = prop.sharedMemPerBlock;}for (int id = 0; id < g_device_count; ++id) {- g_tensor_split[id] /= total_vram;+ g_default_tensor_split[id] /= total_vram;}for (int id = 0; id < g_device_count; ++id) {@@ -7545,30 +7914,6 @@ GGML_CALL void ggml_init_cublas(bool log_disable) {}}-GGML_CALL void ggml_cuda_set_tensor_split(const float * tensor_split) {- if (tensor_split == nullptr) {- return;- }- bool all_zero = true;- for (int i = 0; i < g_device_count; ++i) {- if (tensor_split[i] != 0.0f) {- all_zero = false;- break;- }- }- if (all_zero) {- return;- }- float split_sum = 0.0f;- for (int i = 0; i < g_device_count; ++i) {- g_tensor_split[i] = split_sum;- split_sum += tensor_split[i];- }- for (int i = 0; i < g_device_count; ++i) {- g_tensor_split[i] /= split_sum;- }-}-GGML_CALL void * ggml_cuda_host_malloc(size_t size) {if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {return nullptr;@@ -8020,11 +8365,11 @@ static void ggml_cuda_op_mul_mat_q((void) src1_ddf_i;}-static int64_t get_row_rounding(ggml_type type) {+static int64_t get_row_rounding(ggml_type type, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {int64_t min_compute_capability = INT_MAX;int64_t max_compute_capability = INT_MIN;for (int id = 0; id < g_device_count; ++id) {- if (g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) {+ if (tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {if (min_compute_capability > g_device_caps[id].cc) {min_compute_capability = g_device_caps[id].cc;}@@ -8053,6 +8398,7 @@ static int64_t get_row_rounding(ggml_type type) {case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:return max_compute_capability >= CC_RDNA2 ? 128 : 64;default:GGML_ASSERT(false);@@ -8074,6 +8420,7 @@ static int64_t get_row_rounding(ggml_type type) {case GGML_TYPE_Q4_K:case GGML_TYPE_Q5_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:return max_compute_capability >= CC_VOLTA ? 128 : 64;case GGML_TYPE_Q6_K:return 64;@@ -8083,6 +8430,21 @@ static int64_t get_row_rounding(ggml_type type) {#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)}+static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {+ const int64_t nrows = ggml_nrows(tensor);+ const int64_t rounding = get_row_rounding(tensor->type, tensor_split);++ *row_low = id == 0 ? 0 : nrows*tensor_split[id];+ *row_low -= *row_low % rounding;++ if (id == g_device_count - 1) {+ *row_high = nrows;+ } else {+ *row_high = nrows*tensor_split[id + 1];+ *row_high -= *row_high % rounding;+ }+}+static void ggml_cuda_op_mul_mat_vec_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,@@ -8127,6 +8489,9 @@ static void ggml_cuda_op_mul_mat_vec_q(case GGML_TYPE_IQ2_XXS:mul_mat_vec_iq2_xxs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);break;+ case GGML_TYPE_IQ2_XS:+ mul_mat_vec_iq2_xs_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, stream);+ break;default:GGML_ASSERT(false);break;@@ -8535,15 +8900,15 @@ static void ggml_cuda_op_soft_max(float scale = 1.0f;memcpy(&scale, dst->op_params, sizeof(float));-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- const bool use_f16_soft_max = false;-#else+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && CUDART_VERSION >= CUDART_HMAX#ifdef GGML_CUDA_F16const bool use_f16_soft_max = true;#elseconst bool use_f16_soft_max = false;#endif // GGML_CUDA_F16-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)+#else+ const bool use_f16_soft_max = false;+#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) && CUDART_VERSION >= CUDART_HMAXif (use_f16_soft_max) {soft_max_f16_cuda(src0_dd, src1 ? src1_dd : nullptr, dst_dd, ne00, nrows_x, nrows_y, scale, main_stream);@@ -8698,6 +9063,11 @@ static void ggml_cuda_set_peer_access(const int n_tokens) {peer_access_enabled = enable_peer_access;}+// FIXME: move this somewhere else+struct ggml_backend_cuda_split_buffer_type_context {+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;+};+static void ggml_cuda_op_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,const bool convert_src1_to_q8_1) {@@ -8749,6 +9119,14 @@ static void ggml_cuda_op_mul_mat(GGML_ASSERT(!(split && ne03 > 1));GGML_ASSERT(!(split && ne02 < ne12));+ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;+ if (split) {+ // TODO: check that src0->buffer->buft is a split buffer type, replace GGML_BACKEND_GPU_SPLIT check+ // GGML_ASSERT(src0->buffer != nullptr && src0->buffer->buft == ...);+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;+ tensor_split = buft_ctx->tensor_split;+ }+struct dev_data {cuda_pool_alloc<char> src0_dd_alloc;cuda_pool_alloc<float> src1_ddf_alloc;@@ -8776,17 +9154,17 @@ static void ggml_cuda_op_mul_mat(// for multi GPU, get the row boundaries from tensor split// and round to mul_mat_q tile sizesif (split) {- const int64_t rounding = get_row_rounding(src0->type);+ const int64_t rounding = get_row_rounding(src0->type, tensor_split);if (id != 0) {- dev[id].row_low = ne01*g_tensor_split[id];+ dev[id].row_low = ne01*tensor_split[id];if (dev[id].row_low < ne01) {dev[id].row_low -= dev[id].row_low % rounding;}}if (id != g_device_count - 1) {- dev[id].row_high = ne01*g_tensor_split[id + 1];+ dev[id].row_high = ne01*tensor_split[id + 1];if (dev[id].row_high < ne01) {dev[id].row_high -= dev[id].row_high % rounding;}@@ -9333,10 +9711,17 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT;int64_t min_compute_capability = INT_MAX;- for (int id = 0; id < g_device_count; ++id) {- if (min_compute_capability > g_device_caps[id].cc && g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) {- min_compute_capability = g_device_caps[id].cc;++ if (split) {+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;+ auto & tensor_split = buft_ctx->tensor_split;+ for (int id = 0; id < g_device_count; ++id) {+ if (min_compute_capability > g_device_caps[id].cc && tensor_split[id] < (id + 1 < g_device_count ? tensor_split[id + 1] : 1.0f)) {+ min_compute_capability = g_device_caps[id].cc;+ }}+ } else {+ min_compute_capability = g_device_caps[g_main_device].cc;}#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)@@ -9375,7 +9760,7 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1} else if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {// KQV single-batchggml_cuda_mul_mat_vec_nc(src0, src1, dst);- } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) {+ } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {// KQ + KQV multi-batchggml_cuda_mul_mat_mat_batched_cublas(src0, src1, dst);} else if (src0->type == GGML_TYPE_F32) {@@ -9593,8 +9978,8 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s// TODO: mmq/mmv support#endif- const int64_t nb11 = src1->nb[1];- const int64_t nb1 = dst->nb[1];+ const size_t nb11 = src1->nb[1];+ const size_t nb1 = dst->nb[1];const struct ggml_tensor * ids = src0;const int32_t id = ((int32_t *) dst->op_params)[0];@@ -9837,303 +10222,41 @@ static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_splreturn nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);}-GGML_CALL void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {- const int64_t nrows = ggml_nrows(tensor);-- const int64_t ne0 = tensor->ne[0];-- const size_t nb1 = tensor->nb[1];-- ggml_backend_type backend = tensor->backend;- ggml_tensor_extra_gpu * extra = new struct ggml_tensor_extra_gpu;- memset(extra, 0, sizeof(*extra));-- for (int id = 0; id < g_device_count; ++id) {- if (backend == GGML_BACKEND_GPU && id != g_main_device) {- continue;- }-- ggml_cuda_set_device(id);-- int64_t row_low, row_high;- if (backend == GGML_BACKEND_GPU) {- row_low = 0;- row_high = nrows;- } else if (backend == GGML_BACKEND_GPU_SPLIT) {- const int64_t rounding = get_row_rounding(tensor->type);+GGML_CALL static void ggml_cuda_set_main_device(const int main_device) {+ if (main_device >= g_device_count) {+ fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n",+ main_device, g_device_count, g_main_device);+ return;+ }- row_low = id == 0 ? 0 : nrows*g_tensor_split[id];- row_low -= row_low % rounding;+ if (g_main_device != main_device && g_device_count > 1) {+ g_main_device = main_device;+ //cudaDeviceProp prop;+ //CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device));+ //fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name);+ }+}- if (id == g_device_count - 1) {- row_high = nrows;- } else {- row_high = nrows*g_tensor_split[id + 1];- row_high -= row_high % rounding;- }- } else {- GGML_ASSERT(false);- }- if (row_low == row_high) {- continue;- }+GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {+ if (!g_cublas_loaded) return false;- int64_t nrows_split = row_high - row_low;+ ggml_cuda_func_t func;+ const bool any_on_device = tensor->backend == GGML_BACKEND_GPU+ || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT))+ || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU);- const size_t offset_split = row_low*nb1;- size_t size = ggml_nbytes_split(tensor, nrows_split);- const size_t original_size = size;+ if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) {+ return false;+ }- // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses- if (ne0 % MATRIX_ROW_PADDING != 0) {- size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ if (tensor->op == GGML_OP_MUL_MAT) {+ if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {+#ifndef NDEBUG+ fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);+#endif+ return false;}-- char * buf;- CUDA_CHECK(cudaMalloc(&buf, size));- char * buf_host = (char *)data + offset_split;-- // set padding to 0 to avoid possible NaN values- if (size > original_size) {- CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));- }-- CUDA_CHECK(cudaMemcpy(buf, buf_host, original_size, cudaMemcpyHostToDevice));-- extra->data_device[id] = buf;-- if (backend == GGML_BACKEND_GPU_SPLIT) {- for (int64_t is = 0; is < MAX_STREAMS; ++is) {- CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));- }- }- }-- tensor->extra = extra;-}--GGML_CALL void ggml_cuda_free_data(struct ggml_tensor * tensor) {- if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_GPU && tensor->backend != GGML_BACKEND_GPU_SPLIT) ) {- return;- }-- ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra;-- for (int id = 0; id < g_device_count; ++id) {- ggml_cuda_set_device(id);- if (extra->data_device[id] != nullptr) {- CUDA_CHECK(cudaFree(extra->data_device[id]));- }-- for (int64_t is = 0; is < MAX_STREAMS; ++is) {- if (extra->events[id][is] != nullptr) {- CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));- }- }- }-- delete extra;-}--static ggml_tensor_extra_gpu * g_temp_tensor_extras = nullptr;-static size_t g_temp_tensor_extra_index = 0;--static ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() {- if (g_temp_tensor_extras == nullptr) {- g_temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES];- }-- size_t alloc_index = g_temp_tensor_extra_index;- g_temp_tensor_extra_index = (g_temp_tensor_extra_index + 1) % GGML_CUDA_MAX_NODES;- ggml_tensor_extra_gpu * extra = &g_temp_tensor_extras[alloc_index];- memset(extra, 0, sizeof(*extra));-- return extra;-}--static void ggml_cuda_assign_buffers_impl(struct ggml_tensor * tensor, bool scratch, bool force_inplace, bool no_alloc) {- if (scratch && g_scratch_size == 0) {- return;- }-- tensor->backend = GGML_BACKEND_GPU;-- // recursively assign CUDA buffers until a compute tensor is found- if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_CPU) {- const ggml_op src0_op = tensor->src[0]->op;- if (src0_op == GGML_OP_RESHAPE || src0_op == GGML_OP_TRANSPOSE || src0_op == GGML_OP_VIEW || src0_op == GGML_OP_PERMUTE) {- ggml_cuda_assign_buffers_impl(tensor->src[0], scratch, force_inplace, no_alloc);- }- }- if (tensor->op == GGML_OP_CPY && tensor->src[1]->backend == GGML_BACKEND_CPU) {- ggml_cuda_assign_buffers_impl(tensor->src[1], scratch, force_inplace, no_alloc);- }-- if (scratch && no_alloc) {- return;- }-- ggml_tensor_extra_gpu * extra;-- const bool inplace = (tensor->src[0] != nullptr && tensor->src[0]->data == tensor->data) ||- tensor->op == GGML_OP_VIEW ||- force_inplace;- const size_t size = ggml_nbytes(tensor);-- ggml_cuda_set_device(g_main_device);- if (inplace && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) {- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra;- char * src0_ddc = (char *) src0_extra->data_device[g_main_device];- size_t offset = 0;- if (tensor->op == GGML_OP_VIEW) {- memcpy(&offset, tensor->op_params, sizeof(size_t));- }- extra = ggml_cuda_alloc_temp_tensor_extra();- extra->data_device[g_main_device] = src0_ddc + offset;- } else if (tensor->op == GGML_OP_CPY) {- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu * ) tensor->src[1]->extra;- void * src1_ddv = src1_extra->data_device[g_main_device];- extra = ggml_cuda_alloc_temp_tensor_extra();- extra->data_device[g_main_device] = src1_ddv;- } else if (scratch) {- GGML_ASSERT(size <= g_scratch_size);- if (g_scratch_offset + size > g_scratch_size) {- g_scratch_offset = 0;- }-- char * data = (char *) g_scratch_buffer;- if (data == nullptr) {- CUDA_CHECK(cudaMalloc(&data, g_scratch_size));- g_scratch_buffer = data;- }- extra = ggml_cuda_alloc_temp_tensor_extra();- extra->data_device[g_main_device] = data + g_scratch_offset;-- g_scratch_offset += size;-- GGML_ASSERT(g_scratch_offset <= g_scratch_size);- } else { // allocate new buffers outside of scratch- void * data;- CUDA_CHECK(cudaMalloc(&data, size));- CUDA_CHECK(cudaMemset(data, 0, size));- extra = new ggml_tensor_extra_gpu;- memset(extra, 0, sizeof(*extra));- extra->data_device[g_main_device] = data;- }-- tensor->extra = extra;-}--GGML_CALL void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset) {- if (g_scratch_size == 0) {- return;- }- if (g_scratch_buffer == nullptr) {- ggml_cuda_set_device(g_main_device);- CUDA_CHECK(cudaMalloc(&g_scratch_buffer, g_scratch_size));- }-- ggml_tensor_extra_gpu * extra = ggml_cuda_alloc_temp_tensor_extra();-- const bool inplace = tensor->view_src != nullptr;-- if (inplace && (tensor->view_src->backend == GGML_BACKEND_GPU || tensor->view_src->backend == GGML_BACKEND_GPU_SPLIT)) {- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->view_src->extra;- char * src0_ddc = (char *) src0_extra->data_device[g_main_device];- size_t view_offset = 0;- if (tensor->op == GGML_OP_VIEW) {- memcpy(&view_offset, tensor->op_params, sizeof(size_t));- }- extra->data_device[g_main_device] = src0_ddc + view_offset;- } else {- extra->data_device[g_main_device] = (char *) g_scratch_buffer + offset;- }-- tensor->extra = extra;-}--GGML_CALL void ggml_cuda_copy_to_device(struct ggml_tensor * tensor) {- GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);- GGML_ASSERT(ggml_is_contiguous(tensor));-- ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra;- ggml_cuda_set_device(g_main_device);- CUDA_CHECK(cudaMemcpy(extra->data_device[g_main_device], tensor->data, ggml_nbytes(tensor), cudaMemcpyHostToDevice));-}--GGML_CALL void ggml_cuda_assign_buffers(struct ggml_tensor * tensor) {- ggml_cuda_assign_buffers_impl(tensor, true, false, false);-}--GGML_CALL void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor * tensor) {- ggml_cuda_assign_buffers_impl(tensor, true, false, true);-}--GGML_CALL void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor) {- ggml_cuda_assign_buffers_impl(tensor, false, false, false);-}--GGML_CALL void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor) {- ggml_cuda_assign_buffers_impl(tensor, false, true, false);-}--GGML_CALL void ggml_cuda_set_main_device(const int main_device) {- if (main_device >= g_device_count) {- fprintf(stderr, "warning: cannot set main_device=%d because there are only %d devices. Using device %d instead.\n",- main_device, g_device_count, g_main_device);- return;- }-- if (g_main_device != main_device && g_device_count > 1) {- g_main_device = main_device;- cudaDeviceProp prop;- CUDA_CHECK(cudaGetDeviceProperties(&prop, g_main_device));- if (!FLAG_log_disable) {- fprintf(stderr, "%s: using device %d (%s) as main device\n", __func__, g_main_device, prop.name);- }- }-}--GGML_CALL void ggml_cuda_set_scratch_size(const size_t scratch_size) {- // this is a hack to not completely break llama.cpp when using multiple models or contexts simultaneously- // it still won't always work as expected, but it's better than nothing- if (scratch_size > g_scratch_size) {- ggml_cuda_free_scratch();- }- g_scratch_size = std::max(g_scratch_size, scratch_size);-}--GGML_CALL void ggml_cuda_free_scratch() {- if (g_scratch_buffer == nullptr) {- return;- }-- CUDA_CHECK(cudaFree(g_scratch_buffer));- g_scratch_buffer = nullptr;-}--GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {- if (!g_cublas_loaded) return false;-- ggml_cuda_func_t func;- const bool any_on_device = tensor->backend == GGML_BACKEND_GPU- || (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT))- || (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU);-- if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) {- return false;- }-- if (tensor->op == GGML_OP_MUL_MAT) {- if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {-#ifndef NDEBUG- if (!FLAG_log_disable) {- fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);- }-#endif- return false;- }- }+ }switch (tensor->op) {case GGML_OP_REPEAT:@@ -10292,21 +10415,31 @@ GGML_CALL void ggml_cuda_get_device_description(int device, char * description,#define UNUSED GGML_UNUSED+struct ggml_backend_cuda_context {+ int device;+ std::string name;+};+// cuda buffer-struct ggml_backend_buffer_context_cuda {+struct ggml_backend_cuda_buffer_context {int device;void * dev_ptr = nullptr;ggml_tensor_extra_gpu * temp_tensor_extras = nullptr;size_t temp_tensor_extra_index = 0;+ std::string name;- ggml_backend_buffer_context_cuda(int device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {}+ ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :+ device(device), dev_ptr(dev_ptr),+ name(GGML_CUDA_NAME + std::to_string(device)) {+ }- ~ggml_backend_buffer_context_cuda() {+ ~ggml_backend_cuda_buffer_context() {delete[] temp_tensor_extras;}ggml_tensor_extra_gpu * ggml_cuda_alloc_temp_tensor_extra() {+ // TODO: remove GGML_CUDA_MAX_NODES, allocate dynamically and reuse in backend_buffer_resetif (temp_tensor_extras == nullptr) {temp_tensor_extras = new ggml_tensor_extra_gpu[GGML_CUDA_MAX_NODES];}@@ -10320,19 +10453,28 @@ struct ggml_backend_buffer_context_cuda {}};+GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ return ctx->name.c_str();+}++GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {+ return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name;+}+GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;CUDA_CHECK(cudaFree(ctx->dev_ptr));delete ctx;}GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;return ctx->dev_ptr;}GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;if (tensor->view_src != NULL && tensor->view_offs == 0) {assert(tensor->view_src->buffer->buft == buffer->buft);@@ -10350,78 +10492,104 @@ GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_tif (ggml_is_quantized(tensor->type)) {// initialize padding to 0 to avoid possible NaN values- int64_t row_low = 0;- int64_t row_high = ggml_nrows(tensor);- int64_t nrows_split = row_high - row_low;-- size_t original_size = ggml_nbytes_split(tensor, nrows_split);+ size_t original_size = ggml_nbytes(tensor);size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);if (padded_size > original_size && tensor->view_src == nullptr) {- CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, g_cudaStreams[ctx->device][0]));+ CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));}}-- UNUSED(buffer);}GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;ggml_cuda_set_device(ctx->device);CUDA_CHECK(cudaDeviceSynchronize());-CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice));+ CUDA_CHECK(cudaDeviceSynchronize());}GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;ggml_cuda_set_device(ctx->device);CUDA_CHECK(cudaDeviceSynchronize());-CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost));+ CUDA_CHECK(cudaDeviceSynchronize());+}++GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {+ if (ggml_backend_buffer_is_cuda(src->buffer)) {+ ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;+ ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context;++ ggml_cuda_set_device(src_ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());+ ggml_cuda_set_device(dst_ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());+ CUDA_CHECK(cudaMemcpy((char *)dst->data, (const char *)src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice));+ CUDA_CHECK(cudaDeviceSynchronize());++ return true;+ }+ return false;}GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;ggml_cuda_set_device(ctx->device);CUDA_CHECK(cudaDeviceSynchronize());-CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));+ CUDA_CHECK(cudaDeviceSynchronize());}-static struct ggml_backend_buffer_i cuda_backend_buffer_interface = {+static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {+ /* .get_name = */ ggml_backend_cuda_buffer_get_name,/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,/* .get_base = */ ggml_backend_cuda_buffer_get_base,/* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,/* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,/* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,- /* .cpy_tensor_from = */ NULL,- /* .cpy_tensor_to = */ NULL,+ /* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,/* .clear = */ ggml_backend_cuda_buffer_clear,+ /* .reset = */ NULL,};// cuda buffer type+struct ggml_backend_cuda_buffer_type_context {+ int device;+ std::string name;+};++GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) {+ ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;++ return ctx->name.c_str();+}GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- int device = (int) (intptr_t) buft->context;+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;- ggml_cuda_set_device(device);+ ggml_cuda_set_device(buft_ctx->device);size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0void * dev_ptr;- CUDA_CHECK(cudaMalloc(&dev_ptr, size));+ cudaError_t err = cudaMalloc(&dev_ptr, size);+ if (err != cudaSuccess) {+ fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err));+ return nullptr;+ }- ggml_backend_buffer_context_cuda * ctx = new ggml_backend_buffer_context_cuda(device, dev_ptr);+ ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);- return g_backend->ggml_backend_buffer_init(buft, cuda_backend_buffer_interface, ctx, size);+ return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);}GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {@@ -10430,13 +10598,8 @@ GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backendUNUSED(buft);}-GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, ggml_tensor * tensor) {- int64_t row_low = 0;- int64_t row_high = ggml_nrows(tensor);- int64_t nrows_split = row_high - row_low;-- size_t size = ggml_nbytes_split(tensor, nrows_split);-+GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ size_t size = ggml_nbytes(tensor);int64_t ne0 = tensor->ne[0];if (ggml_is_quantized(tensor->type)) {@@ -10451,21 +10614,32 @@ GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backen}GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {- return ggml_backend_is_cuda(backend);+ if (!ggml_backend_is_cuda(backend)) {+ return false;+ }- UNUSED(buft);+ ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;++ return buft_ctx->device == cuda_ctx->device;}static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {+ /* .get_name = */ ggml_backend_cuda_buffer_type_name,/* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,/* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,/* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,- /* .is_host = */ nullptr,+ /* .is_host = */ NULL,};GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {- static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];+ // FIXME: this is not thread safe+ if (device >= ggml_backend_cuda_get_device_count()) {+ return nullptr;+ }++ static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];static bool ggml_backend_cuda_buffer_type_initialized = false;@@ -10473,7 +10647,7 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {ggml_backend_cuda_buffer_types[i] = {/* .iface = */ ggml_backend_cuda_buffer_type_interface,- /* .context = */ (ggml_backend_buffer_type_context_t) (intptr_t) i,+ /* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},};}ggml_backend_cuda_buffer_type_initialized = true;@@ -10482,8 +10656,306 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {return &ggml_backend_cuda_buffer_types[device];}+// cuda split buffer++struct ggml_backend_cuda_split_buffer_context {+ ~ggml_backend_cuda_split_buffer_context() {+ for (ggml_tensor_extra_gpu * extra : tensor_extras) {+ for (int id = 0; id < g_device_count; ++id) {+ for (int64_t is = 0; is < MAX_STREAMS; ++is) {+ if (extra->events[id][is] != nullptr) {+ CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));+ }+ }+ if (extra->data_device[id] != nullptr) {+ CUDA_CHECK(cudaFree(extra->data_device[id]));+ }+ }+ delete extra;+ }+ }++ std::vector<ggml_tensor_extra_gpu *> tensor_extras;+};++GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) {+ return GGML_CUDA_NAME "_Split";++ UNUSED(buffer);+}++// unused at the moment+//static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {+// return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;+//}++GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;+ delete ctx;+}++GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {+ // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced+ return (void *)0x1000;++ UNUSED(buffer);+}++GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+ GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported++ ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;+ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;++ const int64_t ne0 = tensor->ne[0];++ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};++ ctx->tensor_extras.push_back(extra);++ for (int id = 0; id < g_device_count; ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);++ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }++ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;++ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }++ // FIXME: do not crash if cudaMalloc fails+ // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first+ ggml_cuda_set_device(id);+ char * buf;+ CUDA_CHECK(cudaMalloc(&buf, size));++ // set padding to 0 to avoid possible NaN values+ if (size > original_size) {+ CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));+ }++ extra->data_device[id] = buf;++ for (int64_t is = 0; is < MAX_STREAMS; ++is) {+ CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));+ }+ }+ tensor->backend = GGML_BACKEND_GPU_SPLIT;+ tensor->extra = extra;+}++GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ // split tensors must always be set in their entirety at once+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(size == ggml_nbytes(tensor));++ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;++ const int64_t ne0 = tensor->ne[0];+ const size_t nb1 = tensor->nb[1];+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;++ for (int id = 0; id < g_device_count; ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);++ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }++ const size_t offset_split = row_low*nb1;+ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;++ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }++ const char * buf_host = (const char *)data + offset_split;+ CUDA_CHECK(cudaMemcpy(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice));+ }+}++GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ // split tensors must always be set in their entirety at once+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(size == ggml_nbytes(tensor));++ ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;++ const int64_t ne0 = tensor->ne[0];+ const size_t nb1 = tensor->nb[1];+ ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;++ for (int id = 0; id < g_device_count; ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);++ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }++ const size_t offset_split = row_low*nb1;+ size_t size = ggml_nbytes_split(tensor, nrows_split);+ const size_t original_size = size;++ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }++ char * buf_host = (char *)data + offset_split;+ CUDA_CHECK(cudaMemcpy(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost));+ }+}++GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ UNUSED(buffer);+ UNUSED(value);+}++static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {+ /* .get_name = */ ggml_backend_cuda_split_buffer_get_name,+ /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,+ /* .get_base = */ ggml_backend_cuda_split_buffer_get_base,+ /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,+ /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,+ /* .cpy_tensor = */ NULL,+ /* .clear = */ ggml_backend_cuda_split_buffer_clear,+ /* .reset = */ NULL,+};++// cuda split buffer type++GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) {+ return GGML_CUDA_NAME "_Split";++ UNUSED(buft);+}++GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point+ // instead, we allocate them for each tensor separately in init_tensor+ // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,+ // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.+ ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();++ return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);+}++GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ return 128;++ UNUSED(buft);+}++GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;++ size_t total_size = 0;++ const int64_t ne0 = tensor->ne[0];++ for (int id = 0; id < g_device_count; ++id) {+ int64_t row_low, row_high;+ get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);++ int64_t nrows_split = row_high - row_low;+ if (nrows_split == 0) {+ continue;+ }++ total_size += ggml_nbytes_split(tensor, nrows_split);++ // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses+ if (ne0 % MATRIX_ROW_PADDING != 0) {+ total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);+ }+ }++ return total_size;+}++GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {+ return ggml_backend_is_cuda(backend);++ UNUSED(buft);+}++GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {+ return false;++ UNUSED(buft);+}++static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {+ /* .get_name = */ ggml_backend_cuda_split_buffer_type_name,+ /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,+ /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,+ /* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend,+ /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,+};++GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) {+ // FIXME: this is not thread safe+ static std::map<std::array<float, GGML_CUDA_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map;++ std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split_arr = {};++ bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });+ if (all_zero) {+ tensor_split_arr = g_default_tensor_split;+ } else {+ float split_sum = 0.0f;+ for (int i = 0; i < g_device_count; ++i) {+ tensor_split_arr[i] = split_sum;+ split_sum += tensor_split[i];+ }+ for (int i = 0; i < g_device_count; ++i) {+ tensor_split_arr[i] /= split_sum;+ }+ }++ auto it = buft_map.find(tensor_split_arr);+ if (it != buft_map.end()) {+ return &it->second;+ }++ struct ggml_backend_buffer_type buft {+ /* .iface = */ ggml_backend_cuda_split_buffer_type_interface,+ /* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr},+ };++ auto result = buft_map.emplace(tensor_split_arr, buft);+ return &result.first->second;+}+// host buffer type+GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {+ return GGML_CUDA_NAME "_Host";++ UNUSED(buft);+}++GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) {+ return GGML_CUDA_NAME "_Host";++ UNUSED(buffer);+}+GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {ggml_cuda_host_free(buffer->context);}@@ -10496,9 +10968,9 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);}- // FIXME: this is a hack to avoid having to implement a new buffer type- ggml_backend_buffer_t buffer = g_backend->ggml_backend_cpu_buffer_from_ptr(ptr, size);+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);buffer->buft = buft;+ buffer->iface.get_name = ggml_backend_cuda_host_buffer_name;buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;return buffer;@@ -10507,6 +10979,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {/* .iface = */ {+ /* .get_name = */ ggml_backend_cuda_host_buffer_type_name,/* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,/* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,@@ -10521,31 +10994,27 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {// backend-struct ggml_backend_context_cuda {- int device;-};-GGML_CALL static const char * ggml_backend_cuda_name(ggml_backend_t backend) {- return GGML_CUDA_NAME;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;- UNUSED(backend);+ return cuda_ctx->name.c_str();}GGML_CALL static void ggml_backend_cuda_free(ggml_backend_t backend) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;delete cuda_ctx;delete backend;}GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;return ggml_backend_cuda_buffer_type(cuda_ctx->device);}GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);@@ -10554,7 +11023,7 @@ GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend,}GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);@@ -10562,39 +11031,27 @@ GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend,CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0]));}-GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;-- CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0]));+GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;- UNUSED(backend);-}--GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cuda_graph_plan_create(ggml_backend_t backend, ggml_cgraph * cgraph) {- GGML_ASSERT(!"not implemented");-- return nullptr;+ if (dst->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && ggml_backend_buffer_is_cuda(src->buffer)) {+ CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, g_cudaStreams[cuda_ctx->device][0]));+ return true;+ }- UNUSED(backend);- UNUSED(cgraph);+ return false;}-GGML_CALL static void ggml_backend_cuda_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {- GGML_ASSERT(!"not implemented");-- UNUSED(backend);- UNUSED(plan);-}+GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;-GGML_CALL static void ggml_backend_cuda_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {- GGML_ASSERT(!"not implemented");+ CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[cuda_ctx->device][0]));UNUSED(backend);- UNUSED(plan);}GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {- ggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;ggml_cuda_set_main_device(cuda_ctx->device);@@ -10604,53 +11061,31 @@ GGML_CALL static bool ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggfor (int i = 0; i < cgraph->n_nodes; i++) {ggml_tensor * node = cgraph->nodes[i];- if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE)+ if (node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {continue;+ }- assert(node->backend == GGML_BACKEND_GPU);+#ifndef NDEBUG+ assert(node->backend == GGML_BACKEND_GPU || node->backend == GGML_BACKEND_GPU_SPLIT);assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));assert(node->extra != nullptr);for (int j = 0; j < GGML_MAX_SRC; j++) {if (node->src[j] != nullptr) {- assert(node->src[j]->backend == GGML_BACKEND_GPU);+ assert(node->src[j]->backend == GGML_BACKEND_GPU || node->src[j]->backend == GGML_BACKEND_GPU_SPLIT);assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));assert(node->src[j]->extra != nullptr);}}+#endifbool ok = ggml_cuda_compute_forward(¶ms, node);if (!ok) {fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));}GGML_ASSERT(ok);--#if 0- if (node->type == GGML_TYPE_F32) {- cudaDeviceSynchronize();- std::vector<float> tmp(ggml_nelements(node), 0.0f);- cudaMemcpy(tmp.data(), node->data, ggml_nelements(node)*sizeof(float), cudaMemcpyDeviceToHost);- printf("\n%s (%s) (%s %s) (%s %s): ", node->name, ggml_op_name(node->op),- ggml_type_name(node->src[0]->type),- node->src[1] ? ggml_type_name(node->src[1]->type) : "none",- node->src[0]->name,- node->src[1] ? node->src[1]->name : "none");- double sum = 0.0;- double sq_sum = 0.0;- for (int i = 0; i < ggml_nelements(node); i++) {- printf("%f ", tmp[i]);- sum += tmp[i];- sq_sum += tmp[i]*tmp[i];- }- printf("\n");- printf("sum: %f, ", sum);- printf("sq_sum: %f\n", sq_sum);- }-#endif}- UNUSED(backend);-return true;}@@ -10683,6 +11118,12 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, consif (a->ne[3] != b->ne[3]) {return false;}+ ggml_type a_type = a->type;+ if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS) {+ if (b->ne[1] == 1 && ggml_nrows(b) > 1) {+ return false;+ }+ }return true;} break;case GGML_OP_GET_ROWS:@@ -10765,24 +11206,23 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, consUNUSED(backend);}-static ggml_backend_i cuda_backend_i = {+static ggml_backend_i ggml_backend_cuda_interface = {/* .get_name = */ ggml_backend_cuda_name,/* .free = */ ggml_backend_cuda_free,/* .get_default_buffer_type = */ ggml_backend_cuda_get_default_buffer_type,/* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async,/* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async,- /* .cpy_tensor_from_async = */ NULL,- /* .cpy_tensor_to_async = */ NULL,+ /* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async,/* .synchronize = */ ggml_backend_cuda_synchronize,- /* .graph_plan_create = */ ggml_backend_cuda_graph_plan_create,- /* .graph_plan_free = */ ggml_backend_cuda_graph_plan_free,- /* .graph_plan_compute = */ ggml_backend_cuda_graph_plan_compute,+ /* .graph_plan_create = */ NULL,+ /* .graph_plan_free = */ NULL,+ /* .graph_plan_compute = */ NULL,/* .graph_compute = */ ggml_backend_cuda_graph_compute,/* .supports_op = */ ggml_backend_cuda_supports_op,};GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {- ggml_init_cublas(FLAG_log_disable); // TODO: remove from ggml.c+ ggml_init_cublas(); // TODO: remove from ggml.cif (device < 0 || device >= ggml_cuda_get_device_count()) {fprintf(stderr, "%s: error: invalid device %d\n", __func__, device);@@ -10792,12 +11232,13 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {// not strictly necessary, but it may reduce the overhead of the first graph_computeggml_cuda_set_main_device(device);- ggml_backend_context_cuda * ctx = new ggml_backend_context_cuda {- /* .device = */ device+ ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context {+ /* .device = */ device,+ /* .name = */ GGML_CUDA_NAME + std::to_string(device),};ggml_backend_t cuda_backend = new ggml_backend {- /* .interface = */ cuda_backend_i,+ /* .interface = */ ggml_backend_cuda_interface,/* .context = */ ctx};@@ -10805,12 +11246,40 @@ GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {}GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) {- return backend->iface.get_name == ggml_backend_cuda_name;+ return backend && backend->iface.get_name == ggml_backend_cuda_name;+}++GGML_CALL int ggml_backend_cuda_get_device_count() {+ return ggml_cuda_get_device_count();+}++GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {+ ggml_cuda_get_device_description(device, description, description_size);+}++GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {+ ggml_cuda_set_device(device);++ CUDA_CHECK(cudaMemGetInfo(free, total));}-GGML_CALL ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {+// backend registry+GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data);return cuda_backend;UNUSED(params);}++extern "C" GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();++GGML_CALL int ggml_backend_cuda_reg_devices() {+ int device_count = ggml_cuda_get_device_count();+ //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization+ for (int i = 0; i < device_count; i++) {+ char name[128];+ snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);+ ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);+ }+ return device_count;+}diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex 59e74857..37cb0d5b 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -3,8 +3,13 @@#include "ggml.h"#include "ggml-backend.h"-#define GGML_CUDA_NAME "ROCm/CUDA"-#define GGML_CUBLAS_NAME "hipBLAS/cuBLAS/tinyBLAS"+#ifdef GGML_USE_HIPBLAS+#define GGML_CUDA_NAME "ROCm"+#define GGML_CUBLAS_NAME "hipBLAS"+#else+#define GGML_CUDA_NAME "CUDA"+#define GGML_CUBLAS_NAME "cuBLAS"+#endif#ifdef __cplusplusextern "C" {@@ -12,51 +17,37 @@ extern "C" {#define GGML_CUDA_MAX_DEVICES 16+GGML_API GGML_CALL void ggml_cuda_link(const struct ggml_backend_api * backend_api);+// Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`.-GGML_API void ggml_init_cublas(bool log_disable) GGML_CALL;-GGML_API bool ggml_cuda_supported(void);-GGML_API void ggml_cuda_link(const struct ggml_backend_api *) GGML_CALL;+GGML_API GGML_CALL void ggml_init_cublas(void);// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.-GGML_API bool ggml_cublas_loaded(void) GGML_CALL;--GGML_API void * ggml_cuda_host_malloc(size_t size) GGML_CALL;-GGML_API void ggml_cuda_host_free(void * ptr) GGML_CALL;+GGML_API GGML_CALL bool ggml_cublas_loaded(void);-GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) GGML_CALL;-GGML_API void ggml_cuda_set_tensor_split(const float * tensor_split) GGML_CALL;-GGML_API void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) GGML_CALL;-GGML_API void ggml_cuda_free_data(struct ggml_tensor * tensor) GGML_CALL;+GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size);+GGML_API GGML_CALL void ggml_cuda_host_free(void * ptr);-GGML_API void ggml_cuda_assign_buffers(struct ggml_tensor * tensor) GGML_CALL;-GGML_API void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor) GGML_CALL;-GGML_API void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor) GGML_CALL;+GGML_API GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);+GGML_API GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);-GGML_API void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor * tensor) GGML_CALL;-GGML_API void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset) GGML_CALL;-GGML_API void ggml_cuda_copy_to_device(struct ggml_tensor * tensor) GGML_CALL;--GGML_API void ggml_cuda_set_main_device(int main_device) GGML_CALL;-GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q) GGML_CALL;-GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size) GGML_CALL;-GGML_API void ggml_cuda_free_scratch(void) GGML_CALL;-GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) GGML_CALL;--GGML_API int ggml_cuda_get_device_count(void) GGML_CALL;-GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size) GGML_CALL;+GGML_API GGML_CALL int ggml_cuda_get_device_count(void);+GGML_API GGML_CALL void ggml_cuda_get_device_description(int device, char * description, size_t description_size);// backend API-GGML_API ggml_backend_t ggml_backend_cuda_init(int device) GGML_CALL;--GGML_API bool ggml_backend_is_cuda(ggml_backend_t backend) GGML_CALL;-GGML_API int ggml_backend_cuda_get_device(ggml_backend_t backend) GGML_CALL;+GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);-GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) GGML_CALL;+GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);-// pinned host buffer for use with CPU backend for faster copies between CPU and GPU-GGML_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void) GGML_CALL;+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);+// split tensor buffer that splits matrices by rows across multiple devices+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);+// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);-GGML_API ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) GGML_CALL;+GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);+GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);+GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);#ifdef __cplusplus}diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex d116adfe..50000b62 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -5,6 +5,7 @@// GGML internal header#include <assert.h>+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/#include <stddef.h>#include <stdbool.h>#include <string.h> // memcpy@@ -232,6 +233,8 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {#define GGML_HASHTABLE_FULL ((size_t)-1)#define GGML_HASHTABLE_ALREADY_EXISTS ((size_t)-2)+struct ggml_hash_set ggml_hash_set_new(size_t size);+bool ggml_hash_contains (const struct ggml_hash_set hash_set, struct ggml_tensor * key);// returns GGML_HASHTABLE_FULL if table is full, otherwise the current index of the key or where it should be inserteddiff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.hindex bc16462e..7da759ff 100644--- a/llama.cpp/ggml-metal.h+++ b/llama.cpp/ggml-metal.h@@ -27,7 +27,6 @@// max memory buffers that can be mapped to the device#define GGML_METAL_MAX_BUFFERS 64-#define GGML_METAL_MAX_COMMAND_BUFFERS 32struct ggml_tensor;struct ggml_cgraph;@@ -36,75 +35,24 @@ struct ggml_cgraph;extern "C" {#endif-//-// internal API-// temporary exposed to user-code-//--struct ggml_metal_context;--bool ggml_metal_supported(void);void ggml_metal_link(const struct ggml_backend_api *);-void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);--// number of command buffers to use-struct ggml_metal_context * ggml_metal_init(int n_cb);-void ggml_metal_free(struct ggml_metal_context * ctx);--void * ggml_metal_host_malloc(size_t n);-void ggml_metal_host_free (void * data);--// set the number of command buffers to use-void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb);--// creates a mapping between a host memory buffer and a device memory buffer-// - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute-// - the mapping is used during computation to determine the arguments of the compute kernels-// - you don't need to keep the host memory buffer allocated as it is never accessed by Metal-// - max_size specifies the maximum size of a tensor and is used to create shared views such-// that it is guaranteed that the tensor will fit in at least one of the views-//-bool ggml_metal_add_buffer(- struct ggml_metal_context * ctx,- const char * name,- void * data,- size_t size,- size_t max_size);--// set data from host memory into the device-void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);--// get data from the device into host memory-void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);--// try to find operations that can be run concurrently in the graph-// you should run it again if the topology of your graph changes-void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf, bool check_mem);--// if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized-int ggml_metal_if_optimized(struct ggml_metal_context * ctx);--// output the concur_list for ggml_alloc-int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx);--// same as ggml_graph_compute but uses Metal-// creates gf->n_threads command buffers in parallel-bool ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);//// backend API// user-code should use only these functions//+GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);+GGML_API ggml_backend_t ggml_backend_metal_init(void);GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);-GGML_API ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) GGML_CALL;+GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);-GGML_API ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) GGML_CALL;+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);// helper to check if the device supports a specific family// ideally, the user code should be doing these checksdiff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 941f75d4..95d2a9c8 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -10,6 +10,28 @@#import <Metal/Metal.h>+static const struct ggml_backend_api *g_backend;++#define FLAG_log_disable (*g_backend->FLAG_log_disable)+#define ggml_backend_buffer_init g_backend->ggml_backend_buffer_init+#define ggml_backend_buffer_is_host g_backend->ggml_backend_buffer_is_host+#define ggml_backend_is_cpu g_backend->ggml_backend_is_cpu+#define ggml_blck_size g_backend->ggml_blck_size+#define ggml_get_unary_op g_backend->ggml_get_unary_op+#define ggml_is_contiguous g_backend->ggml_is_contiguous+#define ggml_is_quantized g_backend->ggml_is_quantized+#define ggml_is_transposed g_backend->ggml_is_transposed+#define ggml_nbytes g_backend->ggml_nbytes+#define ggml_nelements g_backend->ggml_nelements+#define ggml_nrows g_backend->ggml_nrows+#define ggml_op_desc g_backend->ggml_op_desc+#define ggml_op_name g_backend->ggml_op_name+#define ggml_type_size g_backend->ggml_type_size++void ggml_metal_link(const struct ggml_backend_api *backend_api) {+ g_backend = backend_api;+}+#undef MIN#undef MAX#define MIN(a, b) ((a) < (b) ? (a) : (b))@@ -27,31 +49,134 @@#define UNUSED(x) (void)(x)-#define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE)+#define GGML_METAL_MAX_KERNELS 256-static const struct ggml_backend_api *g_backend;-#define ggml_type_size g_backend->ggml_type_size-#define ggml_blck_size g_backend->ggml_blck_size-#define ggml_is_transposed g_backend->ggml_is_transposed-#define ggml_nbytes g_backend->ggml_nbytes-#define ggml_get_unary_op g_backend->ggml_get_unary_op-#define ggml_nelements g_backend->ggml_nelements-#define ggml_nrows g_backend->ggml_nrows-#define ggml_is_contiguous g_backend->ggml_is_contiguous-#define ggml_op_name g_backend->ggml_op_name-#define ggml_op_desc g_backend->ggml_op_desc--void ggml_metal_link(const struct ggml_backend_api *backend_api) {- g_backend = backend_api;-}--struct ggml_metal_buffer {- const char * name;-- void * data;- size_t size;+struct ggml_metal_kernel {+ id<MTLFunction> function;+ id<MTLComputePipelineState> pipeline;+};- id<MTLBuffer> metal;+enum ggml_metal_kernel_type {+ GGML_METAL_KERNEL_TYPE_ADD,+ GGML_METAL_KERNEL_TYPE_ADD_ROW,+ GGML_METAL_KERNEL_TYPE_MUL,+ GGML_METAL_KERNEL_TYPE_MUL_ROW,+ GGML_METAL_KERNEL_TYPE_DIV,+ GGML_METAL_KERNEL_TYPE_DIV_ROW,+ GGML_METAL_KERNEL_TYPE_SCALE,+ GGML_METAL_KERNEL_TYPE_SCALE_4,+ GGML_METAL_KERNEL_TYPE_TANH,+ GGML_METAL_KERNEL_TYPE_RELU,+ GGML_METAL_KERNEL_TYPE_GELU,+ GGML_METAL_KERNEL_TYPE_GELU_QUICK,+ GGML_METAL_KERNEL_TYPE_SILU,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX,+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_4,+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F16,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS,+ GGML_METAL_KERNEL_TYPE_GET_ROWS_I32,+ GGML_METAL_KERNEL_TYPE_RMS_NORM,+ GGML_METAL_KERNEL_TYPE_GROUP_NORM,+ GGML_METAL_KERNEL_TYPE_NORM,+ GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW,+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32,+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32,+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW,+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32,+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32,+ GGML_METAL_KERNEL_TYPE_ROPE_F32,+ GGML_METAL_KERNEL_TYPE_ROPE_F16,+ GGML_METAL_KERNEL_TYPE_ALIBI_F32,+ GGML_METAL_KERNEL_TYPE_IM2COL_F16,+ GGML_METAL_KERNEL_TYPE_UPSCALE_F32,+ GGML_METAL_KERNEL_TYPE_PAD_F32,+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,+ GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,+ GGML_METAL_KERNEL_TYPE_CPY_F32_F16,+ GGML_METAL_KERNEL_TYPE_CPY_F32_F32,+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0,+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1,+ //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,+ //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,+ GGML_METAL_KERNEL_TYPE_CPY_F16_F16,+ GGML_METAL_KERNEL_TYPE_CPY_F16_F32,+ GGML_METAL_KERNEL_TYPE_CONCAT,+ GGML_METAL_KERNEL_TYPE_SQR,+ GGML_METAL_KERNEL_TYPE_SUM_ROWS,++ GGML_METAL_KERNEL_TYPE_COUNT};struct ggml_metal_context {@@ -61,137 +186,12 @@ struct ggml_metal_context {id<MTLCommandQueue> queue;id<MTLLibrary> library;- id<MTLCommandBuffer> command_buffers [GGML_METAL_MAX_COMMAND_BUFFERS];- id<MTLComputeCommandEncoder> command_encoders[GGML_METAL_MAX_COMMAND_BUFFERS];-dispatch_queue_t d_queue;- int n_buffers;- struct ggml_metal_buffer buffers[GGML_METAL_MAX_BUFFERS];-- int concur_list[GGML_MAX_CONCUR];- int concur_list_len;-- // custom kernels-#define GGML_METAL_DECL_KERNEL(name) \- id<MTLFunction> function_##name; \- id<MTLComputePipelineState> pipeline_##name-- GGML_METAL_DECL_KERNEL(add);- GGML_METAL_DECL_KERNEL(add_row); // TODO: avoid this extra kernel, instead extend the "add" kernel to support broadcast- GGML_METAL_DECL_KERNEL(mul);- GGML_METAL_DECL_KERNEL(mul_row); // TODO: avoid this extra kernel, instead extend the "mul" kernel to support broadcast- GGML_METAL_DECL_KERNEL(div);- GGML_METAL_DECL_KERNEL(div_row);- GGML_METAL_DECL_KERNEL(scale);- GGML_METAL_DECL_KERNEL(scale_4);- GGML_METAL_DECL_KERNEL(tanh);- GGML_METAL_DECL_KERNEL(relu);- GGML_METAL_DECL_KERNEL(gelu);- GGML_METAL_DECL_KERNEL(gelu_quick);- GGML_METAL_DECL_KERNEL(silu);- GGML_METAL_DECL_KERNEL(soft_max);- GGML_METAL_DECL_KERNEL(soft_max_4);- GGML_METAL_DECL_KERNEL(diag_mask_inf);- GGML_METAL_DECL_KERNEL(diag_mask_inf_8);- GGML_METAL_DECL_KERNEL(get_rows_f32);- GGML_METAL_DECL_KERNEL(get_rows_f16);- GGML_METAL_DECL_KERNEL(get_rows_q4_0);- GGML_METAL_DECL_KERNEL(get_rows_q4_1);- GGML_METAL_DECL_KERNEL(get_rows_q5_0);- GGML_METAL_DECL_KERNEL(get_rows_q5_1);- GGML_METAL_DECL_KERNEL(get_rows_q8_0);- GGML_METAL_DECL_KERNEL(get_rows_q2_K);- GGML_METAL_DECL_KERNEL(get_rows_q3_K);- GGML_METAL_DECL_KERNEL(get_rows_q4_K);- GGML_METAL_DECL_KERNEL(get_rows_q5_K);- GGML_METAL_DECL_KERNEL(get_rows_q6_K);- GGML_METAL_DECL_KERNEL(get_rows_i32);- GGML_METAL_DECL_KERNEL(get_rows_iq2_xxs);- GGML_METAL_DECL_KERNEL(rms_norm);- GGML_METAL_DECL_KERNEL(group_norm);- GGML_METAL_DECL_KERNEL(norm);- GGML_METAL_DECL_KERNEL(mul_mv_f32_f32);- GGML_METAL_DECL_KERNEL(mul_mv_f16_f16);- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32);- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_1row);- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_l4);- GGML_METAL_DECL_KERNEL(mul_mv_q4_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q4_1_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q5_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q5_1_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q8_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q2_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q3_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q4_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_iq2_xxs_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_f32_f32);- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f16);- GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32);- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_1row);- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_l4);- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_1_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_1_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q8_0_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q2_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q3_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_q6_K_f32);- GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xxs_f32);- GGML_METAL_DECL_KERNEL(mul_mm_f32_f32);- GGML_METAL_DECL_KERNEL(mul_mm_f16_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q4_1_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q5_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q5_1_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q8_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q2_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q3_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q4_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_iq2_xxs_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_f32_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_f16_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_1_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_1_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q8_0_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q2_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q3_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_q6_K_f32);- GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xxs_f32);- GGML_METAL_DECL_KERNEL(rope_f32);- GGML_METAL_DECL_KERNEL(rope_f16);- GGML_METAL_DECL_KERNEL(alibi_f32);- GGML_METAL_DECL_KERNEL(im2col_f16);- GGML_METAL_DECL_KERNEL(upscale_f32);- GGML_METAL_DECL_KERNEL(pad_f32);- GGML_METAL_DECL_KERNEL(argsort_f32_i32_asc);- GGML_METAL_DECL_KERNEL(argsort_f32_i32_desc);- GGML_METAL_DECL_KERNEL(leaky_relu_f32);- GGML_METAL_DECL_KERNEL(cpy_f32_f16);- GGML_METAL_DECL_KERNEL(cpy_f32_f32);- GGML_METAL_DECL_KERNEL(cpy_f32_q8_0);- GGML_METAL_DECL_KERNEL(cpy_f32_q4_0);- GGML_METAL_DECL_KERNEL(cpy_f32_q4_1);- //GGML_METAL_DECL_KERNEL(cpy_f32_q5_0);- //GGML_METAL_DECL_KERNEL(cpy_f32_q5_1);- GGML_METAL_DECL_KERNEL(cpy_f16_f16);- GGML_METAL_DECL_KERNEL(cpy_f16_f32);- GGML_METAL_DECL_KERNEL(concat);- GGML_METAL_DECL_KERNEL(sqr);- GGML_METAL_DECL_KERNEL(sum_rows);--#undef GGML_METAL_DECL_KERNEL+ struct ggml_metal_kernel kernels[GGML_METAL_MAX_KERNELS];++ bool support_simdgroup_reduction;+ bool support_simdgroup_mm;};// MSL code@@ -205,8 +205,9 @@ struct ggml_metal_context {@implementation GGMLMetalClass@end-static void ggml_metal_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {+ if (FLAG_log_disable) return;+fprintf(stderr, "%s", msg);UNUSED(level);@@ -216,11 +217,6 @@ static void ggml_metal_default_log_callback(enum ggml_log_level level, const chaggml_log_callback ggml_metal_log_callback = ggml_metal_default_log_callback;void * ggml_metal_log_user_data = NULL;-void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {- ggml_metal_log_callback = log_callback;- ggml_metal_log_user_data = user_data;-}-GGML_ATTRIBUTE_FORMAT(2, 3)static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){if (ggml_metal_log_callback != NULL) {@@ -231,46 +227,50 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){if (len < 128) {ggml_metal_log_callback(level, buffer, ggml_metal_log_user_data);} else {- char* buffer2 = g_backend->malloc(len+1);+ char* buffer2 = malloc(len+1);va_end(args);va_start(args, format);vsnprintf(buffer2, len+1, format, args);buffer2[len] = 0;ggml_metal_log_callback(level, buffer2, ggml_metal_log_user_data);- g_backend->free(buffer2);+ free(buffer2);}va_end(args);}}-struct ggml_metal_context * ggml_metal_init(int n_cb) {- GGML_METAL_LOG_INFO("%s: allocating\n", __func__);+static void * ggml_metal_host_malloc(size_t n) {+ void * data = NULL;+ const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);+ if (result != 0) {+ GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);+ return NULL;+ }- id<MTLDevice> device;- NSString * s;+ return data;+}-#if TARGET_OS_OSX+static struct ggml_metal_context * ggml_metal_init(int n_cb) {+ GGML_METAL_LOG_INFO("%s: allocating\n", __func__);++#if TARGET_OS_OSX && !GGML_METAL_NDEBUG// Show all the Metal device instances in the systemNSArray * devices = MTLCopyAllDevices();- for (device in devices) {- s = [device name];- GGML_METAL_LOG_INFO("%s: found device: %s\n", __func__, [s UTF8String]);+ for (id<MTLDevice> device in devices) {+ GGML_METAL_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);}+ [devices release]; // since it was created by a *Copy* C method#endif// Pick and show default Metal device- device = MTLCreateSystemDefaultDevice();- s = [device name];- GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [s UTF8String]);+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();+ GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);// Configure context- struct ggml_metal_context * ctx = g_backend->malloc(sizeof(struct ggml_metal_context));+ struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));ctx->device = device;ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);ctx->queue = [ctx->device newCommandQueue];- ctx->n_buffers = 0;- ctx->concur_list_len = 0;-ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);// load library@@ -288,6 +288,10 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {NSURL * libURL = [NSURL fileURLWithPath:libPath];GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];+ if (error) {+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);+ return NULL;+ }} else {GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);@@ -312,43 +316,73 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {return NULL;}- MTLCompileOptions* options = nil;+ @autoreleasepool {+ // dictionary of preprocessor macros+ NSMutableDictionary * prep = [NSMutableDictionary dictionary];+#ifdef GGML_QKK_64- options = [MTLCompileOptions new];- options.preprocessorMacros = @{ @"QK_K" : @(64) };+ prep[@"QK_K"] = @(64);#endif- // try to disable fast-math- // NOTE: this seems to have no effect whatsoever- // instead, in order to disable fast-math, we have to build default.metallib from the command line- // using xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air- // and go through the "pre-compiled library found" path above- //[options setFastMathEnabled:false];-- ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];- }- if (error) {- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);- return NULL;+ MTLCompileOptions* options = [MTLCompileOptions new];+ options.preprocessorMacros = prep;++ //[options setFastMathEnabled:false];++ ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];+ if (error) {+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);+ return NULL;+ }+ }}}-#if TARGET_OS_OSX// print MTL GPU family:GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);+ const NSInteger MTLGPUFamilyMetal3 = 5001;+// determine max supported GPU family// https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {- if ([ctx->device supportsFamily:i]) {- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);- break;+ {+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {+ if ([ctx->device supportsFamily:i]) {+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);+ break;+ }+ }++ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {+ if ([ctx->device supportsFamily:i]) {+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);+ break;+ }+ }++ for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) {+ if ([ctx->device supportsFamily:i]) {+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i);+ break;+ }}}+ ctx->support_simdgroup_reduction = [ctx->device supportsFamily:MTLGPUFamilyApple7];+ ctx->support_simdgroup_reduction |= [ctx->device supportsFamily:MTLGPUFamilyMetal3];++ ctx->support_simdgroup_mm = [ctx->device supportsFamily:MTLGPUFamilyApple7];++ GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false");+ GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false");GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");- GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);++#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)+ if (@available(macOS 10.12, iOS 16.0, *)) {+ GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);+ }+#elif TARGET_OS_OSXif (ctx->device.maxTransferRate != 0) {GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6);} else {@@ -360,267 +394,165 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {{NSError * error = nil;+ for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {+ ctx->kernels[i].function = nil;+ ctx->kernels[i].pipeline = nil;+ }+/*- GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) ctx->pipeline_##name, \- (int) ctx->pipeline_##name.maxTotalThreadsPerThreadgroup, \- (int) ctx->pipeline_##name.threadExecutionWidth); \+ GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \+ (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \+ (int) kernel->pipeline.threadExecutionWidth); \*/-#define GGML_METAL_ADD_KERNEL(name) \- ctx->function_##name = [ctx->library newFunctionWithName:@"kernel_"#name]; \- ctx->pipeline_##name = [ctx->device newComputePipelineStateWithFunction:ctx->function_##name error:&error]; \- if (error) { \- GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \- return NULL; \+#define GGML_METAL_ADD_KERNEL(e, name, supported) \+ if (supported) { \+ struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \+ kernel->function = [ctx->library newFunctionWithName:@"kernel_"#name]; \+ kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:kernel->function error:&error]; \+ if (error) { \+ GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \+ return NULL; \+ } \+ } else { \+ GGML_METAL_LOG_WARN("%s: skipping %-32s (not supported)\n", __func__, "kernel_"#name); \}- GGML_METAL_ADD_KERNEL(add);- GGML_METAL_ADD_KERNEL(add_row);- GGML_METAL_ADD_KERNEL(mul);- GGML_METAL_ADD_KERNEL(mul_row);- GGML_METAL_ADD_KERNEL(div);- GGML_METAL_ADD_KERNEL(div_row);- GGML_METAL_ADD_KERNEL(scale);- GGML_METAL_ADD_KERNEL(scale_4);- GGML_METAL_ADD_KERNEL(tanh);- GGML_METAL_ADD_KERNEL(relu);- GGML_METAL_ADD_KERNEL(gelu);- GGML_METAL_ADD_KERNEL(gelu_quick);- GGML_METAL_ADD_KERNEL(silu);- GGML_METAL_ADD_KERNEL(soft_max);- GGML_METAL_ADD_KERNEL(soft_max_4);- GGML_METAL_ADD_KERNEL(diag_mask_inf);- GGML_METAL_ADD_KERNEL(diag_mask_inf_8);- GGML_METAL_ADD_KERNEL(get_rows_f32);- GGML_METAL_ADD_KERNEL(get_rows_f16);- GGML_METAL_ADD_KERNEL(get_rows_q4_0);- GGML_METAL_ADD_KERNEL(get_rows_q4_1);- GGML_METAL_ADD_KERNEL(get_rows_q5_0);- GGML_METAL_ADD_KERNEL(get_rows_q5_1);- GGML_METAL_ADD_KERNEL(get_rows_q8_0);- GGML_METAL_ADD_KERNEL(get_rows_q2_K);- GGML_METAL_ADD_KERNEL(get_rows_q3_K);- GGML_METAL_ADD_KERNEL(get_rows_q4_K);- GGML_METAL_ADD_KERNEL(get_rows_q5_K);- GGML_METAL_ADD_KERNEL(get_rows_q6_K);- GGML_METAL_ADD_KERNEL(get_rows_i32);- GGML_METAL_ADD_KERNEL(get_rows_iq2_xxs);- GGML_METAL_ADD_KERNEL(rms_norm);- GGML_METAL_ADD_KERNEL(group_norm);- GGML_METAL_ADD_KERNEL(norm);- GGML_METAL_ADD_KERNEL(mul_mv_f32_f32);- GGML_METAL_ADD_KERNEL(mul_mv_f16_f16);- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32);- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_1row);- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_l4);- GGML_METAL_ADD_KERNEL(mul_mv_q4_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q4_1_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q5_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q5_1_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q8_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q2_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q3_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q4_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_iq2_xxs_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_f32_f32);- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f16);- GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32);- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_1row);- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_l4);- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_1_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_1_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q8_0_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q2_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q3_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_q6_K_f32);- GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xxs_f32);- if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {- GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);- GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q4_1_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q5_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q5_1_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q2_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q3_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_iq2_xxs_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_f32_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_f16_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_1_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_1_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q8_0_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q2_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q3_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_q6_K_f32);- GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xxs_f32);- }- GGML_METAL_ADD_KERNEL(rope_f32);- GGML_METAL_ADD_KERNEL(rope_f16);- GGML_METAL_ADD_KERNEL(alibi_f32);- GGML_METAL_ADD_KERNEL(im2col_f16);- GGML_METAL_ADD_KERNEL(upscale_f32);- GGML_METAL_ADD_KERNEL(pad_f32);- GGML_METAL_ADD_KERNEL(argsort_f32_i32_asc);- GGML_METAL_ADD_KERNEL(argsort_f32_i32_desc);- GGML_METAL_ADD_KERNEL(leaky_relu_f32);- GGML_METAL_ADD_KERNEL(cpy_f32_f16);- GGML_METAL_ADD_KERNEL(cpy_f32_f32);- GGML_METAL_ADD_KERNEL(cpy_f32_q8_0);- GGML_METAL_ADD_KERNEL(cpy_f32_q4_0);- GGML_METAL_ADD_KERNEL(cpy_f32_q4_1);- //GGML_METAL_ADD_KERNEL(cpy_f32_q5_0);- //GGML_METAL_ADD_KERNEL(cpy_f32_q5_1);- GGML_METAL_ADD_KERNEL(cpy_f16_f16);- GGML_METAL_ADD_KERNEL(cpy_f16_f32);- GGML_METAL_ADD_KERNEL(concat);- GGML_METAL_ADD_KERNEL(sqr);- GGML_METAL_ADD_KERNEL(sum_rows);--#undef GGML_METAL_ADD_KERNEL+ // simd_sum and simd_max requires MTLGPUFamilyApple7++ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX, soft_max, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, soft_max_4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);}return ctx;}-void ggml_metal_free(struct ggml_metal_context * ctx) {+static void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);-#define GGML_METAL_DEL_KERNEL(name) \- [ctx->function_##name release]; \- [ctx->pipeline_##name release];-- GGML_METAL_DEL_KERNEL(add);- GGML_METAL_DEL_KERNEL(add_row);- GGML_METAL_DEL_KERNEL(mul);- GGML_METAL_DEL_KERNEL(mul_row);- GGML_METAL_DEL_KERNEL(div);- GGML_METAL_DEL_KERNEL(div_row);- GGML_METAL_DEL_KERNEL(scale);- GGML_METAL_DEL_KERNEL(scale_4);- GGML_METAL_DEL_KERNEL(tanh);- GGML_METAL_DEL_KERNEL(relu);- GGML_METAL_DEL_KERNEL(gelu);- GGML_METAL_DEL_KERNEL(gelu_quick);- GGML_METAL_DEL_KERNEL(silu);- GGML_METAL_DEL_KERNEL(soft_max);- GGML_METAL_DEL_KERNEL(soft_max_4);- GGML_METAL_DEL_KERNEL(diag_mask_inf);- GGML_METAL_DEL_KERNEL(diag_mask_inf_8);- GGML_METAL_DEL_KERNEL(get_rows_f32);- GGML_METAL_DEL_KERNEL(get_rows_f16);- GGML_METAL_DEL_KERNEL(get_rows_q4_0);- GGML_METAL_DEL_KERNEL(get_rows_q4_1);- GGML_METAL_DEL_KERNEL(get_rows_q5_0);- GGML_METAL_DEL_KERNEL(get_rows_q5_1);- GGML_METAL_DEL_KERNEL(get_rows_q8_0);- GGML_METAL_DEL_KERNEL(get_rows_q2_K);- GGML_METAL_DEL_KERNEL(get_rows_q3_K);- GGML_METAL_DEL_KERNEL(get_rows_q4_K);- GGML_METAL_DEL_KERNEL(get_rows_q5_K);- GGML_METAL_DEL_KERNEL(get_rows_q6_K);- GGML_METAL_DEL_KERNEL(get_rows_i32);- GGML_METAL_DEL_KERNEL(get_rows_iq2_xxs);- GGML_METAL_DEL_KERNEL(rms_norm);- GGML_METAL_DEL_KERNEL(group_norm);- GGML_METAL_DEL_KERNEL(norm);- GGML_METAL_DEL_KERNEL(mul_mv_f32_f32);- GGML_METAL_DEL_KERNEL(mul_mv_f16_f16);- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32);- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_1row);- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_l4);- GGML_METAL_DEL_KERNEL(mul_mv_q4_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q4_1_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q5_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q5_1_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q8_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q2_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q3_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q4_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_iq2_xxs_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_f32_f32);- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f16);- GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32);- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_1row);- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_l4);- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_1_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_1_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q8_0_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q2_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q3_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_q6_K_f32);- GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xxs_f32);- if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {- GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);- GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q4_1_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q5_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q5_1_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q2_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q3_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_iq2_xxs_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_f32_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_f16_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_1_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_1_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q8_0_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q2_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q3_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_q6_K_f32);- GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xxs_f32);- }- GGML_METAL_DEL_KERNEL(rope_f32);- GGML_METAL_DEL_KERNEL(rope_f16);- GGML_METAL_DEL_KERNEL(alibi_f32);- GGML_METAL_DEL_KERNEL(im2col_f16);- GGML_METAL_DEL_KERNEL(upscale_f32);- GGML_METAL_DEL_KERNEL(pad_f32);- GGML_METAL_DEL_KERNEL(argsort_f32_i32_asc);- GGML_METAL_DEL_KERNEL(argsort_f32_i32_desc);- GGML_METAL_DEL_KERNEL(leaky_relu_f32);- GGML_METAL_DEL_KERNEL(cpy_f32_f16);- GGML_METAL_DEL_KERNEL(cpy_f32_f32);- GGML_METAL_DEL_KERNEL(cpy_f32_q8_0);- GGML_METAL_DEL_KERNEL(cpy_f32_q4_0);- GGML_METAL_DEL_KERNEL(cpy_f32_q4_1);- //GGML_METAL_DEL_KERNEL(cpy_f32_q5_0);- //GGML_METAL_DEL_KERNEL(cpy_f32_q5_1);- GGML_METAL_DEL_KERNEL(cpy_f16_f16);- GGML_METAL_DEL_KERNEL(cpy_f16_f32);- GGML_METAL_DEL_KERNEL(concat);- GGML_METAL_DEL_KERNEL(sqr);- GGML_METAL_DEL_KERNEL(sum_rows);--#undef GGML_METAL_DEL_KERNEL-- for (int i = 0; i < ctx->n_buffers; ++i) {- [ctx->buffers[i].metal release];++ for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {+ if (ctx->kernels[i].pipeline) {+ [ctx->kernels[i].pipeline release];+ }++ if (ctx->kernels[i].function) {+ [ctx->kernels[i].function release];+ }}[ctx->library release];@@ -629,34 +561,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {dispatch_release(ctx->d_queue);- g_backend->free(ctx);-}--void * ggml_metal_host_malloc(size_t n) {- void * data = NULL;- const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);- if (result != 0) {- GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);- return NULL;- }-- return data;-}--void ggml_metal_host_free(void * data) {- free(data);-}--void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {- ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);-}--int ggml_metal_if_optimized(struct ggml_metal_context * ctx) {- return ctx->concur_list_len;-}--int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) {- return ctx->concur_list;+ free(ctx);}// temporarily defined here for compatibility between ggml-backend and the old API@@ -682,260 +587,35 @@ struct ggml_backend_metal_buffer_context {// the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the// Metal buffer based on the host memory pointer//-static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, struct ggml_tensor * t, size_t * offs) {+static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offs) {//GGML_METAL_LOG_INFO("%s: data tensor '%16s', offs_data = %8ld, offs_eval = %8ld, offs_cach = %8ld\n", __func__, t->name, offs_data, offs_eval, offs_cach);const int64_t tsize = ggml_nbytes(t);ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;- // compatibility with ggml-backend- if (buffer && buffer->buft == ggml_backend_metal_buffer_type()) {- struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) buffer->context;-- // find the view that contains the tensor fully- for (int i = 0; i < buf_ctx->n_buffers; ++i) {- const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->buffers[i].data;-- //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf_ctx->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf_ctx->buffers[i].size);- if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf_ctx->buffers[i].size) {- *offs = (size_t) ioffs;-- //GGML_METAL_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);-- return buf_ctx->buffers[i].metal;- }- }-- GGML_METAL_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);-- return nil;- }+ struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) buffer->context;// find the view that contains the tensor fully- for (int i = 0; i < ctx->n_buffers; ++i) {- const int64_t ioffs = (int64_t) t->data - (int64_t) ctx->buffers[i].data;+ for (int i = 0; i < buf_ctx->n_buffers; ++i) {+ const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->buffers[i].data;- //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name);- if (ioffs >= 0 && ioffs + tsize <= (int64_t) ctx->buffers[i].size) {+ //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf_ctx->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf_ctx->buffers[i].size);+ if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf_ctx->buffers[i].size) {*offs = (size_t) ioffs;- //GGML_METAL_LOG_INFO("%s: '%s' tensor '%16s', offs = %8ld\n", __func__, ctx->buffers[i].name, t->name, *offs);+ //GGML_METAL_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);- return ctx->buffers[i].metal;+ return buf_ctx->buffers[i].metal;}}- // TODO(jart): Why does this happen?- // GGML_METAL_LOG_ERROR("%s: error: buffer is nil\n", __func__);+ GGML_METAL_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);return nil;}-bool ggml_metal_add_buffer(- struct ggml_metal_context * ctx,- const char * name,- void * data,- size_t size,- size_t max_size) {- if (ctx->n_buffers >= GGML_METAL_MAX_BUFFERS) {- GGML_METAL_LOG_ERROR("%s: error: too many buffers\n", __func__);- return false;- }-- if (data) {- // verify that the buffer does not overlap with any of the existing buffers- for (int i = 0; i < ctx->n_buffers; ++i) {- const int64_t ioffs = (int64_t) data - (int64_t) ctx->buffers[i].data;-- if (ioffs >= 0 && ioffs < (int64_t) ctx->buffers[i].size) {- GGML_METAL_LOG_ERROR("%s: error: buffer '%s' overlaps with '%s'\n", __func__, name, ctx->buffers[i].name);- return false;- }- }-- const size_t size_page = sysconf(_SC_PAGESIZE);-- size_t size_aligned = size;- if ((size_aligned % size_page) != 0) {- size_aligned += (size_page - (size_aligned % size_page));- }-- // the buffer fits into the max buffer size allowed by the device- if (size_aligned <= ctx->device.maxBufferLength) {- ctx->buffers[ctx->n_buffers].name = name;- ctx->buffers[ctx->n_buffers].data = data;- ctx->buffers[ctx->n_buffers].size = size;-- ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];-- if (ctx->buffers[ctx->n_buffers].metal == nil) {- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_aligned / 1024.0 / 1024.0);- return false;- }-- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB", __func__, name, size_aligned / 1024.0 / 1024.0);-- ++ctx->n_buffers;- } else {- // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into- // one of the views- const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case- const size_t size_step = ctx->device.maxBufferLength - size_ovlp;- const size_t size_view = ctx->device.maxBufferLength;-- for (size_t i = 0; i < size; i += size_step) {- const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);-- ctx->buffers[ctx->n_buffers].name = name;- ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i);- ctx->buffers[ctx->n_buffers].size = size_step_aligned;-- ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];-- if (ctx->buffers[ctx->n_buffers].metal == nil) {- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0);- return false;- }-- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i);- if (i + size_step < size) {- GGML_METAL_LOG_INFO("\n");- }-- ++ctx->n_buffers;- }- }--#if TARGET_OS_OSX- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",- ctx->device.currentAllocatedSize / 1024.0 / 1024.0,- ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);-- if (ctx->device.currentAllocatedSize > ctx->device.recommendedMaxWorkingSetSize) {- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);- } else {- GGML_METAL_LOG_INFO("\n");- }-#else- GGML_METAL_LOG_INFO(", (%8.2f)\n", ctx->device.currentAllocatedSize / 1024.0 / 1024.0);-#endif- }-- return true;-}--void ggml_metal_set_tensor(- struct ggml_metal_context * ctx,- struct ggml_tensor * t) {- size_t offs;- id<MTLBuffer> id_dst = ggml_metal_get_buffer(ctx, t, &offs);-- memcpy((void *) ((uint8_t *) id_dst.contents + offs), t->data, ggml_nbytes(t));-}--void ggml_metal_get_tensor(- struct ggml_metal_context * ctx,- struct ggml_tensor * t) {- size_t offs;- id<MTLBuffer> id_src = ggml_metal_get_buffer(ctx, t, &offs);-- memcpy(t->data, (void *) ((uint8_t *) id_src.contents + offs), ggml_nbytes(t));-}--void ggml_metal_graph_find_concurrency(- struct ggml_metal_context * ctx,- struct ggml_cgraph * gf, bool check_mem) {- int search_depth = gf->n_nodes; //we only find concurrency in this range to avoid wasting too much time- int nodes_unused[GGML_MAX_CONCUR];-- for (int i = 0; i < GGML_MAX_CONCUR; i++) { ctx->concur_list[i] = 0; }- for (int i = 0; i < gf->n_nodes; i++) { nodes_unused[i] = 1; }- ctx->concur_list_len = 0;-- int n_left = gf->n_nodes;- int n_start = 0; // all nodes before n_start at nodes_unused array have been sorted and store back to ctx->concur_list- int level_pos = 0; // at ctx->concur_list, the last layer (level) ends at level_pos-- while (n_left > 0) {- // number of nodes at a layer (that can be issued concurrently)- int concurrency = 0;- for (int i = n_start; i < ((n_start + search_depth > gf->n_nodes) ? gf->n_nodes : n_start + search_depth); i++) {- if (nodes_unused[i]) {- // if the requirements for gf->nodes[i] are satisfied- int exe_flag = 1;-- // scan all srcs- for (int src_ind = 0; src_ind < GGML_MAX_SRC; src_ind++) {- struct ggml_tensor * src_cur = gf->nodes[i]->src[src_ind];- if (src_cur) {- // if is leaf nodes it's satisfied.- // TODO: ggml_is_leaf()- if (src_cur->op == GGML_OP_NONE && src_cur->grad == NULL) {- continue;- }-- // otherwise this src should be the output from previous nodes.- int is_found = 0;-- // scan 2*search_depth back because we inserted barrier.- //for (int j = ((level_pos - 2*search_depth) < 0 ? 0 : (level_pos - 2*search_depth)); j < level_pos; j++) {- for (int j = MAX(0, level_pos - 2*search_depth); j < level_pos; j++) {- if (ctx->concur_list[j] >= 0 && gf->nodes[ctx->concur_list[j]] == src_cur) {- is_found = 1;- break;- }- }- if (is_found == 0) {- exe_flag = 0;- break;- }- }- }- if (exe_flag && check_mem) {- // check if nodes[i]'s data will be overwritten by a node before nodes[i].- // if node[5] and node[3] write to the same memory region, then we can't issue node[5] before node[3]- int64_t data_start = (int64_t) gf->nodes[i]->data;- int64_t length = (int64_t) ggml_nbytes(gf->nodes[i]);- for (int j = n_start; j < i; j++) {- if (nodes_unused[j] && gf->nodes[j]->op != GGML_OP_RESHAPE \- && gf->nodes[j]->op != GGML_OP_VIEW \- && gf->nodes[j]->op != GGML_OP_TRANSPOSE \- && gf->nodes[j]->op != GGML_OP_PERMUTE) {- if (((int64_t)gf->nodes[j]->data) >= data_start + length || \- ((int64_t)gf->nodes[j]->data) + (int64_t) ggml_nbytes(gf->nodes[j]) <= data_start) {- continue;- }-- exe_flag = 0;- }- }- }- if (exe_flag) {- ctx->concur_list[level_pos + concurrency] = i;- nodes_unused[i] = 0;- concurrency++;- ctx->concur_list_len++;- }- }- }- n_left -= concurrency;- // adding a barrier different layer- ctx->concur_list[level_pos + concurrency] = -1;- ctx->concur_list_len++;- // jump all sorted nodes at nodes_bak- while (!nodes_unused[n_start]) {- n_start++;- }- level_pos += concurrency + 1;- }-- if (ctx->concur_list_len > GGML_MAX_CONCUR) {- GGML_METAL_LOG_WARN("%s: too many elements for metal ctx->concur_list!\n", __func__);- }-}--static bool ggml_metal_supports_op(const struct ggml_tensor * op) {+static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {switch (op->op) {case GGML_OP_UNARY:switch (ggml_get_unary_op(op)) {@@ -961,9 +641,11 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {case GGML_OP_SCALE:case GGML_OP_SQR:case GGML_OP_SUM_ROWS:+ return true;case GGML_OP_SOFT_MAX:case GGML_OP_RMS_NORM:case GGML_OP_GROUP_NORM:+ return ctx->support_simdgroup_reduction;case GGML_OP_NORM:case GGML_OP_ALIBI:case GGML_OP_ROPE:@@ -972,9 +654,11 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {case GGML_OP_PAD:case GGML_OP_ARGSORT:case GGML_OP_LEAKY_RELU:+ return true;case GGML_OP_MUL_MAT:case GGML_OP_MUL_MAT_ID:- return true;+ return ctx->support_simdgroup_reduction &&+ (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F32);case GGML_OP_CPY:case GGML_OP_DUP:case GGML_OP_CONT:@@ -1012,1457 +696,1545 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {return false;}}-bool ggml_metal_graph_compute(++static bool ggml_metal_graph_compute(struct ggml_metal_context * ctx,struct ggml_cgraph * gf) {- @autoreleasepool {- // if there is ctx->concur_list, dispatch concurrently- // else fallback to serial dispatchMTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor;-- const bool has_concur = ctx->concur_list_len && ctx->concur_list_len <= GGML_MAX_CONCUR;-- const int n_nodes = has_concur ? ctx->concur_list_len : gf->n_nodes;- edesc.dispatchType = has_concur ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial;+ edesc.dispatchType = MTLDispatchTypeSerial;// create multiple command buffers and enqueue them// then, we encode the graph into the command buffers in parallel+ const int n_nodes = gf->n_nodes;const int n_cb = ctx->n_cb;+ const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;- for (int i = 0; i < n_cb; ++i) {- ctx->command_buffers[i] = [ctx->queue commandBuffer];+ id<MTLCommandBuffer> command_buffer_builder[n_cb];+ for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {+ id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];+ command_buffer_builder[cb_idx] = command_buffer;// enqueue the command buffers in order to specify their execution order- [ctx->command_buffers[i] enqueue];-- ctx->command_encoders[i] = [ctx->command_buffers[i] computeCommandEncoderWithDescriptor: edesc];+ [command_buffer enqueue];}+ const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;- for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {- const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;+ dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {+ const int cb_idx = iter;- dispatch_async(ctx->d_queue, ^{- size_t offs_src0 = 0;- size_t offs_src1 = 0;- size_t offs_dst = 0;+ size_t offs_src0 = 0;+ size_t offs_src1 = 0;+ size_t offs_dst = 0;- id<MTLCommandBuffer> command_buffer = ctx->command_buffers[cb_idx];- id<MTLComputeCommandEncoder> encoder = ctx->command_encoders[cb_idx];+ id<MTLCommandBuffer> command_buffer = command_buffers[cb_idx];+ id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc];- const int node_start = (cb_idx + 0) * n_nodes_per_cb;- const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);+ const int node_start = (cb_idx + 0) * n_nodes_per_cb;+ const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);- for (int ind = node_start; ind < node_end; ++ind) {- const int i = has_concur ? ctx->concur_list[ind] : ind;+ for (int i = node_start; i < node_end; ++i) {+ if (i == -1) {+ [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];+ continue;+ }- if (i == -1) {- [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];- continue;- }+ //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));++ struct ggml_tensor * src0 = gf->nodes[i]->src[0];+ struct ggml_tensor * src1 = gf->nodes[i]->src[1];+ struct ggml_tensor * dst = gf->nodes[i];++ switch (dst->op) {+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_VIEW:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_PERMUTE:+ {+ // noop -> next node+ } continue;+ default:+ {+ } break;+ }- //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));-- struct ggml_tensor * src0 = gf->nodes[i]->src[0];- struct ggml_tensor * src1 = gf->nodes[i]->src[1];- struct ggml_tensor * dst = gf->nodes[i];-- switch (dst->op) {- case GGML_OP_NONE:- case GGML_OP_RESHAPE:- case GGML_OP_VIEW:- case GGML_OP_TRANSPOSE:- case GGML_OP_PERMUTE:- {- // noop -> next node- } continue;- default:- {- } break;- }+ if (!ggml_metal_supports_op(ctx, dst)) {+ GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));+ GGML_ASSERT(!"unsupported op");+ }- if (!ggml_metal_supports_op(dst)) {- GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));- GGML_ASSERT(!"unsupported op");- }+#ifndef GGML_METAL_NDEBUG+ [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];+#endif- const int64_t ne00 = src0 ? src0->ne[0] : 0;- const int64_t ne01 = src0 ? src0->ne[1] : 0;- const int64_t ne02 = src0 ? src0->ne[2] : 0;- const int64_t ne03 = src0 ? src0->ne[3] : 0;-- const uint64_t nb00 = src0 ? src0->nb[0] : 0;- const uint64_t nb01 = src0 ? src0->nb[1] : 0;- const uint64_t nb02 = src0 ? src0->nb[2] : 0;- const uint64_t nb03 = src0 ? src0->nb[3] : 0;-- const int64_t ne10 = src1 ? src1->ne[0] : 0;- const int64_t ne11 = src1 ? src1->ne[1] : 0;- const int64_t ne12 = src1 ? src1->ne[2] : 0;- const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);-- const uint64_t nb10 = src1 ? src1->nb[0] : 0;- const uint64_t nb11 = src1 ? src1->nb[1] : 0;- const uint64_t nb12 = src1 ? src1->nb[2] : 0;- const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13);-- const int64_t ne0 = dst ? dst->ne[0] : 0;- const int64_t ne1 = dst ? dst->ne[1] : 0;- const int64_t ne2 = dst ? dst->ne[2] : 0;- const int64_t ne3 = dst ? dst->ne[3] : 0;-- const uint64_t nb0 = dst ? dst->nb[0] : 0;- const uint64_t nb1 = dst ? dst->nb[1] : 0;- const uint64_t nb2 = dst ? dst->nb[2] : 0;- const uint64_t nb3 = dst ? dst->nb[3] : 0;-- const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;- const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;- const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;-- id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(ctx, src0, &offs_src0) : nil;- id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil;- id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil;-- //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));- //if (src0) {- // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,- // ggml_is_contiguous(src0), src0->name);- //}- //if (src1) {- // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,- // ggml_is_contiguous(src1), src1->name);- //}- //if (dst) {- // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,- // dst->name);- //}-- switch (dst->op) {- case GGML_OP_CONCAT:- {- const int64_t nb = ne00;-- [encoder setComputePipelineState:ctx->pipeline_concat];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];- [encoder setBytes:&nb length:sizeof(nb) atIndex:27];-- const int nth = MIN(1024, ne0);-- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_ADD:- case GGML_OP_MUL:- case GGML_OP_DIV:- {- const size_t offs = 0;-- bool bcast_row = false;-- int64_t nb = ne00;+ const int64_t ne00 = src0 ? src0->ne[0] : 0;+ const int64_t ne01 = src0 ? src0->ne[1] : 0;+ const int64_t ne02 = src0 ? src0->ne[2] : 0;+ const int64_t ne03 = src0 ? src0->ne[3] : 0;++ const uint64_t nb00 = src0 ? src0->nb[0] : 0;+ const uint64_t nb01 = src0 ? src0->nb[1] : 0;+ const uint64_t nb02 = src0 ? src0->nb[2] : 0;+ const uint64_t nb03 = src0 ? src0->nb[3] : 0;++ const int64_t ne10 = src1 ? src1->ne[0] : 0;+ const int64_t ne11 = src1 ? src1->ne[1] : 0;+ const int64_t ne12 = src1 ? src1->ne[2] : 0;+ const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);++ const uint64_t nb10 = src1 ? src1->nb[0] : 0;+ const uint64_t nb11 = src1 ? src1->nb[1] : 0;+ const uint64_t nb12 = src1 ? src1->nb[2] : 0;+ const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13);++ const int64_t ne0 = dst ? dst->ne[0] : 0;+ const int64_t ne1 = dst ? dst->ne[1] : 0;+ const int64_t ne2 = dst ? dst->ne[2] : 0;+ const int64_t ne3 = dst ? dst->ne[3] : 0;++ const uint64_t nb0 = dst ? dst->nb[0] : 0;+ const uint64_t nb1 = dst ? dst->nb[1] : 0;+ const uint64_t nb2 = dst ? dst->nb[2] : 0;+ const uint64_t nb3 = dst ? dst->nb[3] : 0;++ const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;+ const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;+ const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;++ id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(src0, &offs_src0) : nil;+ id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(src1, &offs_src1) : nil;+ id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(dst, &offs_dst) : nil;++ //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));+ //if (src0) {+ // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,+ // ggml_is_contiguous(src0), src0->name);+ //}+ //if (src1) {+ // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,+ // ggml_is_contiguous(src1), src1->name);+ //}+ //if (dst) {+ // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,+ // dst->name);+ //}++ switch (dst->op) {+ case GGML_OP_CONCAT:+ {+ const int64_t nb = ne00;++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];+ [encoder setBytes:&nb length:sizeof(nb) atIndex:27];++ const int nth = MIN(1024, ne0);++ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_ADD:+ case GGML_OP_MUL:+ case GGML_OP_DIV:+ {+ const size_t offs = 0;++ bool bcast_row = false;++ int64_t nb = ne00;++ id<MTLComputePipelineState> pipeline = nil;++ if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {+ GGML_ASSERT(ggml_is_contiguous(src0));- id<MTLComputePipelineState> pipeline = nil;+ // src1 is a row+ GGML_ASSERT(ne11 == 1);- if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {- GGML_ASSERT(ggml_is_contiguous(src0));+ nb = ne00 / 4;+ switch (dst->op) {+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break;+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break;+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break;+ default: GGML_ASSERT(false);+ }- // src1 is a row- GGML_ASSERT(ne11 == 1);+ bcast_row = true;+ } else {+ switch (dst->op) {+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break;+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break;+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break;+ default: GGML_ASSERT(false);+ }+ }- nb = ne00 / 4;- switch (dst->op) {- case GGML_OP_ADD: pipeline = ctx->pipeline_add_row; break;- case GGML_OP_MUL: pipeline = ctx->pipeline_mul_row; break;- case GGML_OP_DIV: pipeline = ctx->pipeline_div_row; break;- default: GGML_ASSERT(false);- }+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];+ [encoder setBytes:&nb length:sizeof(nb) atIndex:28];++ if (bcast_row) {+ const int64_t n = ggml_nelements(dst)/4;- bcast_row = true;- } else {- switch (dst->op) {- case GGML_OP_ADD: pipeline = ctx->pipeline_add; break;- case GGML_OP_MUL: pipeline = ctx->pipeline_mul; break;- case GGML_OP_DIV: pipeline = ctx->pipeline_div; break;- default: GGML_ASSERT(false);- }- }+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } else {+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);- [encoder setComputePipelineState:pipeline];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];- [encoder setBytes:&nb length:sizeof(nb) atIndex:28];-- if (bcast_row) {- const int64_t n = ggml_nelements(dst)/4;+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ }+ } break;+ case GGML_OP_ACC:+ {+ GGML_ASSERT(src0t == GGML_TYPE_F32);+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ GGML_ASSERT(dstt == GGML_TYPE_F32);- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } else {- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);+ GGML_ASSERT(ggml_is_contiguous(src0));+ GGML_ASSERT(ggml_is_contiguous(src1));- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- }- } break;- case GGML_OP_ACC:- {- GGML_ASSERT(src0t == GGML_TYPE_F32);- GGML_ASSERT(src1t == GGML_TYPE_F32);- GGML_ASSERT(dstt == GGML_TYPE_F32);+ const size_t pnb1 = ((int32_t *) dst->op_params)[0];+ const size_t pnb2 = ((int32_t *) dst->op_params)[1];+ const size_t pnb3 = ((int32_t *) dst->op_params)[2];+ const size_t offs = ((int32_t *) dst->op_params)[3];- GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(ggml_is_contiguous(src1));-- const size_t pnb1 = ((int32_t *) dst->op_params)[0];- const size_t pnb2 = ((int32_t *) dst->op_params)[1];- const size_t pnb3 = ((int32_t *) dst->op_params)[2];- const size_t offs = ((int32_t *) dst->op_params)[3];-- const bool inplace = (bool) ((int32_t *) dst->op_params)[4];-- if (!inplace) {- // run a separete kernel to cpy src->dst- // not sure how to avoid this- // TODO: make a simpler cpy_bytes kernel-- const int nth = MIN((int) ctx->pipeline_cpy_f32_f32.maxTotalThreadsPerThreadgroup, ne00);-- [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];-- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- }+ const bool inplace = (bool) ((int32_t *) dst->op_params)[4];- [encoder setComputePipelineState:ctx->pipeline_add];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];-- const int nth = MIN((int) ctx->pipeline_add.maxTotalThreadsPerThreadgroup, ne00);-- [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_SCALE:- {- GGML_ASSERT(ggml_is_contiguous(src0));+ if (!inplace) {+ // run a separete kernel to cpy src->dst+ // not sure how to avoid this+ // TODO: make a simpler cpy_bytes kernel- const float scale = *(const float *) dst->op_params;+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline;- int64_t n = ggml_nelements(dst);+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];- if (n % 4 == 0) {- n /= 4;- [encoder setComputePipelineState:ctx->pipeline_scale_4];- } else {- [encoder setComputePipelineState:ctx->pipeline_scale];- }+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&scale length:sizeof(scale) atIndex:2];+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ }- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_OP_UNARY:- switch (ggml_get_unary_op(gf->nodes[i])) {- case GGML_UNARY_OP_TANH:- {- [encoder setComputePipelineState:ctx->pipeline_tanh];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];++ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);++ [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_SCALE:+ {+ GGML_ASSERT(ggml_is_contiguous(src0));++ const float scale = *(const float *) dst->op_params;++ int64_t n = ggml_nelements(dst);++ id<MTLComputePipelineState> pipeline = nil;++ if (n % 4 == 0) {+ n /= 4;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline;+ } else {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline;+ }- const int64_t n = ggml_nelements(dst);+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&scale length:sizeof(scale) atIndex:2];- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_UNARY_OP_RELU:- {- [encoder setComputePipelineState:ctx->pipeline_relu];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_OP_UNARY:+ switch (ggml_get_unary_op(gf->nodes[i])) {+ case GGML_UNARY_OP_TANH:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline;- const int64_t n = ggml_nelements(dst);+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_UNARY_OP_GELU:- {- [encoder setComputePipelineState:ctx->pipeline_gelu];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ const int64_t n = ggml_nelements(dst);- const int64_t n = ggml_nelements(dst);- GGML_ASSERT(n % 4 == 0);+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_UNARY_OP_RELU:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline;- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_UNARY_OP_GELU_QUICK:- {- [encoder setComputePipelineState:ctx->pipeline_gelu_quick];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- const int64_t n = ggml_nelements(dst);- GGML_ASSERT(n % 4 == 0);+ const int64_t n = ggml_nelements(dst);- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_UNARY_OP_SILU:- {- [encoder setComputePipelineState:ctx->pipeline_silu];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_UNARY_OP_GELU:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline;- const int64_t n = ggml_nelements(dst);- GGML_ASSERT(n % 4 == 0);+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- default:- {- GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));- GGML_ASSERT(false);- }- } break;- case GGML_OP_SQR:- {- GGML_ASSERT(ggml_is_contiguous(src0));+ const int64_t n = ggml_nelements(dst);+ GGML_ASSERT(n % 4 == 0);- [encoder setComputePipelineState:ctx->pipeline_sqr];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_UNARY_OP_GELU_QUICK:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline;- const int64_t n = ggml_nelements(dst);- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_OP_SUM_ROWS:- {- GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setComputePipelineState:ctx->pipeline_sum_rows];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];-- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_OP_SOFT_MAX:- {- int nth = 32; // SIMD width-- if (ne00%4 == 0) {- while (nth < ne00/4 && nth < 256) {- nth *= 2;- }- [encoder setComputePipelineState:ctx->pipeline_soft_max_4];- } else {- while (nth < ne00 && nth < 1024) {- nth *= 2;- }- [encoder setComputePipelineState:ctx->pipeline_soft_max];- }+ const int64_t n = ggml_nelements(dst);+ GGML_ASSERT(n % 4 == 0);- const float scale = ((float *) dst->op_params)[0];+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_UNARY_OP_SILU:+ {+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline;- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- if (id_src1) {- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- } else {- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];++ const int64_t n = ggml_nelements(dst);+ GGML_ASSERT(n % 4 == 0);++ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ default:+ {+ GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));+ GGML_ASSERT(false);}- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&scale length:sizeof(scale) atIndex:6];- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];-- [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_DIAG_MASK_INF:- {- const int n_past = ((int32_t *)(dst->op_params))[0];-- if (ne00%8 == 0) {- [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf_8];- } else {- [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf];+ } break;+ case GGML_OP_SQR:+ {+ GGML_ASSERT(ggml_is_contiguous(src0));++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];++ const int64_t n = ggml_nelements(dst);++ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_OP_SUM_ROWS:+ {+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_OP_SOFT_MAX:+ {+ int nth = 32; // SIMD width++ id<MTLComputePipelineState> pipeline = nil;++ if (ne00%4 == 0) {+ while (nth < ne00/4 && nth < 256) {+ nth *= 2;}- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];- [encoder setBytes:&n_past length:sizeof(int) atIndex:4];-- if (ne00%8 == 0) {- [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline;+ } else {+ while (nth < ne00 && nth < 1024) {+ nth *= 2;}- else {- [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;+ }++ const float scale = ((float *) dst->op_params)[0];++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ if (id_src1) {+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ } else {+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];+ }+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&scale length:sizeof(scale) atIndex:6];+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_DIAG_MASK_INF:+ {+ const int n_past = ((int32_t *)(dst->op_params))[0];++ id<MTLComputePipelineState> pipeline = nil;++ if (ne00%8 == 0) {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline;+ } else {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline;+ }++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];+ [encoder setBytes:&n_past length:sizeof(int) atIndex:4];++ if (ne00%8 == 0) {+ [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ }+ else {+ [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ }+ } break;+ case GGML_OP_MUL_MAT:+ {+ GGML_ASSERT(ne00 == ne10);++ // TODO: assert that dim2 and dim3 are contiguous+ GGML_ASSERT(ne12 % ne02 == 0);+ GGML_ASSERT(ne13 % ne03 == 0);++ const uint r2 = ne12/ne02;+ const uint r3 = ne13/ne03;++ // find the break-even point where the matrix-matrix kernel becomes more efficient compared+ // to the matrix-vector kernel+ int ne11_mm_min = 1;++#if 0+ // the numbers below are measured on M2 Ultra for 7B and 13B models+ // these numbers do not translate to other devices or model sizes+ // TODO: need to find a better approach+ if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {+ switch (src0t) {+ case GGML_TYPE_F16: ne11_mm_min = 2; break;+ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;+ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;+ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;+ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;+ case GGML_TYPE_Q5_0: // not tested yet+ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet+ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;+ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;+ default: ne11_mm_min = 1; break;}- } break;- case GGML_OP_MUL_MAT:- {- GGML_ASSERT(ne00 == ne10);+ }+#endif- // TODO: assert that dim2 and dim3 are contiguous- GGML_ASSERT(ne12 % ne02 == 0);- GGML_ASSERT(ne13 % ne03 == 0);+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&+ !ggml_is_transposed(src0) &&+ !ggml_is_transposed(src1) &&+ src1t == GGML_TYPE_F32 &&+ ne00 % 32 == 0 && ne00 >= 64 &&+ (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {+ //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);- const uint r2 = ne12/ne02;- const uint r3 = ne13/ne03;+ id<MTLComputePipelineState> pipeline = nil;++ switch (src0->type) {+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break;+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break;+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break;+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break;+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break;+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break;+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break;+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break;+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break;+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break;+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break;+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break;+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break;+ default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");+ }- // find the break-even point where the matrix-matrix kernel becomes more efficient compared- // to the matrix-vector kernel- int ne11_mm_min = 1;+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];+ [encoder setThreadgroupMemoryLength:8192 atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];+ } else {+ int nth0 = 32;+ int nth1 = 1;+ int nrows = 1;+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);-#if 0- // the numbers below are measured on M2 Ultra for 7B and 13B models- // these numbers do not translate to other devices or model sizes- // TODO: need to find a better approach- if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {- switch (src0t) {- case GGML_TYPE_F16: ne11_mm_min = 2; break;- case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;- case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;- case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;- case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;- case GGML_TYPE_Q5_0: // not tested yet- case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet- case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;- case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;- default: ne11_mm_min = 1; break;- }- }-#endif+ id<MTLComputePipelineState> pipeline = nil;- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&- !ggml_is_transposed(src0) &&- !ggml_is_transposed(src1) &&- src1t == GGML_TYPE_F32 &&- ne00 % 32 == 0 && ne00 >= 64 &&- (ne11 > ne11_mm_min || (g_backend->ggml_is_quantized(src0t) && ne12 > 1))) {- //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);- switch (src0->type) {- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break;- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f16_f32]; break;- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_0_f32]; break;- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_1_f32]; break;- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_0_f32]; break;- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_1_f32]; break;- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q8_0_f32]; break;- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q2_K_f32]; break;- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q3_K_f32]; break;- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_K_f32]; break;- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_K_f32]; break;- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q6_K_f32]; break;- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xxs_f32]; break;- default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");- }- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];- [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];- [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];- [encoder setThreadgroupMemoryLength:8192 atIndex:0];- [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];- } else {- int nth0 = 32;- int nth1 = 1;- int nrows = 1;- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);-- // use custom matrix x vector kernel- switch (src0t) {- case GGML_TYPE_F32:- {- GGML_ASSERT(src1t == GGML_TYPE_F32);- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f32_f32];- nrows = 4;- } break;- case GGML_TYPE_F16:- {- nth0 = 32;- nth1 = 1;- if (src1t == GGML_TYPE_F32) {- if (ne11 * ne12 < 4) {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row];- } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4];- nrows = ne11;- } else {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32];- nrows = 4;- }+ // use custom matrix x vector kernel+ switch (src0t) {+ case GGML_TYPE_F32:+ {+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline;+ nrows = 4;+ } break;+ case GGML_TYPE_F16:+ {+ nth0 = 32;+ nth1 = 1;+ if (src1t == GGML_TYPE_F32) {+ if (ne11 * ne12 < 4) {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline;+ } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline;+ nrows = ne11;} else {- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f16];+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline;nrows = 4;}- } break;- case GGML_TYPE_Q4_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_0_f32];- } break;- case GGML_TYPE_Q4_1:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_1_f32];- } break;- case GGML_TYPE_Q5_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_0_f32];- } break;- case GGML_TYPE_Q5_1:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_1_f32];- } break;- case GGML_TYPE_Q8_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q8_0_f32];- } break;- case GGML_TYPE_Q2_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q2_K_f32];- } break;- case GGML_TYPE_Q3_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q3_K_f32];- } break;- case GGML_TYPE_Q4_K:- {- nth0 = 4; //1;- nth1 = 8; //32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_K_f32];- } break;- case GGML_TYPE_Q5_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_K_f32];- } break;- case GGML_TYPE_Q6_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q6_K_f32];- } break;- case GGML_TYPE_IQ2_XXS:- {- nth0 = 4;- nth1 = 16;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xxs_f32];- } break;- default:- {- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);- GGML_ASSERT(false && "not implemented");+ } else {+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline;+ nrows = 4;}- };-- if (g_backend->ggml_is_quantized(src0t)) {- GGML_ASSERT(ne00 >= nth0*nth1);- }+ } break;+ case GGML_TYPE_Q4_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q4_1:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_1:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline;+ } break;+ case GGML_TYPE_Q8_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q2_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q3_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q4_K:+ {+ nth0 = 4; //1;+ nth1 = 8; //32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q6_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline;+ } break;+ case GGML_TYPE_IQ2_XXS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline;+ } break;+ case GGML_TYPE_IQ2_XS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline;+ } break;+ default:+ {+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);+ GGML_ASSERT(false && "not implemented");+ }+ };- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];- [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];- [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];-- if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||- src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||- //src0t == GGML_TYPE_IQ2_XXS ||- src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src0t == GGML_TYPE_IQ2_XXS) {- [encoder setThreadgroupMemoryLength:(256*8+128) atIndex:0];- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src0t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src0t == GGML_TYPE_Q3_K) {-#ifdef GGML_QKK_64- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];-#else- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];-#endif- }- else if (src0t == GGML_TYPE_Q5_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src0t == GGML_TYPE_Q6_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- } else {- const int64_t ny = (ne11 + nrows - 1)/nrows;- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }+ if (ggml_is_quantized(src0t)) {+ GGML_ASSERT(ne00 >= nth0*nth1);}- } break;- case GGML_OP_MUL_MAT_ID:- {- //GGML_ASSERT(ne00 == ne10);- //GGML_ASSERT(ne03 == ne13);-- GGML_ASSERT(src0t == GGML_TYPE_I32);-- const int n_as = ((int32_t *) dst->op_params)[1];-- // TODO: make this more general- GGML_ASSERT(n_as <= 8);-- // max size of the src1ids array in the kernel stack- GGML_ASSERT(ne11 <= 512);-- struct ggml_tensor * src2 = gf->nodes[i]->src[2];-- const int64_t ne20 = src2 ? src2->ne[0] : 0;- const int64_t ne21 = src2 ? src2->ne[1] : 0;- const int64_t ne22 = src2 ? src2->ne[2] : 0;- const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);-- const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);- const uint64_t nb21 = src2 ? src2->nb[1] : 0;- const uint64_t nb22 = src2 ? src2->nb[2] : 0;- const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23);-- const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);-- GGML_ASSERT(!ggml_is_transposed(src2));- GGML_ASSERT(!ggml_is_transposed(src1));-- GGML_ASSERT(src1t == GGML_TYPE_F32);-- const uint r2 = ne12/ne22;- const uint r3 = ne13/ne23;-- // find the break-even point where the matrix-matrix kernel becomes more efficient compared- // to the matrix-vector kernel- int ne11_mm_min = n_as;-- const int idx = ((int32_t *) dst->op_params)[0];-- // batch size- GGML_ASSERT(ne01 == ne11);-- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel- // !!!- // TODO: for now, always use mat-vec kernels until we figure out how to improve the- // indirect matrix multiplication- // !!!- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&- ne20 % 32 == 0 && ne20 >= 64 &&- ne11 > ne11_mm_min) {- switch (src2->type) {- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f32_f32]; break;- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f16_f32]; break;- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_0_f32]; break;- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_1_f32]; break;- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_0_f32]; break;- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_1_f32]; break;- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q8_0_f32]; break;- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q2_K_f32]; break;- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q3_K_f32]; break;- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_K_f32]; break;- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_K_f32]; break;- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q6_K_f32]; break;- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xxs_f32]; break;- default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");- }- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];- [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5];- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];- [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];- [encoder setBytes:&r2 length:sizeof(r2) atIndex:16];- [encoder setBytes:&r3 length:sizeof(r3) atIndex:17];- [encoder setBytes:&idx length:sizeof(idx) atIndex:18];- // TODO: how to make this an array? read Metal docs- for (int j = 0; j < 8; ++j) {- // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8- struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];-- size_t offs_src_cur = 0;- id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);-- [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];- }-- [encoder setThreadgroupMemoryLength:8192 atIndex:0];-- [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];- } else {- int nth0 = 32;- int nth1 = 1;- int nrows = 1;- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);-- // use custom matrix x vector kernel- switch (src2t) {- case GGML_TYPE_F32:- {- GGML_ASSERT(src1t == GGML_TYPE_F32);- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f32_f32];- } break;- case GGML_TYPE_F16:- {- GGML_ASSERT(src1t == GGML_TYPE_F32);- nth0 = 32;- nth1 = 1;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f16_f32];- } break;- case GGML_TYPE_Q4_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_0_f32];- } break;- case GGML_TYPE_Q4_1:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_1_f32];- } break;- case GGML_TYPE_Q5_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_0_f32];- } break;- case GGML_TYPE_Q5_1:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_1_f32];- } break;- case GGML_TYPE_Q8_0:- {- nth0 = 8;- nth1 = 8;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q8_0_f32];- } break;- case GGML_TYPE_Q2_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q2_K_f32];- } break;- case GGML_TYPE_Q3_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q3_K_f32];- } break;- case GGML_TYPE_Q4_K:- {- nth0 = 4; //1;- nth1 = 8; //32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_K_f32];- } break;- case GGML_TYPE_Q5_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_K_f32];- } break;- case GGML_TYPE_Q6_K:- {- nth0 = 2;- nth1 = 32;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q6_K_f32];- } break;- case GGML_TYPE_IQ2_XXS:- {- nth0 = 4;- nth1 = 16;- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xxs_f32];- } break;- default:- {- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);- GGML_ASSERT(false && "not implemented");- }- };-- if (g_backend->ggml_is_quantized(src2t)) {- GGML_ASSERT(ne20 >= nth0*nth1);- }- const int64_t _ne1 = 1; // kernels needs a reference in constant memory-- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];- [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];- [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6];- [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7];- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8];- [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9];- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];- [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11];- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];- [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];- [encoder setBytes:&r2 length:sizeof(r2) atIndex:20];- [encoder setBytes:&r3 length:sizeof(r3) atIndex:21];- [encoder setBytes:&idx length:sizeof(idx) atIndex:22];- // TODO: how to make this an array? read Metal docs- for (int j = 0; j < 8; ++j) {- // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8- struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];-- size_t offs_src_cur = 0;- id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);-- [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j];- }-- if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||- src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||- //src2t == GGML_TYPE_IQ2_XXS ||- src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src2t == GGML_TYPE_IQ2_XXS) {- [encoder setThreadgroupMemoryLength:(256*8+128) atIndex:0];- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src2t == GGML_TYPE_Q4_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src2t == GGML_TYPE_Q3_K) {+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];++ if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||+ src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||+ src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {+ const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src0t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src0t == GGML_TYPE_Q3_K) {#ifdef GGML_QKK_64- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];#else- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];#endif- }- else if (src2t == GGML_TYPE_Q5_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }- else if (src2t == GGML_TYPE_Q6_K) {- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- } else {- const int64_t ny = (_ne1 + nrows - 1)/nrows;- [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];- }}- } break;- case GGML_OP_GET_ROWS:- {- switch (src0->type) {- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_get_rows_f32]; break;- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_get_rows_f16]; break;- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_0]; break;- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_1]; break;- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_0]; break;- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_1]; break;- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q8_0]; break;- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q2_K]; break;- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q3_K]; break;- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_K]; break;- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_K]; break;- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q6_K]; break;- case GGML_TYPE_I32: [encoder setComputePipelineState:ctx->pipeline_get_rows_i32]; break;- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xxs]; break;- default: GGML_ASSERT(false && "not implemented");+ else if (src0t == GGML_TYPE_Q5_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}-- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];- [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];- [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];- [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];-- [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];- } break;- case GGML_OP_RMS_NORM:- {- GGML_ASSERT(ne00 % 4 == 0);-- float eps;- memcpy(&eps, dst->op_params, sizeof(float));-- int nth = 32; // SIMD width-- while (nth < ne00/4 && nth < 1024) {- nth *= 2;+ else if (src0t == GGML_TYPE_Q6_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ } else {+ const int64_t ny = (ne11 + nrows - 1)/nrows;+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];}+ }+ } break;+ case GGML_OP_MUL_MAT_ID:+ {+ //GGML_ASSERT(ne00 == ne10);+ //GGML_ASSERT(ne03 == ne13);- [encoder setComputePipelineState:ctx->pipeline_rms_norm];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];- [encoder setBytes:&eps length:sizeof( float) atIndex:4];- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];-- const int64_t nrows = ggml_nrows(src0);-- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_GROUP_NORM:- {- GGML_ASSERT(ne00 % 4 == 0);-- //float eps;- //memcpy(&eps, dst->op_params, sizeof(float));-- const float eps = 1e-6f; // TODO: temporarily hardcoded-- const int32_t n_groups = ((int32_t *) dst->op_params)[0];-- int nth = 32; // SIMD width-- //while (nth < ne00/4 && nth < 1024) {- // nth *= 2;- //}-- [encoder setComputePipelineState:ctx->pipeline_group_norm];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];- [encoder setBytes:&eps length:sizeof( float) atIndex:9];- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];-- [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_NORM:- {- float eps;- memcpy(&eps, dst->op_params, sizeof(float));-- const int nth = MIN(256, ne00);-- [encoder setComputePipelineState:ctx->pipeline_norm];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];- [encoder setBytes:&eps length:sizeof( float) atIndex:4];- [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];-- const int64_t nrows = ggml_nrows(src0);-- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_ALIBI:- {- GGML_ASSERT((src0t == GGML_TYPE_F32));-- const int nth = MIN(1024, ne00);-- //const int n_past = ((int32_t *) dst->op_params)[0];- const int n_head = ((int32_t *) dst->op_params)[1];- float max_bias;- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));-- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);+ GGML_ASSERT(src0t == GGML_TYPE_I32);- [encoder setComputePipelineState:ctx->pipeline_alibi_f32];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];- [encoder setBytes:&m0 length:sizeof( float) atIndex:18];- [encoder setBytes:&m1 length:sizeof( float) atIndex:19];- [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20];+ const int n_as = ((int32_t *) dst->op_params)[1];- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_ROPE:- {- GGML_ASSERT(ne10 == ne02);-- const int nth = MIN(1024, ne00);-- const int n_past = ((int32_t *) dst->op_params)[0];- const int n_dims = ((int32_t *) dst->op_params)[1];- const int mode = ((int32_t *) dst->op_params)[2];- // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal- const int n_orig_ctx = ((int32_t *) dst->op_params)[4];-- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;- memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));- memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));- memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));- memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));- memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));- memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));+ // TODO: make this more general+ GGML_ASSERT(n_as <= 8);- switch (src0->type) {- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_rope_f32]; break;- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_rope_f16]; break;- default: GGML_ASSERT(false);- };+ // max size of the src1ids array in the kernel stack+ GGML_ASSERT(ne11 <= 512);- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];- [encoder setBytes:&n_past length:sizeof( int) atIndex:19];- [encoder setBytes:&n_dims length:sizeof( int) atIndex:20];- [encoder setBytes:&mode length:sizeof( int) atIndex:21];- [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22];- [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];- [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];- [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];- [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];- [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];- [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];+ struct ggml_tensor * src2 = gf->nodes[i]->src[2];- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_IM2COL:- {- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16);+ const int64_t ne20 = src2 ? src2->ne[0] : 0;+ const int64_t ne21 = src2 ? src2->ne[1] : 0;+ const int64_t ne22 = src2 ? src2->ne[2] : 0;+ const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);- const int32_t s0 = ((const int32_t *)(dst->op_params))[0];- const int32_t s1 = ((const int32_t *)(dst->op_params))[1];- const int32_t p0 = ((const int32_t *)(dst->op_params))[2];- const int32_t p1 = ((const int32_t *)(dst->op_params))[3];- const int32_t d0 = ((const int32_t *)(dst->op_params))[4];- const int32_t d1 = ((const int32_t *)(dst->op_params))[5];- const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;+ const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);+ const uint64_t nb21 = src2 ? src2->nb[1] : 0;+ const uint64_t nb22 = src2 ? src2->nb[2] : 0;+ const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23);- const int32_t N = src1->ne[is_2D ? 3 : 2];- const int32_t IC = src1->ne[is_2D ? 2 : 1];- const int32_t IH = is_2D ? src1->ne[1] : 1;- const int32_t IW = src1->ne[0];+ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);- const int32_t KH = is_2D ? src0->ne[1] : 1;- const int32_t KW = src0->ne[0];+ GGML_ASSERT(!ggml_is_transposed(src2));+ GGML_ASSERT(!ggml_is_transposed(src1));- const int32_t OH = is_2D ? dst->ne[2] : 1;- const int32_t OW = dst->ne[1];+ GGML_ASSERT(src1t == GGML_TYPE_F32);- const int32_t CHW = IC * KH * KW;+ const uint r2 = ne12/ne22;+ const uint r3 = ne13/ne23;- const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;- const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared+ // to the matrix-vector kernel+ int ne11_mm_min = n_as;- switch (src0->type) {- case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_im2col_f16]; break;- default: GGML_ASSERT(false);- };+ const int idx = ((int32_t *) dst->op_params)[0];- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];- [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];- [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];- [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];- [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];- [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];- [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];- [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];- [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];- [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];- [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];-- [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];- } break;- case GGML_OP_UPSCALE:- {- GGML_ASSERT(src0->type == GGML_TYPE_F32);-- const int sf = dst->op_params[0];-- [encoder setComputePipelineState:ctx->pipeline_upscale_f32];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];- [encoder setBytes:&sf length:sizeof(sf) atIndex:18];-- const int nth = MIN((int) ctx->pipeline_upscale_f32.maxTotalThreadsPerThreadgroup, ne0);-- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_PAD:- {- GGML_ASSERT(src0->type == GGML_TYPE_F32);-- [encoder setComputePipelineState:ctx->pipeline_pad_f32];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];-- const int nth = MIN(1024, ne0);-- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- case GGML_OP_ARGSORT:- {- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_I32);-- const int nrows = ggml_nrows(src0);-- enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];-- switch (order) {- case GGML_SORT_ASC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_asc]; break;- case GGML_SORT_DESC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_desc]; break;- default: GGML_ASSERT(false);- };+ // batch size+ GGML_ASSERT(ne01 == ne11);- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel+ // !!!+ // TODO: for now, always use mat-vec kernels until we figure out how to improve the+ // indirect matrix multiplication+ // !!!+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&+ ne20 % 32 == 0 && ne20 >= 64 &&+ ne11 > ne11_mm_min) {- [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)];- } break;- case GGML_OP_LEAKY_RELU:- {- GGML_ASSERT(src0->type == GGML_TYPE_F32);+ id<MTLComputePipelineState> pipeline = nil;- float slope;- memcpy(&slope, dst->op_params, sizeof(float));+ switch (src2->type) {+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break;+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break;+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break;+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break;+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break;+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break;+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break;+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break;+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break;+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break;+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break;+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break;+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break;+ default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");+ }- [encoder setComputePipelineState:ctx->pipeline_leaky_relu_f32];- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&slope length:sizeof(slope) atIndex:2];+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5];+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:16];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:17];+ [encoder setBytes:&idx length:sizeof(idx) atIndex:18];+ // TODO: how to make this an array? read Metal docs+ for (int j = 0; j < 8; ++j) {+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];++ size_t offs_src_cur = 0;+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(src_cur, &offs_src_cur);++ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];+ }- const int64_t n = ggml_nelements(dst);+ [encoder setThreadgroupMemoryLength:8192 atIndex:0];- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];- } break;- case GGML_OP_DUP:- case GGML_OP_CPY:- case GGML_OP_CONT:- {- GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);+ [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];+ } else {+ int nth0 = 32;+ int nth1 = 1;+ int nrows = 1;+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);- int nth = MIN(1024, ne00/ggml_blck_size(src0->type));+ id<MTLComputePipelineState> pipeline = nil;- switch (src0t) {+ // use custom matrix x vector kernel+ switch (src2t) {case GGML_TYPE_F32:{- GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);-- switch (dstt) {- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f16]; break;- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; break;- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q8_0]; break;- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_0]; break;- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_1]; break;- //case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_0]; break;- //case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_1]; break;- default: GGML_ASSERT(false && "not implemented");- };+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline;} break;case GGML_TYPE_F16:{- switch (dstt) {- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f16]; break;- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f32]; break;- default: GGML_ASSERT(false && "not implemented");- };+ GGML_ASSERT(src1t == GGML_TYPE_F32);+ nth0 = 32;+ nth1 = 1;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline;+ } break;+ case GGML_TYPE_Q4_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q4_1:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_1:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline;+ } break;+ case GGML_TYPE_Q8_0:+ {+ nth0 = 8;+ nth1 = 8;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline;+ } break;+ case GGML_TYPE_Q2_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q3_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q4_K:+ {+ nth0 = 4; //1;+ nth1 = 8; //32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q5_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline;+ } break;+ case GGML_TYPE_Q6_K:+ {+ nth0 = 2;+ nth1 = 32;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline;+ } break;+ case GGML_TYPE_IQ2_XXS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline;+ } break;+ case GGML_TYPE_IQ2_XS:+ {+ nth0 = 4;+ nth1 = 16;+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline;} break;- default: GGML_ASSERT(false && "not implemented");+ default:+ {+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);+ GGML_ASSERT(false && "not implemented");+ }+ };++ if (ggml_is_quantized(src2t)) {+ GGML_ASSERT(ne20 >= nth0*nth1);}- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];+ const int64_t _ne1 = 1; // kernels needs a reference in constant memory- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];- } break;- default:- {- GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));- GGML_ASSERT(false);+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];+ [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6];+ [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7];+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8];+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9];+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11];+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:20];+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:21];+ [encoder setBytes:&idx length:sizeof(idx) atIndex:22];+ // TODO: how to make this an array? read Metal docs+ for (int j = 0; j < 8; ++j) {+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];++ size_t offs_src_cur = 0;+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(src_cur, &offs_src_cur);++ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j];+ }++ if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||+ src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||+ src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) {+ const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q4_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q3_K) {+#ifdef GGML_QKK_64+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+#else+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+#endif+ }+ else if (src2t == GGML_TYPE_Q5_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ else if (src2t == GGML_TYPE_Q6_K) {+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ } else {+ const int64_t ny = (_ne1 + nrows - 1)/nrows;+ [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];+ }+ }+ } break;+ case GGML_OP_GET_ROWS:+ {+ id<MTLComputePipelineState> pipeline = nil;++ switch (src0->type) {+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break;+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break;+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break;+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break;+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break;+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break;+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break;+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break;+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break;+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break;+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break;+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break;+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break;+ case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break;+ default: GGML_ASSERT(false && "not implemented");+ }++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];+ [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];++ [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];+ } break;+ case GGML_OP_RMS_NORM:+ {+ GGML_ASSERT(ne00 % 4 == 0);++ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));++ int nth = 32; // SIMD width++ while (nth < ne00/4 && nth < 1024) {+ nth *= 2;+ }++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];++ const int64_t nrows = ggml_nrows(src0);++ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_GROUP_NORM:+ {+ GGML_ASSERT(ne00 % 4 == 0);++ //float eps;+ //memcpy(&eps, dst->op_params, sizeof(float));++ const float eps = 1e-6f; // TODO: temporarily hardcoded++ const int32_t n_groups = ((int32_t *) dst->op_params)[0];++ int nth = 32; // SIMD width++ //while (nth < ne00/4 && nth < 1024) {+ // nth *= 2;+ //}++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];+ [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];+ [encoder setBytes:&eps length:sizeof( float) atIndex:9];+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];++ [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_NORM:+ {+ float eps;+ memcpy(&eps, dst->op_params, sizeof(float));++ const int nth = MIN(256, ne00);++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];+ [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];++ const int64_t nrows = ggml_nrows(src0);++ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_ALIBI:+ {+ GGML_ASSERT((src0t == GGML_TYPE_F32));++ const int nth = MIN(1024, ne00);++ //const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_head = ((int32_t *) dst->op_params)[1];+ float max_bias;+ memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));++ const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));+ const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];+ [encoder setBytes:&m0 length:sizeof( float) atIndex:18];+ [encoder setBytes:&m1 length:sizeof( float) atIndex:19];+ [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_ROPE:+ {+ GGML_ASSERT(ne10 == ne02);++ const int nth = MIN(1024, ne00);++ const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_dims = ((int32_t *) dst->op_params)[1];+ const int mode = ((int32_t *) dst->op_params)[2];+ // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal+ const int n_orig_ctx = ((int32_t *) dst->op_params)[4];++ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));++ id<MTLComputePipelineState> pipeline = nil;++ switch (src0->type) {+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F32].pipeline; break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F16].pipeline; break;+ default: GGML_ASSERT(false);+ };++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];+ [encoder setBytes:&n_past length:sizeof( int) atIndex:19];+ [encoder setBytes:&n_dims length:sizeof( int) atIndex:20];+ [encoder setBytes:&mode length:sizeof( int) atIndex:21];+ [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22];+ [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];+ [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];+ [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];+ [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];+ [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];+ [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_IM2COL:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16);++ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;++ const int32_t N = src1->ne[is_2D ? 3 : 2];+ const int32_t IC = src1->ne[is_2D ? 2 : 1];+ const int32_t IH = is_2D ? src1->ne[1] : 1;+ const int32_t IW = src1->ne[0];++ const int32_t KH = is_2D ? src0->ne[1] : 1;+ const int32_t KW = src0->ne[0];++ const int32_t OH = is_2D ? dst->ne[2] : 1;+ const int32_t OW = dst->ne[1];++ const int32_t CHW = IC * KH * KW;++ const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;+ const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;++ id<MTLComputePipelineState> pipeline = nil;++ switch (src0->type) {+ case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break;+ default: GGML_ASSERT(false);+ };++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];+ [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];+ [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];+ [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];+ [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];+ [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];+ [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];+ [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];+ [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];+ [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];+ [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];++ [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];+ } break;+ case GGML_OP_UPSCALE:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);++ const int sf = dst->op_params[0];++ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];+ [encoder setBytes:&sf length:sizeof(sf) atIndex:18];++ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);++ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_PAD:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];++ const int nth = MIN(1024, ne0);++ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ case GGML_OP_ARGSORT:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_I32);++ const int nrows = ggml_nrows(src0);++ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];++ id<MTLComputePipelineState> pipeline = nil;++ switch (order) {+ case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break;+ case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break;+ default: GGML_ASSERT(false);+ };++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];++ [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)];+ } break;+ case GGML_OP_LEAKY_RELU:+ {+ GGML_ASSERT(src0->type == GGML_TYPE_F32);++ float slope;+ memcpy(&slope, dst->op_params, sizeof(float));++ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline;++ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&slope length:sizeof(slope) atIndex:2];++ const int64_t n = ggml_nelements(dst);++ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];+ } break;+ case GGML_OP_DUP:+ case GGML_OP_CPY:+ case GGML_OP_CONT:+ {+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);++ int nth = MIN(1024, ne00/ggml_blck_size(src0->type));++ id<MTLComputePipelineState> pipeline = nil;++ switch (src0t) {+ case GGML_TYPE_F32:+ {+ GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);++ switch (dstt) {+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;+ //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;+ //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;+ default: GGML_ASSERT(false && "not implemented");+ };+ } break;+ case GGML_TYPE_F16:+ {+ switch (dstt) {+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break;+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break;+ default: GGML_ASSERT(false && "not implemented");+ };+ } break;+ default: GGML_ASSERT(false && "not implemented");}- }- }- if (encoder != nil) {- [encoder endEncoding];- encoder = nil;+ [encoder setComputePipelineState:pipeline];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];++ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];+ } break;+ default:+ {+ GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));+ GGML_ASSERT(false);+ }}- [command_buffer commit];- });- }+#ifndef GGML_METAL_NDEBUG+ [encoder popDebugGroup];+#endif+ }- // wait for all threads to finish- dispatch_barrier_sync(ctx->d_queue, ^{});+ [encoder endEncoding];- // check status of command buffers+ [command_buffer commit];+ });++ // Wait for completion and check status of each command buffer// needed to detect if the device ran out-of-memory for example (#1881)- for (int i = 0; i < n_cb; i++) {- [ctx->command_buffers[i] waitUntilCompleted];- MTLCommandBufferStatus status = (MTLCommandBufferStatus) [ctx->command_buffers[i] status];+ for (int i = 0; i < n_cb; ++i) {+ id<MTLCommandBuffer> command_buffer = command_buffers[i];+ [command_buffer waitUntilCompleted];++ MTLCommandBufferStatus status = [command_buffer status];if (status != MTLCommandBufferStatusCompleted) {GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);return false;@@ -2470,7 +2242,6 @@ bool ggml_metal_graph_compute(}return true;- }}////////////////////////////////////////////////////////////////////////////////@@ -2502,10 +2273,10 @@ static void ggml_backend_metal_free_device(void) {}}-GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {- struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;+GGML_CALL static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) {+ return "Metal";- return ctx->all_data;+ UNUSED(buffer);}GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {@@ -2520,7 +2291,13 @@ GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_free(ctx->all_data);}- g_backend->free(ctx);+ free(ctx);+}++GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;++ return ctx->all_data;}GGML_CALL static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {@@ -2535,14 +2312,12 @@ GGML_CALL static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_tUNUSED(buffer);}-GGML_CALL static void ggml_backend_metal_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {- g_backend->ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));-- UNUSED(buffer);-}--GGML_CALL static void ggml_backend_metal_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {- g_backend->ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));+GGML_CALL static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {+ if (ggml_backend_buffer_is_host(src->buffer)) {+ memcpy(dst->data, src->data, ggml_nbytes(src));+ return true;+ }+ return false;UNUSED(buffer);}@@ -2554,20 +2329,46 @@ GGML_CALL static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buff}static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = {+ /* .get_name = */ ggml_backend_metal_buffer_get_name,/* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,/* .get_base = */ ggml_backend_metal_buffer_get_base,/* .init_tensor = */ NULL,/* .set_tensor = */ ggml_backend_metal_buffer_set_tensor,/* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,- /* .cpy_tensor_from = */ ggml_backend_metal_buffer_cpy_tensor_from,- /* .cpy_tensor_to = */ ggml_backend_metal_buffer_cpy_tensor_to,+ /* .cpy_tensor = */ ggml_backend_metal_buffer_cpy_tensor,/* .clear = */ ggml_backend_metal_buffer_clear,+ /* .reset = */ NULL,};// default buffer type+GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) {+ return "Metal";++ UNUSED(buft);+}++static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {+#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)+ if (@available(macOS 10.12, iOS 16.0, *)) {+ GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",+ device.currentAllocatedSize / 1024.0 / 1024.0,+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);++ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);+ } else {+ GGML_METAL_LOG_INFO("\n");+ }+ } else {+ GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);+ }+#endif+ UNUSED(device);+}+GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- struct ggml_backend_metal_buffer_context * ctx = g_backend->malloc(sizeof(struct ggml_backend_metal_buffer_context));+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));const size_t size_page = sysconf(_SC_PAGESIZE);@@ -2592,30 +2393,15 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffif (ctx->buffers[0].metal == nil) {GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);- g_backend->free(ctx);+ free(ctx);ggml_backend_metal_free_device();return NULL;}GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);+ ggml_backend_metal_log_allocated_size(device);--#if TARGET_OS_OSX- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",- device.currentAllocatedSize / 1024.0 / 1024.0,- device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);-- if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);- } else {- GGML_METAL_LOG_INFO("\n");- }-#else- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);-#endif--- return g_backend->ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);+ return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);}GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {@@ -2624,7 +2410,7 @@ GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backen}GGML_CALL static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {- return ggml_backend_is_metal(backend) || g_backend->ggml_backend_is_cpu(backend);+ return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);UNUSED(buft);}@@ -2638,6 +2424,7 @@ GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_bufferGGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = {/* .iface = */ {+ /* .get_name = */ ggml_backend_metal_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes@@ -2653,7 +2440,7 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {// buffer from ptrGGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {- struct ggml_backend_metal_buffer_context * ctx = g_backend->malloc(sizeof(struct ggml_backend_metal_buffer_context));+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));ctx->all_data = data;ctx->all_size = size;@@ -2661,6 +2448,14 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,ctx->n_buffers = 0;const size_t size_page = sysconf(_SC_PAGESIZE);++ // page-align the data ptr+ {+ const uintptr_t offs = (uintptr_t) data % size_page;+ data = (void *) ((char *) data - offs);+ size += offs;+ }+size_t size_aligned = size;if ((size_aligned % size_page) != 0) {size_aligned += (size_page - (size_aligned % size_page));@@ -2712,21 +2507,9 @@ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data,}}-#if TARGET_OS_OSX- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",- device.currentAllocatedSize / 1024.0 / 1024.0,- device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);-- if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);- } else {- GGML_METAL_LOG_INFO("\n");- }-#else- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);-#endif+ ggml_backend_metal_log_allocated_size(device);- return g_backend->ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);+ return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);}// backend@@ -2740,7 +2523,7 @@ GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;ggml_metal_free(ctx);- g_backend->free(backend);+ free(backend);}GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {@@ -2756,19 +2539,18 @@ GGML_CALL static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, s}GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {- return ggml_metal_supports_op(op);+ struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;- UNUSED(backend);+ return ggml_metal_supports_op(metal_ctx, op);}-static struct ggml_backend_i metal_backend_i = {+static struct ggml_backend_i ggml_backend_metal_i = {/* .get_name = */ ggml_backend_metal_name,/* .free = */ ggml_backend_metal_free,/* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type,/* .set_tensor_async = */ NULL,/* .get_tensor_async = */ NULL,- /* .cpy_tensor_from_async = */ NULL,- /* .cpy_tensor_to_async = */ NULL,+ /* .cpy_tensor_async = */ NULL,/* .synchronize = */ NULL,/* .graph_plan_create = */ NULL,/* .graph_plan_free = */ NULL,@@ -2777,6 +2559,12 @@ static struct ggml_backend_i metal_backend_i = {/* .supports_op = */ ggml_backend_metal_supports_op,};+void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {+ if (log_callback) return; // [jart] unsafe across dso boundary+ ggml_metal_log_callback = log_callback;+ ggml_metal_log_user_data = user_data;+}+ggml_backend_t ggml_backend_metal_init(void) {struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);@@ -2784,10 +2572,10 @@ ggml_backend_t ggml_backend_metal_init(void) {return NULL;}- ggml_backend_t metal_backend = g_backend->malloc(sizeof(struct ggml_backend));+ ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));*metal_backend = (struct ggml_backend) {- /* .interface = */ metal_backend_i,+ /* .interface = */ ggml_backend_metal_i,/* .context = */ ctx,};@@ -2795,7 +2583,7 @@ ggml_backend_t ggml_backend_metal_init(void) {}bool ggml_backend_is_metal(ggml_backend_t backend) {- return backend->iface.get_name == ggml_backend_metal_name;+ return backend && backend->iface.get_name == ggml_backend_metal_name;}void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {@@ -2803,7 +2591,7 @@ void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;- ggml_metal_set_n_cb(ctx, n_cb);+ ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);}bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex 229efb8b..029578dc 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -2452,6 +2452,13 @@ typedef struct {} block_iq2_xxs;// 66 bytes / block for QK_K = 256, so 2.0625 bpw+typedef struct {+ half d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+// 74 bytes / block for QK_K = 256, so 2.3125 bpw+//====================================== dot products =========================void kernel_mul_mv_q2_K_f32_impl(@@ -3476,7 +3483,7 @@ kernel void kernel_mul_mv_q6_K_f32(// ======================= "True" 2-bit-constexpr constant static uint64_t kgrid_iq2xxs[256] = {+constexpr constant static uint64_t iq2xxs_grid[256] = {0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,@@ -3543,6 +3550,137 @@ constexpr constant static uint64_t kgrid_iq2xxs[256] = {0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,};+constexpr constant static uint64_t iq2xs_grid[512] = {+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+};+constexpr constant static uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -3600,7 +3738,7 @@ void kernel_mul_mv_iq2_xxs_f32_impl({int nval = 4;int pos = (32*sgitg + tiisg)*nval;- for (int i = 0; i < nval; ++i) values[pos + i] = kgrid_iq2xxs[pos + i];+ for (int i = 0; i < nval; ++i) values[pos + i] = iq2xxs_grid[pos + i];nval = 2;pos = (32*sgitg + tiisg)*nval;for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];@@ -3689,6 +3827,149 @@ kernel void kernel_mul_mv_iq2_xxs_f32(kernel_mul_mv_iq2_xxs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);}+void kernel_mul_mv_iq2_xs_f32_impl(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant int64_t & ne10,+ constant int64_t & ne12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ const int nb = ne00/QK_K;+ const int r0 = tgpig.x;+ const int r1 = tgpig.y;+ const int im = tgpig.z;++ const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int ib_row = first_row * nb;++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ device const block_iq2_xs * x = (device const block_iq2_xs *) src0 + ib_row + offset0;+ device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1;++ float yl[32];+ float sumf[N_DST]={0.f}, all_sum;++ const int nb32 = nb * (QK_K / 32);++ threadgroup uint64_t * values = (threadgroup uint64_t *)shared_values;+ threadgroup uint8_t * shared_signs = (threadgroup uint8_t *)(values + 512);+ {+ int nval = 8;+ int pos = (32*sgitg + tiisg)*nval;+ for (int i = 0; i < nval; ++i) values[pos + i] = iq2xs_grid[pos + i];+ nval = 2;+ pos = (32*sgitg + tiisg)*nval;+ for (int i = 0; i < nval; ++i) shared_signs[pos+i] = ksigns_iq2xs[pos+i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++#if QK_K == 256+ const int ix = tiisg;++ device const float * y4 = y + 32 * ix;++ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {++ for (int i = 0; i < 32; ++i) {+ yl[i] = y4[i];+ }++ const int ibl = ib32 / (QK_K / 32);+ const int ib = ib32 % (QK_K / 32);++ device const block_iq2_xs * xr = x + ibl;+ device const uint16_t * q2 = xr->qs + 4 * ib;+ device const uint8_t * sc = xr->scales + ib;+ device const half * dh = &xr->d;++ for (int row = 0; row < N_DST; row++) {++ const float db = dh[0];+ const uint8_t ls1 = sc[0] & 0xf;+ const uint8_t ls2 = sc[0] >> 4;+ const float d1 = db * (0.5f + ls1);+ const float d2 = db * (0.5f + ls2);++ float sum1 = 0, sum2 = 0;+ for (int l = 0; l < 2; ++l) {+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511));+ const uint8_t signs = shared_signs[(q2[l] >> 9)];+ for (int j = 0; j < 8; ++j) {+ sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ }+ for (int l = 2; l < 4; ++l) {+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(values + (q2[l] & 511));+ const uint8_t signs = shared_signs[(q2[l] >> 9)];+ for (int j = 0; j < 8; ++j) {+ sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ }+ sumf[row] += d1 * sum1 + d2 * sum2;++ dh += nb*sizeof(block_iq2_xs)/2;+ q2 += nb*sizeof(block_iq2_xs)/2;+ sc += nb*sizeof(block_iq2_xs);+ }++ y4 += 32 * 32;+ }+#else+ // TODO+#endif++ for (int row = 0; row < N_DST; ++row) {+ all_sum = simd_sum(sumf[row]);+ if (tiisg == 0) {+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum * 0.25f;+ }+ }+}++[[host_name("kernel_mul_mv_iq2_xs_f32")]]+kernel void kernel_mul_mv_iq2_xs_f32(+ device const void * src0,+ device const float * src1,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ kernel_mul_mv_iq2_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg);+}+//============================= templates and their specializations =============================// NOTE: this is not dequantizing - we are simply fitting the template@@ -3973,18 +4254,39 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4xconst uint32_t aux32_s = q2[2] | (q2[3] << 16);thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g;const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f;- constant uint8_t * grid = (constant uint8_t *)(kgrid_iq2xxs + aux8[2*il+0]);+ constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]);uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127];for (int i = 0; i < 8; ++i) {reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);}- grid = (constant uint8_t *)(kgrid_iq2xxs + aux8[2*il+1]);+ grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]);signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127];for (int i = 0; i < 8; ++i) {reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);}}+template <typename type4x4>+void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) {+ // il is 0...15 for QK_K = 256 => index of block of 32 is il/2+ const float d = xb->d;+ const int ib32 = il/2;+ il = il%2;+ // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16+ device const uint16_t * q2 = xb->qs + 4*ib32;+ const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f;+ constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511));+ uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9];+ for (int i = 0; i < 8; ++i) {+ reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);+ }+ grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511));+ signs = ksigns_iq2xs[q2[2*il+1] >> 9];+ for (int i = 0; i < 8; ++i) {+ reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);+ }+}+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>kernel void kernel_get_rows(device const void * src0,@@ -4525,6 +4827,7 @@ template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_t kernel_get_rowstemplate [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_t kernel_get_rows<block_q5_K, QK_NL, dequantize_q5_K>;template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_t kernel_get_rows<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_t kernel_get_rows<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;+template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_t kernel_get_rows<block_iq2_xs, QK_NL, dequantize_iq2_xs>;//// matrix-matrix multiplication@@ -4562,6 +4865,7 @@ template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mat_mm_t kernel_mul_mm<btemplate [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q5_K, QK_NL, dequantize_q5_K>;template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;+template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mat_mm_t kernel_mul_mm<block_iq2_xs, QK_NL, dequantize_iq2_xs>;//// indirect matrix-matrix multiplication@@ -4611,6 +4915,7 @@ template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mat_mm_id_t kernel_mutemplate [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q5_K, QK_NL, dequantize_q5_K>;template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_q6_K, QK_NL, dequantize_q6_K>;template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;+template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id<block_iq2_xs, QK_NL, dequantize_iq2_xs>;//// matrix-vector multiplication@@ -5448,3 +5753,68 @@ kernel void kernel_mul_mv_id_iq2_xxs_f32(tiisg,sgitg);}++[[host_name("kernel_mul_mv_id_iq2_xs_f32")]]+kernel void kernel_mul_mv_id_iq2_xs_f32(+ device const char * ids,+ device const char * src1,+ device float * dst,+ constant uint64_t & nbi1,+ constant int64_t & ne00,+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant int64_t & ne13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint64_t & nb1,+ constant uint & r2,+ constant uint & r3,+ constant int & idx,+ device const char * src00,+ device const char * src01,+ device const char * src02,+ device const char * src03,+ device const char * src04,+ device const char * src05,+ device const char * src06,+ device const char * src07,+ threadgroup int8_t * shared_values [[threadgroup(0)]],+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {+ device const char * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};++ const int64_t bid = tgpig.z/(ne12*ne13);++ tgpig.z = tgpig.z%(ne12*ne13);++ const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];++ kernel_mul_mv_iq2_xs_f32_impl(+ src0[id],+ (device const float *) (src1 + bid*nb11),+ dst + bid*ne0,+ ne00,+ ne01,+ ne02,+ ne10,+ ne12,+ ne0,+ ne1,+ r2,+ r3,+ shared_values,+ tgpig,+ tiisg,+ sgitg);+}diff --git a/llama.cpp/ggml-quants.c b/llama.cpp/ggml-quants.cindex 4da75520..ed4b6d5f 100644--- a/llama.cpp/ggml-quants.c+++ b/llama.cpp/ggml-quants.c@@ -306,10 +306,13 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128// vaddvq_s16// vpaddq_s16+// vpaddq_s32// vaddvq_s32// vaddvq_f32// vmaxvq_f32// vcvtnq_s32_f32+// vzip1_u8+// vzip2_u8inline static int32_t vaddvq_s16(int16x8_t v) {return@@ -325,6 +328,12 @@ inline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {return vcombine_s16(a0, b0);}+inline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) {+ int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));+ int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));+ return vcombine_s32(a0, b0);+}+inline static int32_t vaddvq_s32(int32x4_t v) {return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);}@@ -350,6 +359,28 @@ inline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {return res;}+inline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) {+ uint8x8_t res;++ res[0] = a[0]; res[1] = b[0];+ res[2] = a[1]; res[3] = b[1];+ res[4] = a[2]; res[5] = b[2];+ res[6] = a[3]; res[7] = b[3];++ return res;+}++inline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) {+ uint8x8_t res;++ res[0] = a[4]; res[1] = b[4];+ res[2] = a[5]; res[3] = b[5];+ res[4] = a[6]; res[5] = b[6];+ res[6] = a[7]; res[7] = b[7];++ return res;+}+// vld1q_s16_x2// vld1q_u8_x2// vld1q_u8_x4@@ -1245,7 +1276,8 @@ static inline int nearest_int(float fval) {return (i & 0x007fffff) - 0x00400000;}-static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, int rmse_type) {+static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * restrict L, int rmse_type,+ const float * restrict qw) {float max = 0;float amax = 0;for (int i = 0; i < n; ++i) {@@ -1271,14 +1303,18 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t *rmse_type = -rmse_type;return_early = true;}- int weight_type = rmse_type%2;float sumlx = 0;float suml2 = 0;+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 0; i < n; ++i) {+#elsefor (int i = 0; i < n; ++i) {+#endifint l = nearest_int(iscale * x[i]);l = MAX(-nmax, MIN(nmax-1, l));L[i] = l + nmax;- float w = weight_type == 1 ? x[i] * x[i] : 1;+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));sumlx += w*x[i]*l;suml2 += w*l*l;}@@ -1294,7 +1330,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t *for (int i = 0; i < n; ++i) {int l = nearest_int(iscale * x[i]);l = MAX(-nmax, MIN(nmax-1, l));- float w = weight_type == 1 ? x[i] * x[i] : 1;+ float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i]));sumlx += w*x[i]*l;suml2 += w*l*l;}@@ -1642,6 +1678,246 @@ size_t ggml_quantize_q2_K(const float * restrict src, void * restrict dst, int nreturn (n/QK_K*sizeof(block_q2_K));}+static float make_qkx3_quants(int n, int nmax, const float * restrict x, const float * restrict weights,+ uint8_t * restrict L, float * restrict the_min, uint8_t * restrict Laux,+ float rmin, float rdelta, int nstep, bool use_mad) {+ float min = x[0];+ float max = x[0];+ float sum_w = weights ? weights[0] : x[0]*x[0];+ float sum_x = sum_w * x[0];+#ifdef HAVE_BUGGY_APPLE_LINKER+ // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7+ for (volatile int i = 1; i < n; ++i) {+#else+ for (int i = 1; i < n; ++i) {+#endif+ if (x[i] < min) min = x[i];+ if (x[i] > max) max = x[i];+ float w = weights ? weights[i] : x[i]*x[i];+ sum_w += w;+ sum_x += w * x[i];+ }+ if (min > 0) {+ min = 0;+ }+ if (max <= min) {+ memset(L, 0, n);+ *the_min = -min;+ return 0.f;+ }+ float iscale = nmax/(max - min);+ float scale = 1/iscale;+ float best_mad = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ L[i] = MAX(0, MIN(nmax, l));+ float diff = scale * L[i] + min - x[i];+ diff = use_mad ? fabsf(diff) : diff*diff;+ float w = weights ? weights[i] : x[i]*x[i];+ best_mad += w * diff;+ }+ if (nstep < 1) {+ *the_min = -min;+ return scale;+ }+ for (int is = 0; is <= nstep; ++is) {+ iscale = (rmin + rdelta*is + nmax)/(max - min);+ float sum_l = 0, sum_l2 = 0, sum_xl = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale*(x[i] - min));+ l = MAX(0, MIN(nmax, l));+ Laux[i] = l;+ float w = weights ? weights[i] : x[i]*x[i];+ sum_l += w*l;+ sum_l2 += w*l*l;+ sum_xl += w*l*x[i];+ }+ float D = sum_w * sum_l2 - sum_l * sum_l;+ if (D > 0) {+ float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D;+ float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D;+ if (this_min > 0) {+ this_min = 0;+ this_scale = sum_xl / sum_l2;+ }+ float mad = 0;+ for (int i = 0; i < n; ++i) {+ float diff = this_scale * Laux[i] + this_min - x[i];+ diff = use_mad ? fabsf(diff) : diff*diff;+ float w = weights ? weights[i] : x[i]*x[i];+ mad += w * diff;+ }+ if (mad < best_mad) {+ for (int i = 0; i < n; ++i) {+ L[i] = Laux[i];+ }+ best_mad = mad;+ scale = this_scale;+ min = this_min;+ }+ }+ }+ *the_min = -min;+ return scale;+}++static float make_qp_quants(int n, int nmax, const float * restrict x, uint8_t * restrict L, const float * quant_weights) {+ float max = 0;+ for (int i = 0; i < n; ++i) {+ max = MAX(max, x[i]);+ }+ if (!max) { // all zero+ for (int i = 0; i < n; ++i) { L[i] = 0; }+ return 0.f;+ }+ float iscale = nmax / max;+ for (int i = 0; i < n; ++i) {+ L[i] = nearest_int(iscale * x[i]);+ }+ float scale = 1/iscale;+ float best_mse = 0;+ for (int i = 0; i < n; ++i) {+ float diff = x[i] - scale*L[i];+ float w = quant_weights[i];+ best_mse += w*diff*diff;+ }+ for (int is = -4; is <= 4; ++is) {+ if (is == 0) continue;+ float iscale_is = (0.1f*is + nmax)/max;+ float scale_is = 1/iscale_is;+ float mse = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale_is*x[i]);+ l = MIN(nmax, l);+ float diff = x[i] - scale_is*l;+ float w = quant_weights[i];+ mse += w*diff*diff;+ }+ if (mse < best_mse) {+ best_mse = mse;+ iscale = iscale_is;+ }+ }+ float sumlx = 0;+ float suml2 = 0;+ for (int i = 0; i < n; ++i) {+ int l = nearest_int(iscale * x[i]);+ l = MIN(nmax, l);+ L[i] = l;+ float w = quant_weights[i];+ sumlx += w*x[i]*l;+ suml2 += w*l*l;+ }+ for (int itry = 0; itry < 5; ++itry) {+ int n_changed = 0;+ for (int i = 0; i < n; ++i) {+ float w = quant_weights[i];+ float slx = sumlx - w*x[i]*L[i];+ float sl2 = suml2 - w*L[i]*L[i];+ if (slx > 0 && sl2 > 0) {+ int new_l = nearest_int(x[i] * sl2 / slx);+ new_l = MIN(nmax, new_l);+ if (new_l != L[i]) {+ slx += w*x[i]*new_l;+ sl2 += w*new_l*new_l;+ if (slx*slx*suml2 > sumlx*sumlx*sl2) {+ L[i] = new_l; sumlx = slx; suml2 = sl2;+ ++n_changed;+ }+ }+ }+ }+ if (!n_changed) {+ break;+ }+ }+ return sumlx / suml2;+}++static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restrict y, int k, const float * restrict quant_weights) {+ GGML_ASSERT(quant_weights);+ assert(k % QK_K == 0);+ const int nb = k / QK_K;+ const bool requantize = true;++ uint8_t L[QK_K];+ uint8_t Laux[16];+ float mins[QK_K/16];+ float scales[QK_K/16];+ float sw[QK_K/16];+ float weight[QK_K/16];+ uint8_t Ls[QK_K/16], Lm[QK_K/16];++ for (int i = 0; i < nb; i++) {+ memset(sw, 0, QK_K/16*sizeof(float));+ float sumx2 = 0;+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];+ float sigma2 = sumx2/QK_K;+ for (int j = 0; j < QK_K/16; ++j) {+ const float * restrict qw = quant_weights + QK_K * i + 16*j;+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]);+ for (int l = 0; l < 16; ++l) sw[j] += weight[l];+ scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ }++ float dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw);+ float mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw);+ y[i].d = GGML_FP32_TO_FP16(dm);+ y[i].dmin = GGML_FP32_TO_FP16(mm);+ dm = GGML_FP16_TO_FP32(y[i].d);+ mm = GGML_FP16_TO_FP32(y[i].dmin);++ for (int j = 0; j < QK_K/16; ++j) {+ y[i].scales[j] = Ls[j] | (Lm[j] << 4);+ }++ if (requantize) {+ for (int j = 0; j < QK_K/16; ++j) {+ const float d = dm * (y[i].scales[j] & 0xF);+ if (!d) continue;+ const float m = mm * (y[i].scales[j] >> 4);+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int((x[16*j + ii] + m)/d);+ l = MAX(0, MIN(3, l));+ L[16*j + ii] = l;+ }+ }+ }++#if QK_K == 256+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }+#else+ for (int l = 0; l < 16; ++l) {+ y[i].qs[l] = L[l] | (L[l + 16] << 2) | (L[l + 32] << 4) | (L[l + 48] << 6);+ }+#endif++ x += QK_K;++ }+}++size_t quantize_q2_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q2_K_reference(src, dst, nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}+//========================= 3-bit (de)-quantizationvoid quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k) {@@ -1855,6 +2131,112 @@ size_t ggml_quantize_q3_K(const float * restrict src, void * restrict dst, int nreturn (n/QK_K*sizeof(block_q3_K));}+static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrict y, int n_per_row, const float * restrict quant_weights) {+#if QK_K != 256+ (void)quant_weights;+ quantize_row_q3_K_reference(x, y, n_per_row);+#else+ assert(n_per_row % QK_K == 0);+ const int nb = n_per_row / QK_K;++ int8_t L[QK_K];+ float scales[QK_K / 16];+ float weight[16];+ float sw[QK_K / 16];+ int8_t Ls[QK_K / 16];++ for (int i = 0; i < nb; i++) {++ float sumx2 = 0;+ for (int j = 0; j < QK_K; ++j) sumx2 += x[j]*x[j];+ float sigma2 = 2*sumx2/QK_K;++ for (int j = 0; j < QK_K/16; ++j) {+ if (quant_weights) {+ const float * qw = quant_weights ? quant_weights + QK_K * i + 16*j : NULL;+ for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]);+ } else {+ for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l];+ }+ float sumw = 0;+ for (int l = 0; l < 16; ++l) sumw += weight[l];+ sw[j] = sumw;++ scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight);++ }++ memset(y[i].scales, 0, 12);++ float d_block = make_qx_quants(QK_K/16, 32, scales, Ls, 1, sw);+ for (int j = 0; j < QK_K/16; ++j) {+ int l = Ls[j];+ if (j < 8) {+ y[i].scales[j] = l & 0xF;+ } else {+ y[i].scales[j-8] |= ((l & 0xF) << 4);+ }+ l >>= 4;+ y[i].scales[j%4 + 8] |= (l << (2*(j/4)));+ }+ y[i].d = GGML_FP32_TO_FP16(d_block);++ int8_t sc;+ for (int j = 0; j < QK_K/16; ++j) {+ sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4;+ sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32;+ float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-4, MIN(3, l));+ L[16*j + ii] = l + 4;+ }+ }++ memset(y[i].hmask, 0, QK_K/8);+ // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc.+ int m = 0;+ uint8_t hm = 1;+ for (int j = 0; j < QK_K; ++j) {+ if (L[j] > 3) {+ y[i].hmask[m] |= hm;+ L[j] -= 4;+ }+ if (++m == QK_K/8) {+ m = 0; hm <<= 1;+ }+ }+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);+ }+ }++ x += QK_K;+ }+#endif+}++size_t quantize_q3_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q3_K_reference(src, dst, nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}+// ====================== 4-bit (de)-quantizationvoid quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k) {@@ -2020,36 +2402,38 @@ size_t ggml_quantize_q4_K(const float * restrict src, void * restrict dst, int nreturn (n/QK_K*sizeof(block_q4_K));}-// ====================== 5-bit (de)-quantization--void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k) {- assert(k % QK_K == 0);- const int nb = k / QK_K;+static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restrict y, int n_per_row, const float * quant_weights) {+#if QK_K != 256+ (void)quant_weights;+ quantize_row_q4_K_reference(x, y, n_per_row);+#else+ assert(n_per_row % QK_K == 0);+ const int nb = n_per_row / QK_K;-#if QK_K == 256uint8_t L[QK_K];+ uint8_t Laux[32];+ float weights[32];float mins[QK_K/32];float scales[QK_K/32];- float weights[32];- uint8_t Laux[32];-#else- int8_t L[QK_K];- float scales[QK_K/16];-#endiffor (int i = 0; i < nb; i++) {-#if QK_K == 256+ float sum_x2 = 0;+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];+ float sigma2 = sum_x2/QK_K;+ float av_x = sqrtf(sigma2);float max_scale = 0; // as we are deducting the min, scales are always positivefloat max_min = 0;for (int j = 0; j < QK_K/32; ++j) {- //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);- float sum_x2 = 0;- for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];- float av_x = sqrtf(sum_x2/32);- for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);- scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false);+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 32*j;+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);+ } else {+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ }+ scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ //scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false);float scale = scales[j];if (scale > max_scale) {max_scale = scale;@@ -2087,15 +2471,115 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrictconst float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;for (int ii = 0; ii < 32; ++ii) {int l = nearest_int((x[32*j + ii] + dm)/d);- l = MAX(0, MIN(31, l));+ l = MAX(0, MIN(15, l));L[32*j + ii] = l;}}+ uint8_t * q = y[i].qs;+ for (int j = 0; j < QK_K; j += 64) {+ for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4);+ q += 32;+ }++ x += QK_K;++ }+#endif+}++size_t quantize_q4_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q4_K_reference(src, dst, nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++// ====================== 5-bit (de)-quantization++void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k) {+ assert(k % QK_K == 0);+ const int nb = k / QK_K;++#if QK_K == 256+ uint8_t L[QK_K];+ float mins[QK_K/32];+ float scales[QK_K/32];+ float weights[32];+ uint8_t Laux[32];+#else+ int8_t L[QK_K];+ float scales[QK_K/16];+#endif++ for (int i = 0; i < nb; i++) {++#if QK_K == 256++ float max_scale = 0; // as we are deducting the min, scales are always positive+ float max_min = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f);+ float sum_x2 = 0;+ for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l];+ float av_x = sqrtf(sum_x2/32);+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false);+ float scale = scales[j];+ if (scale > max_scale) {+ max_scale = scale;+ }+ float min = mins[j];+ if (min > max_min) {+ max_min = min;+ }+ }++ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;+ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = nearest_int(inv_scale*scales[j]);+ uint8_t lm = nearest_int(inv_min*mins[j]);+ ls = MIN(63, ls);+ lm = MIN(63, lm);+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(31, l));+ L[32*j + ii] = l;+ }+ }++ uint8_t * restrict qh = y[i].qh;+ uint8_t * restrict ql = y[i].qs;+ memset(qh, 0, QK_K/8);- uint8_t * restrict qh = y[i].qh;- uint8_t * restrict ql = y[i].qs;- memset(qh, 0, QK_K/8);-uint8_t m1 = 1, m2 = 2;for (int n = 0; n < QK_K; n += 64) {for (int j = 0; j < 32; ++j) {@@ -2115,7 +2599,7 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict#elsefloat max_scale = 0, amax = 0;for (int j = 0; j < QK_K/16; ++j) {- scales[j] = make_qx_quants(16, 16, x + 16*j, L + 16*j, 1);+ scales[j] = make_qx_quants(16, 16, x + 16*j, L + 16*j, 1, NULL);float abs_scale = fabsf(scales[j]);if (abs_scale > amax) {amax = abs_scale;@@ -2226,6 +2710,123 @@ size_t ggml_quantize_q5_K(const float * restrict src, void * restrict dst, int nreturn (n/QK_K*sizeof(block_q5_K));}+static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restrict y, int n_per_row, const float * quant_weights) {+#if QK_K != 256+ (void)quant_weights;+ quantize_row_q5_K_reference(x, y, n_per_row);+#else+ assert(n_per_row % QK_K == 0);+ const int nb = n_per_row / QK_K;++ uint8_t L[QK_K];+ float mins[QK_K/32];+ float scales[QK_K/32];+ float weights[32];+ uint8_t Laux[32];++ for (int i = 0; i < nb; i++) {++ float sum_x2 = 0;+ for (int l = 0; l < QK_K; ++l) sum_x2 += x[l] * x[l];+ float sigma2 = sum_x2/QK_K;+ float av_x = sqrtf(sigma2);++ float max_scale = 0; // as we are deducting the min, scales are always positive+ float max_min = 0;+ for (int j = 0; j < QK_K/32; ++j) {+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 32*j;+ for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]);+ } else {+ for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]);+ }+ scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false);+ float scale = scales[j];+ if (scale > max_scale) {+ max_scale = scale;+ }+ float min = mins[j];+ if (min > max_min) {+ max_min = min;+ }+ }++ float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f;+ float inv_min = max_min > 0 ? 63.f/max_min : 0.f;+ for (int j = 0; j < QK_K/32; ++j) {+ uint8_t ls = nearest_int(inv_scale*scales[j]);+ uint8_t lm = nearest_int(inv_min*mins[j]);+ ls = MIN(63, ls);+ lm = MIN(63, lm);+ if (j < 4) {+ y[i].scales[j] = ls;+ y[i].scales[j+4] = lm;+ } else {+ y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4);+ y[i].scales[j-4] |= ((ls >> 4) << 6);+ y[i].scales[j-0] |= ((lm >> 4) << 6);+ }+ }+ y[i].d = GGML_FP32_TO_FP16(max_scale/63.f);+ y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f);++ uint8_t sc, m;+ for (int j = 0; j < QK_K/32; ++j) {+ get_scale_min_k4(j, y[i].scales, &sc, &m);+ const float d = GGML_FP16_TO_FP32(y[i].d) * sc;+ if (!d) continue;+ const float dm = GGML_FP16_TO_FP32(y[i].dmin) * m;+ for (int ii = 0; ii < 32; ++ii) {+ int l = nearest_int((x[32*j + ii] + dm)/d);+ l = MAX(0, MIN(31, l));+ L[32*j + ii] = l;+ }+ }++ uint8_t * restrict qh = y[i].qh;+ uint8_t * restrict ql = y[i].qs;+ memset(qh, 0, QK_K/8);++ uint8_t m1 = 1, m2 = 2;+ for (int n = 0; n < QK_K; n += 64) {+ for (int j = 0; j < 32; ++j) {+ int l1 = L[n + j];+ if (l1 > 15) {+ l1 -= 16; qh[j] |= m1;+ }+ int l2 = L[n + j + 32];+ if (l2 > 15) {+ l2 -= 16; qh[j] |= m2;+ }+ ql[j] = l1 | (l2 << 4);+ }+ m1 <<= 2; m2 <<= 2;+ ql += 32;+ }++ x += QK_K;++ }+#endif+}++size_t quantize_q5_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q5_K_reference(src, dst, nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}+// ====================== 6-bit (de)-quantizationvoid quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k) {@@ -2242,7 +2843,7 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrictfor (int ib = 0; ib < QK_K/16; ++ib) {- const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1);+ const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);scales[ib] = scale;const float abs_scale = fabsf(scale);@@ -2351,40 +2952,329 @@ void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, inty[l+32] = d * sc[2] * q3;y[l+48] = d * sc[3] * q4;}- y += 64;-#endif+ y += 64;+#endif++ }+}++void quantize_row_q6_K(const float * restrict x, void * restrict vy, int k) {+ assert(k % QK_K == 0);+ block_q6_K * restrict y = vy;+ quantize_row_q6_K_reference(x, y, k);+}++size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist) {+ assert(k % QK_K == 0);+ (void)hist; // TODO: collect histograms++ for (int j = 0; j < n; j += k) {+ block_q6_K * restrict y = (block_q6_K *)dst + j/QK_K;+ quantize_row_q6_K_reference(src + j, y, k);+ }+ return (n/QK_K*sizeof(block_q6_K));+}++static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restrict y, int n_per_row, const float * quant_weights) {+#if QK_K != 256+ (void)quant_weights;+ quantize_row_q6_K_reference(x, y, n_per_row);+#else+ assert(n_per_row % QK_K == 0);+ const int nb = n_per_row / QK_K;++ int8_t L[QK_K];+ float scales[QK_K/16];+ //float weights[16];++ for (int i = 0; i < nb; i++) {++ //float sum_x2 = 0;+ //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j];+ //float sigma2 = sum_x2/QK_K;++ float max_scale = 0;+ float max_abs_scale = 0;++ for (int ib = 0; ib < QK_K/16; ++ib) {++ float scale;+ if (quant_weights) {+ const float * qw = quant_weights + QK_K*i + 16*ib;+ //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]);+ //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights);+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw);+ } else {+ scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL);+ }+ scales[ib] = scale;++ const float abs_scale = fabsf(scale);+ if (abs_scale > max_abs_scale) {+ max_abs_scale = abs_scale;+ max_scale = scale;+ }++ }++ if (!max_abs_scale) {+ memset(&y[i], 0, sizeof(block_q6_K));+ y[i].d = GGML_FP32_TO_FP16(0.f);+ x += QK_K;+ continue;+ }++ float iscale = -128.f/max_scale;+ y[i].d = GGML_FP32_TO_FP16(1/iscale);+ for (int ib = 0; ib < QK_K/16; ++ib) {+ y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));+ }++ for (int j = 0; j < QK_K/16; ++j) {+ float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j];+ if (!d) {+ continue;+ }+ for (int ii = 0; ii < 16; ++ii) {+ int l = nearest_int(x[16*j + ii]/d);+ l = MAX(-32, MIN(31, l));+ L[16*j + ii] = l + 32;+ }+ }++ uint8_t * restrict ql = y[i].ql;+ uint8_t * restrict qh = y[i].qh;+ for (int j = 0; j < QK_K; j += 128) {+ for (int l = 0; l < 32; ++l) {+ const uint8_t q1 = L[j + l + 0] & 0xF;+ const uint8_t q2 = L[j + l + 32] & 0xF;+ const uint8_t q3 = L[j + l + 64] & 0xF;+ const uint8_t q4 = L[j + l + 96] & 0xF;+ ql[l+ 0] = q1 | (q3 << 4);+ ql[l+32] = q2 | (q4 << 4);+ qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6);+ }+ ql += 64;+ qh += 32;+ }++ x += QK_K;++ }+#endif+}++size_t quantize_q6_K(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row);+ if (!quant_weights) {+ quantize_row_q6_K_reference(src, dst, nrow*n_per_row);+ }+ else {+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ }+ return nrow * row_size;+}++static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restrict y, int n_per_row, const float * quant_weights) {+ static_assert(QK4_0 == 32, "QK4_0 must be 32");++ if (!quant_weights) {+ quantize_row_q4_0_reference(x, y, n_per_row);+ return;+ }++ float weight[QK4_0];+ int8_t L[QK4_0];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int nb = n_per_row/QK4_0;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK4_0 * ib;+ const float * qw = quant_weights + QK4_0 * ib;+ for (int j = 0; j < QK4_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight);+ y[ib].d = GGML_FP32_TO_FP16(d);+ for (int j = 0; j < 16; ++j) {+ y[ib].qs[j] = L[j] | (L[j+16] << 4);+ }+ }+}++size_t quantize_q4_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ if (!quant_weights) {+ return ggml_quantize_q4_0(src, dst, nrow*n_per_row, n_per_row, hist);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row);+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q4_0_impl(src, (block_q4_0*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++static void quantize_row_q4_1_impl(const float * restrict x, block_q4_1 * restrict y, int n_per_row, const float * quant_weights) {+ static_assert(QK4_1 == 32, "QK4_1 must be 32");++ if (!quant_weights) {+ quantize_row_q4_1_reference(x, y, n_per_row);+ return;+ }++ float weight[QK4_1];+ uint8_t L[QK4_1], Laux[QK4_1];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int nb = n_per_row/QK4_1;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK4_1 * ib;+ const float * qw = quant_weights + QK4_1 * ib;+ for (int j = 0; j < QK4_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float min;+ float d = make_qkx3_quants(QK4_1, 15, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);+ y[ib].d = GGML_FP32_TO_FP16(d);+ y[ib].m = GGML_FP32_TO_FP16(-min);+ for (int j = 0; j < 16; ++j) {+ y[ib].qs[j] = L[j] | (L[j+16] << 4);+ }+ }+}++size_t quantize_q4_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ if (!quant_weights) {+ return ggml_quantize_q4_1(src, dst, nrow*n_per_row, n_per_row, hist);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row);+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q4_1_impl(src, (block_q4_1*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;+}++static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restrict y, int n_per_row, const float * quant_weights) {+ static_assert(QK5_0 == 32, "QK5_0 must be 32");++ if (!quant_weights) {+ quantize_row_q5_0_reference(x, y, n_per_row);+ return;+ }++ float weight[QK5_0];+ int8_t L[QK5_0];++ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int nb = n_per_row/QK5_0;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK5_0 * ib;+ const float * qw = quant_weights + QK5_0 * ib;+ for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight);+ y[ib].d = GGML_FP32_TO_FP16(d);++ uint32_t qh = 0;++ for (int j = 0; j < 16; ++j) {+ const uint8_t xi0 = L[j];+ const uint8_t xi1 = L[j+16];+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);++ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);+ }+ memcpy(&y[ib].qh, &qh, sizeof(qh));}}-void quantize_row_q6_K(const float * restrict x, void * restrict vy, int k) {- assert(k % QK_K == 0);- block_q6_K * restrict y = vy;- quantize_row_q6_K_reference(x, y, k);+size_t quantize_q5_0(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ if (!quant_weights) {+ return ggml_quantize_q5_0(src, dst, nrow*n_per_row, n_per_row, hist);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row);+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q5_0_impl(src, (block_q5_0*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;}-size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist) {- assert(k % QK_K == 0);- (void)hist; // TODO: collect histograms+static void quantize_row_q5_1_impl(const float * restrict x, block_q5_1 * restrict y, int n_per_row, const float * quant_weights) {+ static_assert(QK5_1 == 32, "QK5_1 must be 32");- for (int j = 0; j < n; j += k) {- block_q6_K * restrict y = (block_q6_K *)dst + j/QK_K;- quantize_row_q6_K_reference(src + j, y, k);+ if (!quant_weights) {+ quantize_row_q5_1_reference(x, y, n_per_row);+ return;}- return (n/QK_K*sizeof(block_q6_K));-}-// ====================== "True" 2-bit (de)-quantization+ float weight[QK5_1];+ uint8_t L[QK5_1], Laux[QK5_1];-void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k) {- (void)x;- (void)y;- (void)k;- assert(k % QK_K == 0);- //fprintf(stderr, "=========================== %s: not implemented\n", __func__);+ float sum_x2 = 0;+ for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];+ float sigma2 = sum_x2/n_per_row;++ const int nb = n_per_row/QK5_1;+ for (int ib = 0; ib < nb; ++ib) {+ const float * xb = x + QK5_1 * ib;+ const float * qw = quant_weights + QK5_1 * ib;+ for (int j = 0; j < QK5_1; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);+ float min;+ float d = make_qkx3_quants(QK5_1, 31, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false);+ y[ib].d = GGML_FP32_TO_FP16(d);+ y[ib].m = GGML_FP32_TO_FP16(-min);++ uint32_t qh = 0;+ for (int j = 0; j < 16; ++j) {+ const uint8_t xi0 = L[j];+ const uint8_t xi1 = L[j+16];+ y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);+ // get the 5-th bit and store it in qh at the right position+ qh |= ((xi0 & 0x10u) >> 4) << (j + 0);+ qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);+ }+ memcpy(&y[ib].qh, &qh, sizeof(qh));+ }+}++size_t quantize_q5_1(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ if (!quant_weights) {+ return ggml_quantize_q5_1(src, dst, nrow*n_per_row, n_per_row, hist);+ }+ size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row);+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_q5_1_impl(src, (block_q5_1*)qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += row_size;+ }+ return nrow * row_size;}-static const uint64_t iq2xxs_grid[256] = {+// ====================== "True" 2-bit (de)-quantization++static const uint64_t iq2xxs_grid[256] = {0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,@@ -2451,6 +3341,137 @@ static const uint64_t iq2xxs_grid[256] = {0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,};+static const uint64_t iq2xs_grid[512] = {+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+};+static const uint8_t ksigns_iq2xs[128] = {0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,@@ -2461,6 +3482,7 @@ static const uint8_t ksigns_iq2xs[128] = {96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,};+static const uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k) {@@ -2489,21 +3511,31 @@ void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y}}-void quantize_row_iq2_xxs(const float * restrict x, void * restrict vy, int k) {- assert(k % QK_K == 0);- block_iq2_xxs * restrict y = vy;- quantize_row_iq2_xxs_reference(x, y, k);-}+// ====================== 2.3125 bpw (de)-quantization-size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist) {+void dequantize_row_iq2_xs(const block_iq2_xs * restrict x, float * restrict y, int k) {assert(k % QK_K == 0);- (void)hist; // TODO: collect histograms+ const int nb = k / QK_K;- for (int j = 0; j < n; j += k) {- block_iq2_xxs * restrict y = (block_iq2_xxs *)dst + j/QK_K;- quantize_row_iq2_xxs_reference(src + j, y, k);+ float db[2];++ for (int i = 0; i < nb; i++) {++ const float d = GGML_FP16_TO_FP32(x[i].d);++ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f;+ db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f;+ for (int l = 0; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511));+ const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9];+ for (int j = 0; j < 8; ++j) {+ y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);+ }+ y += 8;+ }+ }}- return (n/QK_K*sizeof(block_iq2_xxs));}//===================================== Q8_K ==============================================@@ -2868,11 +3900,9 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;#ifdef __ARM_NEON-const uint8x16_t m3 = vdupq_n_u8(0x3);-#if defined(__ARM_FEATURE_DOTPROD)- const int32x4_t vzero = vdupq_n_s32(0);-#endif++ const int32x4_t vzero = vdupq_n_s32(0);ggml_int8x16x4_t q2bytes;@@ -2906,28 +3936,12 @@ void ggml_vec_dot_q2_K_q8_K(const int n, float * restrict s, const void * restriq2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3));q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3));-#if defined(__ARM_FEATURE_DOTPROD)- isum1 += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];- isum2 += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];- isum1 += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];- isum2 += vaddvq_s32(vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];-#else- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q2bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q2bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum1 += vaddvq_s16(p1) * scales[0];- isum2 += vaddvq_s16(p2) * scales[1];-- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q2bytes.val[2]), vget_high_s8(q8bytes.val[2])));- const int16x8_t p4 = vaddq_s16(vmull_s8(vget_low_s8 (q2bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q2bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum1 += vaddvq_s16(p3) * scales[2];- isum2 += vaddvq_s16(p4) * scales[3];-#endif- sum += d * (isum1 + isum2);+ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0];+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1];+ isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2];+ isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3];+ sum += d * (isum1 + isum2);}*s = sum;@@ -3189,10 +4203,7 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;#ifdef __ARM_NEON--#ifdef __ARM_FEATURE_DOTPROD- const int32x4_t vzero = vdupq_n_s32(0);-#endif+ const int32x4_t vzero = vdupq_n_s32(0);const uint8x16_t m3b = vdupq_n_u8(0x3);const uint8x16_t mh = vdupq_n_u8(4);@@ -3233,22 +4244,10 @@ void ggml_vec_dot_q3_K_q8_K(const int n, float * restrict s, const void * restriq3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2]));q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];- isum += vaddvq_s32(vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];-#else- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q3bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q3bytes.val[1]), vget_high_s8(q8bytes.val[1])));- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q3bytes.val[2]), vget_high_s8(q8bytes.val[2])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q3bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q3bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p0) * scales[0] + vaddvq_s16(p1) * scales[2] + vaddvq_s16(p2) * scales[1] + vaddvq_s16(p3) * scales[3];-#endif+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1];+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3];sum += d * isum;@@ -3590,12 +4589,9 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;#ifdef __ARM_NEON-const uint8x16_t m4b = vdupq_n_u8(0xf);-#ifdef __ARM_FEATURE_DOTPRODconst int32x4_t mzero = vdupq_n_s32(0);-#endiffloat sumf = 0;@@ -3623,41 +4619,20 @@ void ggml_vec_dot_q4_K_q8_K(const int n, float * restrict s, const void * restriconst ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4);-#ifdef __ARM_FEATURE_DOTPRODq8bytes = ggml_vld1q_s8_x4(q8);q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- const int32x4_t p1 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);+ const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);const int32_t sumi1 = vaddvq_s32(p1) * scales[0];q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- const int32x4_t p2 = vdotq_s32(vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);+ const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]);const int32_t sumi2 = vaddvq_s32(p2) * scales[1];-#else- q8bytes = ggml_vld1q_s8_x4(q8);- q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b));- q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b));- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[1])));- int32_t sumi1 = vaddvq_s16(vaddq_s16(p0, p1)) * scales[0];-- q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));- q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[0]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q4bytes.val[0]), vget_high_s8(q8bytes.val[2])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q4bytes.val[1]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q4bytes.val[1]), vget_high_s8(q8bytes.val[3])));- int32_t sumi2 = vaddvq_s16(vaddq_s16(p2, p3)) * scales[1];--#endifsumf += d * (sumi1 + sumi2);-}*s = sumf - sum_mins;@@ -3901,12 +4876,9 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;#ifdef __ARM_NEON-const uint8x16_t m4b = vdupq_n_u8(0xf);const uint8x16_t mh = vdupq_n_u8(16);-#if defined(__ARM_FEATURE_DOTPROD)const int32x4_t mzero = vdupq_n_s32(0);-#endifggml_int8x16x4_t q5bytes;ggml_uint8x16x4_t q5h;@@ -3938,32 +4910,12 @@ void ggml_vec_dot_q5_K_q8_K(const int n, float * restrict s, const void * restriq5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2]));q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3]));-#if defined(__ARM_FEATURE_DOTPROD)-- int32_t sumi1 = sc[0] * vaddvq_s32(vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));- int32_t sumi2 = sc[1] * vaddvq_s32(vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));- int32_t sumi3 = sc[2] * vaddvq_s32(vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));- int32_t sumi4 = sc[3] * vaddvq_s32(vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));+ int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]));+ int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1]));+ int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]));+ int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3]));sumf += d * (sumi1 + sumi2 + sumi3 + sumi4);--#else-- const int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q5bytes.val[0]), vget_high_s8(q8bytes.val[0])));- const int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q5bytes.val[1]), vget_high_s8(q8bytes.val[1])));- int32_t sumi = sc[0] * vaddvq_s16(p0) + sc[1] * vaddvq_s16(p1);-- const int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q5bytes.val[2]), vget_high_s8(q8bytes.val[2])));- const int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q5bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q5bytes.val[3]), vget_high_s8(q8bytes.val[3])));- sumi += sc[2] * vaddvq_s16(p2) + sc[3] * vaddvq_s16(p3);-- sumf += d*sumi;-#endif-}*s = sumf;@@ -4234,14 +5186,11 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restriconst int nb = n / QK_K;#ifdef __ARM_NEON-float sum = 0;const uint8x16_t m4b = vdupq_n_u8(0xF);const int8x16_t m32s = vdupq_n_s8(32);-#if defined(__ARM_FEATURE_DOTPROD)const int32x4_t vzero = vdupq_n_s32(0);-#endifconst uint8x16_t mone = vdupq_n_u8(3);@@ -4277,26 +5226,10 @@ void ggml_vec_dot_q6_K_q8_K(const int n, float * restrict s, const void * restriq6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[2])), m32s);q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[3])), m32s);-#if defined(__ARM_FEATURE_DOTPROD)-- isum += vaddvq_s32(vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +- vaddvq_s32(vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];-#else-- int16x8_t p0 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[0]), vget_low_s8 (q8bytes.val[0])),- vmull_s8(vget_high_s8(q6bytes.val[0]), vget_high_s8(q8bytes.val[0])));- int16x8_t p1 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[1]), vget_low_s8 (q8bytes.val[1])),- vmull_s8(vget_high_s8(q6bytes.val[1]), vget_high_s8(q8bytes.val[1])));- isum += vaddvq_s16(p0) * scale[0] + vaddvq_s16(p1) * scale[1];-- int16x8_t p2 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[2]), vget_low_s8 (q8bytes.val[2])),- vmull_s8(vget_high_s8(q6bytes.val[2]), vget_high_s8(q8bytes.val[2])));- int16x8_t p3 = vaddq_s16(vmull_s8(vget_low_s8 (q6bytes.val[3]), vget_low_s8 (q8bytes.val[3])),- vmull_s8(vget_high_s8(q6bytes.val[3]), vget_high_s8(q8bytes.val[3])));- isum += vaddvq_s16(p2) * scale[2] + vaddvq_s16(p3) * scale[3];-#endif+ isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] ++ vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];sum += isum * d_all * y[i].d;@@ -4601,3 +5534,670 @@ void ggml_vec_dot_iq2_xxs_q8_K(const int n, float * restrict s, const void * res#include "ggml_vec_dot_iq2_xxs_q8_K.inc"#undef static#endif++#ifdef __x86_64__+#define ggml_vec_dot_iq2_xs_q8_K ggml_vec_dot_iq2_xs_q8_K_default+#include "ggml_vec_dot_iq2_xs_q8_K.inc"+#undef ggml_vec_dot_iq2_xs_q8_K+#define ggml_vec_dot_iq2_xs_q8_K __attribute__((__target__("avx2,fma,f16c"))) ggml_vec_dot_iq2_xs_q8_K_avx2+#ifndef __AVX2__+#define __UNDEF_AVX2__+#define __AVX2__+#endif+#include "ggml_vec_dot_iq2_xs_q8_K.inc"+#ifdef __UNDEF_AVX2__+#undef __UNDEF_AVX2__+#undef __AVX2__+#endif+#undef ggml_vec_dot_iq2_xs_q8_K+void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && X86_HAVE(F16C)) {+ ggml_vec_dot_iq2_xs_q8_K_avx2(n, s, vx, vy);+ } else {+ ggml_vec_dot_iq2_xs_q8_K_default(n, s, vx, vy);+ }+}+#else+#define static+#include "ggml_vec_dot_iq2_xs_q8_K.inc"+#undef static+#endif++// ================================ IQ2 quantization =============================================++typedef struct {+ uint64_t * grid;+ int * map;+ uint16_t * neighbours;+} iq2_entry_t;++static iq2_entry_t iq2_data[2] = {+ {NULL, NULL, NULL},+ {NULL, NULL, NULL},+};++static inline int iq2_data_index(int grid_size) {+ GGML_ASSERT(grid_size == 256 || grid_size == 512);+ return grid_size == 256 ? 0 : 1;+}++static int iq2_compare_func(const void * left, const void * right) {+ const int * l = (const int *)left;+ const int * r = (const int *)right;+ return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0;+}++void iq2xs_init_impl(int grid_size) {+ const int gindex = iq2_data_index(grid_size);+ if (iq2_data[gindex].grid) {+ return;+ }+ static const uint16_t kgrid_256[256] = {+ 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97,+ 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642,+ 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288,+ 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113,+ 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240,+ 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400,+ 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260,+ 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872,+ 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516,+ 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561,+ 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488,+ 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545,+ 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874,+ 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856,+ 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142,+ 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268,+ };+ static const uint16_t kgrid_512[512] = {+ 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70,+ 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257,+ 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340,+ 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597,+ 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096,+ 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348,+ 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065,+ 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441,+ 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160,+ 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372,+ 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125,+ 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652,+ 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197,+ 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549,+ 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894,+ 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388,+ 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480,+ 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773,+ 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473,+ 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436,+ 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497,+ 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162,+ 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528,+ 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745,+ 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234,+ 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025,+ 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810,+ 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984,+ 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462,+ 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960,+ 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048,+ 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690,+ };+ const int kmap_size = 43692;+ const int nwant = 2;+ const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512;+ uint64_t * kgrid_q2xs;+ int * kmap_q2xs;+ uint16_t * kneighbors_q2xs;++ printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size);+ uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t));+ for (int k = 0; k < grid_size; ++k) {+ int8_t * pos = (int8_t *)(the_grid + k);+ for (int i = 0; i < 8; ++i) {+ int l = (kgrid[k] >> 2*i) & 0x3;+ pos[i] = 2*l + 1;+ }+ }+ kgrid_q2xs = the_grid;+ iq2_data[gindex].grid = the_grid;+ kmap_q2xs = (int *)malloc(kmap_size*sizeof(int));+ iq2_data[gindex].map = kmap_q2xs;+ for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1;+ uint64_t aux64;+ uint8_t * aux8 = (uint8_t *)&aux64;+ for (int i = 0; i < grid_size; ++i) {+ aux64 = kgrid_q2xs[i];+ uint16_t index = 0;+ for (int k=0; k<8; ++k) {+ uint16_t q = (aux8[k] - 1)/2;+ index |= (q << 2*k);+ }+ kmap_q2xs[index] = i;+ }+ int8_t pos[8];+ int * dist2 = (int *)malloc(2*grid_size*sizeof(int));+ int num_neighbors = 0, num_not_in_map = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) continue;+ ++num_not_in_map;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ int d2 = 0;+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ int n = 0; int d2 = dist2[0];+ int nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ ++n;+ }+ num_neighbors += n;+ }+ printf("%s: %d neighbours in total\n", __func__, num_neighbors);+ kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t));+ iq2_data[gindex].neighbours = kneighbors_q2xs;+ int counter = 0;+ for (int i = 0; i < kmap_size; ++i) {+ if (kmap_q2xs[i] >= 0) continue;+ for (int k = 0; k < 8; ++k) {+ int l = (i >> 2*k) & 0x3;+ pos[k] = 2*l + 1;+ }+ for (int j = 0; j < grid_size; ++j) {+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + j);+ int d2 = 0;+ for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]);+ dist2[2*j+0] = d2;+ dist2[2*j+1] = j;+ }+ qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func);+ kmap_q2xs[i] = -(counter + 1);+ int d2 = dist2[0];+ uint16_t * start = &kneighbors_q2xs[counter++];+ int n = 0, nhave = 1;+ for (int j = 0; j < grid_size; ++j) {+ if (dist2[2*j] > d2) {+ if (nhave == nwant) break;+ d2 = dist2[2*j];+ ++nhave;+ }+ kneighbors_q2xs[counter++] = dist2[2*j+1];+ ++n;+ }+ *start = n;+ }+ free(dist2);+}++void iq2xs_free_impl(int grid_size) {+ GGML_ASSERT(grid_size == 256 || grid_size == 512 || grid_size == 1024);+ const int gindex = iq2_data_index(grid_size);+ if (iq2_data[gindex].grid) {+ free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL;+ free(iq2_data[gindex].map); iq2_data[gindex].map = NULL;+ free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL;+ }+}++static int iq2_find_best_neighbour(const uint16_t * restrict neighbours, const uint64_t * restrict grid,+ const float * restrict xval, const float * restrict weight, float scale, int8_t * restrict L) {+ int num_neighbors = neighbours[0];+ GGML_ASSERT(num_neighbors > 0);+ float best_d2 = FLT_MAX;+ int grid_index = -1;+ for (int j = 1; j <= num_neighbors; ++j) {+ const int8_t * pg = (const int8_t *)(grid + neighbours[j]);+ float d2 = 0;+ for (int i = 0; i < 8; ++i) {+ float q = pg[i];+ float diff = scale*q - xval[i];+ d2 += weight[i]*diff*diff;+ }+ if (d2 < best_d2) {+ best_d2 = d2; grid_index = neighbours[j];+ }+ }+ GGML_ASSERT(grid_index >= 0);+ const int8_t * pg = (const int8_t *)(grid + grid_index);+ for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2;+ return grid_index;+}++static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {++ const int gindex = iq2_data_index(256);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int nbl = n/256;++ block_iq2_xxs * y = vy;++ float scales[QK_K/32];+ float weight[32];+ float xval[32];+ int8_t L[32];+ int8_t Laux[32];+ float waux[32];+ bool is_on_grid[4];+ bool is_on_grid_aux[4];+ uint8_t block_signs[4];+ uint32_t q2[2*(QK_K/32)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/32; ++ib) {+ const float * xb = xbl + 32*ib;+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 4; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);+ if (!max) {+ scales[ib] = 0;+ memset(L, 0, 32);+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 4; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 32; ++i) L[i] = Laux[i];+ for (int k = 0; k < 4; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 4; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 4; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);+ for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 32; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale)+ // and correspondingly flip quant signs.+ scale = -scale;+ for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 4; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ASSERT(false);+ }+ q2[2*ib+0] |= (grid_index << 8*k);+ q2[2*ib+1] |= (block_signs[k] << 7*k);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ float sumqx = 0, sumq2 = 0;+ for (int ib = 0; ib < QK_K/32; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ q2[2*ib+1] |= ((uint32_t)l << 28);+ const float * xb = xbl + 32*ib;+ const float * qw = quant_weights + QK_K*ibl + 32*ib;+ for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ const uint8_t * aux8 = (const uint8_t *)(q2 + 2*ib);+ const float db = d * (1 + 2*l);+ uint32_t u = 0;+ for (int k = 0; k < 4; ++k) {+ const int8_t * signs = keven_signs_q2xs + 8*((q2[2*ib+1] >> 7*k) & 127);+ const float * xk = xb + 8*k;+ const float * wk = weight + 8*k;+ const uint8_t * grid = (const uint8_t *)(kgrid_q2xs + aux8[k]);+ float best_mse = 0; int best_index = aux8[k];+ for (int j = 0; j < 8; ++j) {+ float diff = db * grid[j] * signs[j] - xk[j];+ best_mse += wk[j] * diff * diff;+ }+ for (int idx = 0; idx < 256; ++idx) {+ grid = (const uint8_t *)(kgrid_q2xs + idx);+ float mse = 0;+ for (int j = 0; j < 8; ++j) {+ float diff = db * grid[j] * signs[j] - xk[j];+ mse += wk[j] * diff * diff;+ }+ if (mse < best_mse) {+ best_mse = mse; best_index = idx;+ }+ }+ u |= (best_index << 8*k);+ grid = (const uint8_t *)(kgrid_q2xs + best_index);+ //grid = (const uint8_t *)(kgrid_q2xs + aux8[k]);+ for (int j = 0; j < 8; ++j) {+ float q = db * grid[j] * signs[j];+ sumqx += wk[j] * q * xk[j];+ sumq2 += wk[j] * q * q;+ }+ }+ q2[2*ib] = u;+ if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(d*sumqx/sumq2);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);+ }+}++static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict vy, int n, const float * restrict quant_weights) {++ const int gindex = iq2_data_index(512);++ const uint64_t * kgrid_q2xs = iq2_data[gindex].grid;+ const int * kmap_q2xs = iq2_data[gindex].map;+ const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours;++ GGML_ASSERT(quant_weights && "missing quantization weights");+ GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?");+ GGML_ASSERT(n%QK_K == 0);++ const int kMaxQ = 3;++ const int nbl = n/256;++ block_iq2_xs * y = vy;++ float scales[QK_K/16];+ float weight[16];+ float xval[16];+ int8_t L[16];+ int8_t Laux[16];+ float waux[16];+ bool is_on_grid[2];+ bool is_on_grid_aux[2];+ uint8_t block_signs[2];+ uint16_t q2[2*(QK_K/16)];++ for (int ibl = 0; ibl < nbl; ++ibl) {++ y[ibl].d = GGML_FP32_TO_FP16(0.f);+ memset(q2, 0, QK_K/4);+ memset(y[ibl].scales, 0, QK_K/32);++ float max_scale = 0;++ const float * xbl = x + QK_K*ibl;+ float sumx2 = 0;+ for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i];+ float sigma2 = sumx2/QK_K;++ for (int ib = 0; ib < QK_K/16; ++ib) {+ const float * xb = xbl + 16*ib;+ const float * qw = quant_weights + QK_K*ibl + 16*ib;+ for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]);+ for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]);+ for (int k = 0; k < 2; ++k) {+ int nflip = 0;+ uint8_t s = 0;+ for (int i = 0; i < 8; ++i) {+ if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i];+ else {+ xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i);+ }+ }+ if (nflip%2) {+ int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin];+ for (int i = 1; i < 8; ++i) {+ float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i];+ if (ax < min) {+ min = ax; imin = i;+ }+ }+ xval[8*k+imin] = -xval[8*k+imin];+ s ^= (1 << imin);+ }+ block_signs[k] = s & 127;+ }+ float max = xval[0];+ for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);+ if (!max) {+ scales[ib] = 0;+ memset(L, 0, 16);+ continue;+ }+ float best = 0;+ float scale = max/(2*kMaxQ-1);+ is_on_grid[0] = is_on_grid[1] = true;+ for (int is = -9; is <= 9; ++is) {+ float id = (2*kMaxQ-1+is*0.1f)/max;+ float this_scale = 1/id;+ for (int k = 0; k < 2; ++k) {+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l));+ }+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ is_on_grid_aux[k] = true;+ if (grid_index < 0) {+ is_on_grid_aux[k] = false;+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*Laux[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0 && sumqx*sumqx > best*sumq2) {+ scale = sumqx/sumq2; best = scale*sumqx;+ for (int i = 0; i < 16; ++i) L[i] = Laux[i];+ for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k];+ }+ }+ int n_not_ongrid = 0;+ for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid;+ if (n_not_ongrid > 0 && scale > 0) {+ float id = 1/scale;+ for (int k = 0; k < 2; ++k) {+ if (is_on_grid[k]) continue;+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) {+ int l = nearest_int(0.5f*(id*xval[8*k+i]-1));+ l = MAX(0, MIN(kMaxQ-1, l));+ u |= (l << 2*i);+ L[8*k + i] = l;+ }+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;+ grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);+ }+ }+ float sumqx = 0, sumq2 = 0;+ for (int i = 0; i < 16; ++i) {+ float w = weight[i];+ float q = 2*L[i] + 1;+ sumqx += w*xval[i]*q;+ sumq2 += w*q*q;+ }+ if (sumq2 > 0) scale = sumqx/sumq2;+ }+ if (scale < 0) {+ scale = -scale;+ for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127;+ }+ for (int k = 0; k < 2; ++k) {+ uint16_t u = 0;+ for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i);+ int grid_index = kmap_q2xs[u];+ if (grid_index < 0) {+ printf("Oops: found point %u not on grid:", u);+ for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]);+ printf("\n");+ GGML_ASSERT(false);+ }+ q2[2*ib+k] = grid_index | (block_signs[k] << 9);+ }+ GGML_ASSERT(scale >= 0);+ scales[ib] = scale;+ max_scale = MAX(max_scale, scale);+ }++ if (!max_scale) {+ memset(y[ibl].qs, 0, QK_K/4);+ continue;+ }++ float d = max_scale/31;+ y[ibl].d = GGML_FP32_TO_FP16(d);+ float id = 1/d;+ for (int ib = 0; ib < QK_K/16; ++ib) {+ int l = nearest_int(0.5f*(id*scales[ib]-1));+ l = MAX(0, MIN(15, l));+ if (ib%2 == 0) y[ibl].scales[ib/2] = l;+ else y[ibl].scales[ib/2] |= (l << 4);+ }+ memcpy(y[ibl].qs, q2, QK_K/4);++ }+}++size_t quantize_iq2_xxs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ GGML_ASSERT(n_per_row%QK_K == 0);+ int nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xxs);+ }+ return nrow * nblock * sizeof(block_iq2_xxs);+}++size_t quantize_iq2_xs(const float * src, void * dst, int nrow, int n_per_row, int64_t * hist, const float * quant_weights) {+ (void)hist;+ GGML_ASSERT(n_per_row%QK_K == 0);+ int nblock = n_per_row/QK_K;+ char * qrow = (char *)dst;+ for (int row = 0; row < nrow; ++row) {+ quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights);+ src += n_per_row;+ qrow += nblock*sizeof(block_iq2_xs);+ }+ return nrow * nblock * sizeof(block_iq2_xs);+}+diff --git a/llama.cpp/ggml-quants.h b/llama.cpp/ggml-quants.hindex 8dd911d4..7d7cf917 100644--- a/llama.cpp/ggml-quants.h+++ b/llama.cpp/ggml-quants.h@@ -174,6 +174,14 @@ typedef struct {} block_iq2_xxs;static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");+// 2.3125 bpw quants+typedef struct {+ ggml_fp16_t d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");+// Quantizationvoid quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int k);void quantize_row_q4_1_reference(const float * restrict x, block_q4_1 * restrict y, int k);@@ -188,7 +196,6 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrictvoid quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);-void quantize_row_iq2_xxs_reference(const float * restrict x, block_iq2_xxs * restrict y, int k);void quantize_row_q4_0(const float * restrict x, void * restrict y, int k);void quantize_row_q4_1(const float * restrict x, void * restrict y, int k);@@ -203,7 +210,6 @@ void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);-void quantize_row_iq2_xxs(const float * restrict x, void * restrict y, int k);// Dequantizationvoid dequantize_row_q4_0(const block_q4_0 * restrict x, float * restrict y, int k);@@ -220,6 +226,7 @@ void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, intvoid dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);void dequantize_row_iq2_xxs(const block_iq2_xxs * restrict x, float * restrict y, int k);+void dequantize_row_iq2_xs (const block_iq2_xs * restrict x, float * restrict y, int k);// Dot productvoid ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, const void * restrict vx, const void * restrict vy);@@ -234,3 +241,22 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx,void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);+void ggml_vec_dot_iq2_xs_q8_K (int n, float * restrict s, const void * restrict vx, const void * restrict vy);++//+// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")+//+size_t quantize_iq2_xxs(const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_iq2_xs (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q2_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q3_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q4_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q5_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q6_K (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q4_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q4_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q5_0 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);+size_t quantize_q5_1 (const float * src, void * dst, int nrows, int n_per_row, int64_t * hist, const float * imatrix);++void iq2xs_init_impl(int grid_size);+void iq2xs_free_impl(int grid_size);diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 0e70ab62..f36b2539 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -96,7 +96,7 @@ void ggml_print_backtrace(void) {"-ex", "bt -frame-info source-and-location","-ex", "detach","-ex", "quit",- NULL);+ (char *) NULL);} else {waitpid(pid, NULL, 0);}@@ -168,7 +168,7 @@ inline static void * ggml_aligned_malloc(size_t size) {}void * aligned_memory = NULL;int result;- if (ggml_metal_supported()) {+ if (llamafile_has_metal()) {result = posix_memalign(&aligned_memory, 16384, size);} else {result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);@@ -519,11 +519,22 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.type_size = sizeof(block_iq2_xxs),.is_quantized = true,.to_float = (ggml_to_float_t) dequantize_row_iq2_xxs,- .from_float = quantize_row_iq2_xxs,- .from_float_reference = (ggml_from_float_t) quantize_row_iq2_xxs_reference,+ .from_float = NULL,+ .from_float_reference = NULL,.vec_dot = ggml_vec_dot_iq2_xxs_q8_K,.vec_dot_type = GGML_TYPE_Q8_K,},+ [GGML_TYPE_IQ2_XS] = {+ .type_name = "iq2_xs",+ .blck_size = QK_K,+ .type_size = sizeof(block_iq2_xs),+ .is_quantized = true,+ .to_float = (ggml_to_float_t) dequantize_row_iq2_xs,+ .from_float = NULL,+ .from_float_reference = NULL,+ .vec_dot = ggml_vec_dot_iq2_xs_q8_K,+ .vec_dot_type = GGML_TYPE_Q8_K,+ },[GGML_TYPE_Q8_K] = {.type_name = "q8_K",.blck_size = QK_K,@@ -1443,6 +1454,9 @@ inline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) {inline static void ggml_vec_elu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expf(x[i])-1; }inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }+// TODO: optimize performance+inline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }+inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }static const float GELU_COEF_A = 0.044715f;static const float GELU_QUICK_COEF = -1.702f;@@ -1801,9 +1815,11 @@ static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {"GELU","GELU_QUICK","SILU",+ "HARDSWISH",+ "HARDSIGMOID",};-static_assert(GGML_UNARY_OP_COUNT == 10, "GGML_UNARY_OP_COUNT != 10");+static_assert(GGML_UNARY_OP_COUNT == 12, "GGML_UNARY_OP_COUNT != 12");static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");@@ -2159,6 +2175,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break;case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break;+ case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break;case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break;case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break;}@@ -2308,7 +2325,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {GGML_PRINT_DEBUG("%s: g_state initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);}- ggml_init_cublas(FLAG_log_disable);+ ggml_init_cublas();#if defined(GGML_USE_CLBLAST)ggml_cl_init();#endif@@ -2371,6 +2388,10 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {}void ggml_free(struct ggml_context * ctx) {+ if (ctx == NULL) {+ return;+ }+// make this function thread safeggml_critical_section_start();@@ -3964,6 +3985,20 @@ struct ggml_tensor * ggml_silu_back(return result;}+// ggml hardswish+struct ggml_tensor * ggml_hardswish(+ struct ggml_context * ctx,+ struct ggml_tensor * a) {+ return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSWISH);+}++// ggml hardsigmoid+struct ggml_tensor * ggml_hardsigmoid(+ struct ggml_context * ctx,+ struct ggml_tensor * a) {+ return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSIGMOID);+}+// ggml_normstatic struct ggml_tensor * ggml_norm_impl(@@ -4346,13 +4381,13 @@ struct ggml_tensor * ggml_set_2d_inplace(static struct ggml_tensor * ggml_cpy_impl(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b,- bool inplace) {+ struct ggml_tensor * b) {GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b));bool is_node = false;- if (!inplace && (a->grad || b->grad)) {+ if (a->grad || b->grad) {+ // inplace is false and either one have a gradis_node = true;}@@ -4376,29 +4411,38 @@ struct ggml_tensor * ggml_cpy(struct ggml_context * ctx,struct ggml_tensor * a,struct ggml_tensor * b) {- return ggml_cpy_impl(ctx, a, b, false);+ return ggml_cpy_impl(ctx, a, b);}-struct ggml_tensor * ggml_cpy_inplace(+struct ggml_tensor * ggml_cast(struct ggml_context * ctx,- struct ggml_tensor * a,- struct ggml_tensor * b) {- return ggml_cpy_impl(ctx, a, b, true);+ struct ggml_tensor * a,+ enum ggml_type type) {+ bool is_node = false;++ struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS, a->ne);+ ggml_format_name(result, "%s (copy)", a->name);++ result->op = GGML_OP_CPY;+ result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;+ result->src[0] = a;+ result->src[1] = result;++ return result;}// ggml_contstatic struct ggml_tensor * ggml_cont_impl(struct ggml_context * ctx,- struct ggml_tensor * a,- bool inplace) {+ struct ggml_tensor * a) {bool is_node = false;- if (!inplace && a->grad) {+ if (a->grad) {is_node = true;}- struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);+ struct ggml_tensor * result = ggml_dup_tensor(ctx, a);ggml_format_name(result, "%s (cont)", a->name);result->op = GGML_OP_CONT;@@ -4411,13 +4455,7 @@ static struct ggml_tensor * ggml_cont_impl(struct ggml_tensor * ggml_cont(struct ggml_context * ctx,struct ggml_tensor * a) {- return ggml_cont_impl(ctx, a, false);-}--struct ggml_tensor * ggml_cont_inplace(- struct ggml_context * ctx,- struct ggml_tensor * a) {- return ggml_cont_impl(ctx, a, true);+ return ggml_cont_impl(ctx, a);}// make contiguous, with new shape@@ -5360,6 +5398,31 @@ GGML_API struct ggml_tensor * ggml_conv_transpose_1d(return result;}+// ggml_conv_depthwise+struct ggml_tensor * ggml_conv_depthwise_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int s0,+ int s1,+ int p0,+ int p1,+ int d0,+ int d1) {+ struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]);+ struct ggml_tensor * im2col = ggml_im2col(ctx, new_a,+ ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]),+ s0, s1, p0, p1, d0, d1, true); // [N * IC, OH, OW, KH * KW]++ struct ggml_tensor * result =+ ggml_mul_mat(ctx,+ ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1), // [OC,1, KH, KW] => [1, OC, 1, KH * KW]+ ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3])); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]++ result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], b->ne[2], b->ne[3]); // [N, OC, OH, OW]++ return result;+}// ggml_conv_2d// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]@@ -7179,6 +7242,17 @@ static void ggml_compute_forward_add_f32(const int ith = params->ith;const int nth = params->nth;+#ifdef GGML_USE_CLBLAST+ if (src1->backend == GGML_BACKEND_GPU) {+ // TODO: OpenCL kernel support full broadcast+ GGML_ASSERT(ggml_can_repeat_rows(src1, src0));+ if (ith == 0) {+ ggml_cl_add(src0, src1, dst);+ }+ return;+ }+#endif+const int nr = ggml_nrows(src0);GGML_TENSOR_BINARY_OP_LOCALS@@ -7484,6 +7558,7 @@ static void ggml_compute_forward_add(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:{ggml_compute_forward_add_q_f32(params, src0, src1, dst);} break;@@ -7749,6 +7824,7 @@ static void ggml_compute_forward_add1(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:{ggml_compute_forward_add1_q_f32(params, src0, src1, dst);} break;@@ -7778,6 +7854,9 @@ static void ggml_compute_forward_acc_f32(bool inplace = (bool) ((int32_t *) dst->op_params)[4];if (!inplace && (params->type == GGML_TASK_INIT)) {+ if (params->ith != 0) {+ return;+ }// memcpy needs to be synchronized across threads to avoid race conditions.// => do it in INIT phasememcpy(@@ -7864,6 +7943,7 @@ static void ggml_compute_forward_acc(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:default:{GGML_ASSERT(false);@@ -9346,6 +9426,87 @@ static void ggml_compute_forward_silu_back(}}++static void ggml_compute_forward_hardswish_f32(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ assert(params->ith == 0);+ assert(ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ return;+ }++ const int n = ggml_nrows(src0);+ const int nc = src0->ne[0];++ assert(dst->nb[0] == sizeof(float));+ assert(src0->nb[0] == sizeof(float));++ for (int i = 0; i < n; i++) {+ ggml_vec_hardswish_f32(nc,+ (float *) ((char *) dst->data + i*( dst->nb[1])),+ (float *) ((char *) src0->data + i*(src0->nb[1])));+ }+}+static void ggml_compute_forward_hardswish(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_hardswish_f32(params, src0, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++static void ggml_compute_forward_hardsigmoid_f32(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ assert(params->ith == 0);+ assert(ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ return;+ }++ const int n = ggml_nrows(src0);+ const int nc = src0->ne[0];++ assert(dst->nb[0] == sizeof(float));+ assert(src0->nb[0] == sizeof(float));++ for (int i = 0; i < n; i++) {+ ggml_vec_hardsigmoid_f32(nc,+ (float *) ((char *) dst->data + i*( dst->nb[1])),+ (float *) ((char *) src0->data + i*(src0->nb[1])));+ }+}++static void ggml_compute_forward_hardsigmoid(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ switch (src0->type) {+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_hardsigmoid_f32(params, src0, dst);+ } break;+ default:+ {+ GGML_ASSERT(false);+ } break;+ }+}++// ggml_compute_forward_normstatic void ggml_compute_forward_norm_f32(@@ -9838,11 +9999,30 @@ static void ggml_compute_forward_mul_mat(#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)if (ggml_compute_forward_mul_mat_use_blas(dst)) {- if (params->ith != 0) {- return;- }+ const int64_t ne_plane = ne01*ne00;+ const int64_t desired_wsize = ne13*ne12*ne_plane*sizeof(float);+ UNUSED(desired_wsize);if (params->type == GGML_TASK_INIT) {+ if (type != GGML_TYPE_F32) {+ assert(params->wsize >= desired_wsize);+ // parallelize by src0 rows+ for (int64_t i13 = 0; i13 < ne13; i13++) {+ for (int64_t i12 = 0; i12 < ne12; i12++) {+ // broadcast src0 into src1 across 2nd,3rd dimension+ const int64_t i03 = i13/r3;+ const int64_t i02 = i12/r2;++ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;+ float * const wdata = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;+ ggml_to_float_t const to_float = type_traits[type].to_float;++ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {+ to_float((const char *) x + i01*nb01, wdata + i01*ne00, ne00);+ }+ }+ }+ }return;}@@ -9850,9 +10030,14 @@ static void ggml_compute_forward_mul_mat(return;}+ // perform sgemm, parallelization controlled by blas lib+ if (ith != 0) {+ return;+ }++ //const int64_t tgemm0 = ggml_perf_time_us();for (int64_t i13 = 0; i13 < ne13; i13++) {for (int64_t i12 = 0; i12 < ne12; i12++) {- // broadcast src0 into src1 across 2nd,3rd dimensionconst int64_t i03 = i13/r3;const int64_t i02 = i12/r2;@@ -9861,17 +10046,7 @@ static void ggml_compute_forward_mul_mat(float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);if (type != GGML_TYPE_F32) {- float * const wdata = params->wdata;- ggml_to_float_t const to_float = type_traits[type].to_float;-- size_t id = 0;- for (int64_t i01 = 0; i01 < ne01; ++i01) {- to_float((const char *) x + i01*nb01, wdata + id, ne00);- id += ne00;- }-- assert(id*sizeof(float) <= params->wsize);- x = wdata;+ x = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;}cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,@@ -9881,6 +10056,7 @@ static void ggml_compute_forward_mul_mat(0.0f, d, ne01);}}+ //printf("cblas_sgemm = %.3f ms, %lld flops\n", (ggml_perf_time_us() - tgemm0)/1000.0, ne13*ne12*ne1*ne01*ne10*2);//printf("CBLAS = %f ms, %d x %d x %d x %d\n", (ggml_perf_time_us() - t0)/1000.0, ne0, ne1, ne2, ne3);@@ -9889,6 +10065,9 @@ static void ggml_compute_forward_mul_mat(#endifif (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }if (src1->type != vec_dot_type) {char * wdata = params->wdata;const size_t row_size = ggml_row_size(vec_dot_type, ne10);@@ -10053,6 +10232,9 @@ static void ggml_compute_forward_mul_mat_id(#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ne11 + (i1)]if (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }char * wdata = params->wdata;if (src1->type != vec_dot_type) {const size_t row_size = ggml_row_size(vec_dot_type, ne10);@@ -10238,6 +10420,9 @@ static void ggml_compute_forward_out_prod_f32(return;}#endif+ if (ith != 0) {+ return;+ }ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);return;}@@ -10421,6 +10606,9 @@ static void ggml_compute_forward_out_prod_q_f32(// TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)if (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);return;}@@ -10506,6 +10694,7 @@ static void ggml_compute_forward_out_prod(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:{ggml_compute_forward_out_prod_q_f32(params, src0, src1, dst);} break;@@ -10604,6 +10793,9 @@ static void ggml_compute_forward_set_f32(bool inplace = (bool) ((int32_t *) dst->op_params)[4];if (!inplace && (params->type == GGML_TASK_INIT)) {+ if (params->ith != 0) {+ return;+ }// memcpy needs to be synchronized across threads to avoid race conditions.// => do it in INIT phasememcpy(@@ -10681,6 +10873,7 @@ static void ggml_compute_forward_set(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:default:{GGML_ASSERT(false);@@ -10876,6 +11069,7 @@ static void ggml_compute_forward_get_rows(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:{ggml_compute_forward_get_rows_q(params, src0, src1, dst);} break;@@ -10926,6 +11120,9 @@ static void ggml_compute_forward_get_rows_back_f32_f16(// ggml_compute_forward_dup_same_cont(params, opt0, dst);if (params->type == GGML_TASK_INIT) {+ if (params->ith != 0) {+ return;+ }memset(dst->data, 0, ggml_nbytes(dst));}@@ -10960,6 +11157,9 @@ static void ggml_compute_forward_get_rows_back_f32(// ggml_compute_forward_dup_same_cont(params, opt0, dst);if (params->type == GGML_TASK_INIT) {+ if (params->ith != 0) {+ return;+ }memset(dst->data, 0, ggml_nbytes(dst));}@@ -11097,6 +11297,9 @@ static void ggml_compute_forward_diag_mask_f32(GGML_ASSERT(n_past >= 0);if (!inplace && (params->type == GGML_TASK_INIT)) {+ if (ith != 0) {+ return;+ }// memcpy needs to be synchronized across threads to avoid race conditions.// => do it in INIT phaseGGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));@@ -11513,6 +11716,7 @@ static void ggml_compute_forward_alibi(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:case GGML_TYPE_Q8_K:case GGML_TYPE_I8:case GGML_TYPE_I16:@@ -11588,6 +11792,7 @@ static void ggml_compute_forward_clamp(case GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:case GGML_TYPE_Q8_K:case GGML_TYPE_I8:case GGML_TYPE_I16:@@ -11632,6 +11837,21 @@ static float ggml_rope_yarn_corr_dim(int n_dims, int n_orig_ctx, float n_rot, flreturn n_dims * logf(n_orig_ctx / (n_rot * 2 * (float)M_PI)) / (2 * logf(base));}+static void ggml_rope_cache_init(+ float theta_base, float freq_scale, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,+ float * cache, float sin_sign, float theta_scale+) {+ float theta = theta_base;+ for (int64_t i0 = 0; i0 < ne0; i0 += 2) {+ rope_yarn(+ theta, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]+ );+ cache[i0 + 1] *= sin_sign;++ theta *= theta_scale;+ }+}+GGML_CALL void ggml_rope_yarn_corr_dims(int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]) {@@ -11714,6 +11934,12 @@ static void ggml_compute_forward_rope_f32(for (int64_t i3 = 0; i3 < ne3; i3++) {for (int64_t i2 = 0; i2 < ne2; i2++) {const int64_t p = pos[i2];++ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;+ if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox+ ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);+ }+for (int64_t i1 = 0; i1 < ne1; i1++) {if (ir++ < ir0) continue;if (ir > ir1) break;@@ -11747,18 +11973,13 @@ static void ggml_compute_forward_rope_f32(}} else if (!is_neox) {for (int64_t i0 = 0; i0 < ne0; i0 += 2) {- float cos_theta, sin_theta;- rope_yarn(- theta_base, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta- );- sin_theta *= sin_sign;+ const float cos_theta = cache[i0 + 0];+ const float sin_theta = cache[i0 + 1];// zeta scaling for xPos only:float zeta = xpos_base != 0.0f ? powf((i0 + 0.4f * ne0) / (1.4f * ne0), p / xpos_base) : 1.0f;if (xpos_down) zeta = 1.0f / zeta;- theta_base *= theta_scale;-const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);@@ -11882,6 +12103,12 @@ static void ggml_compute_forward_rope_f16(for (int64_t i3 = 0; i3 < ne3; i3++) {for (int64_t i2 = 0; i2 < ne2; i2++) {const int64_t p = pos[i2];++ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;+ if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox+ ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);+ }+for (int64_t i1 = 0; i1 < ne1; i1++) {if (ir++ < ir0) continue;if (ir > ir1) break;@@ -11915,13 +12142,8 @@ static void ggml_compute_forward_rope_f16(}} else if (!is_neox) {for (int64_t i0 = 0; i0 < ne0; i0 += 2) {- float cos_theta, sin_theta;- rope_yarn(- theta_base, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta- );- sin_theta *= sin_sign;-- theta_base *= theta_scale;+ const float cos_theta = cache[i0 + 0];+ const float sin_theta = cache[i0 + 1];const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);@@ -12048,6 +12270,9 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(GGML_ASSERT(nb10 == sizeof(float));if (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }memset(params->wdata, 0, params->wsize);// permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)@@ -12142,6 +12367,9 @@ static void ggml_compute_forward_conv_transpose_1d_f32(GGML_ASSERT(nb10 == sizeof(float));if (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }memset(params->wdata, 0, params->wsize);// prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)@@ -12340,6 +12568,7 @@ static void ggml_compute_forward_im2col(}}+// ggml_compute_forward_conv_transpose_2dstatic void ggml_compute_forward_conv_transpose_2d(@@ -12365,6 +12594,9 @@ static void ggml_compute_forward_conv_transpose_2d(GGML_ASSERT(nb10 == sizeof(float));if (params->type == GGML_TASK_INIT) {+ if (ith != 0) {+ return;+ }memset(params->wdata, 0, params->wsize);// permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)@@ -13908,6 +14140,14 @@ static void ggml_compute_forward_unary({ggml_compute_forward_silu(params, src0, dst);} break;+ case GGML_UNARY_OP_HARDSWISH:+ {+ ggml_compute_forward_hardswish(params, src0, dst);+ } break;+ case GGML_UNARY_OP_HARDSIGMOID:+ {+ ggml_compute_forward_hardsigmoid(params, src0, dst);+ } break;default:{GGML_ASSERT(false);@@ -13971,6 +14211,9 @@ static void ggml_compute_forward_add_rel_pos_f32(const bool inplace = (bool) ((int32_t *) dst->op_params)[0];if (!inplace && params->type == GGML_TASK_INIT) {+ if (params->ith != 0) {+ return;+ }memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));return;}@@ -14886,7 +15129,7 @@ size_t ggml_hash_find_or_insert(struct ggml_hash_set hash_set, struct ggml_tensoreturn i;}-static struct ggml_hash_set ggml_hash_set_new(size_t size) {+struct ggml_hash_set ggml_hash_set_new(size_t size) {size = ggml_hash_size(size);struct ggml_hash_set result;result.size = size;@@ -16257,8 +16500,9 @@ struct ggml_compute_state_shared {const int n_threads;// synchronization primitives- atomic_int n_active; // num active threads- atomic_int node_n; // active graph node+ atomic_int n_active; // num active threads+ atomic_int node_n; // active graph node+ atomic_int node_task; // active graph node task phasebool (*abort_callback)(void * data); // abort ggml_graph_compute when truevoid * abort_callback_data;@@ -16314,6 +16558,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {case GGML_UNARY_OP_TANH:case GGML_UNARY_OP_ELU:case GGML_UNARY_OP_RELU:+ case GGML_UNARY_OP_HARDSWISH: // to opt for multiple threads+ case GGML_UNARY_OP_HARDSIGMOID: // to opt for multiple threads{n_tasks = 1;} break;@@ -16349,25 +16595,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {//n_tasks = MIN(n_threads, MAX(1, nr0/128));//printf("nr0 = %8d, nr1 = %8d, nr0*nr1 = %8d, n_tasks%d\n", nr0, nr1, nr0*nr1, n_tasks);-- if (ggml_cublas_loaded()) {- if (ggml_cuda_can_mul_mat(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinning- }- }-#if defined(GGML_USE_CLBLAST)- if (ggml_cl_can_mul_mat(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinning- }-#endif-#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)- if (ggml_compute_forward_mul_mat_use_blas(node->src[0], node->src[1], node)) {- n_tasks = 1; // TODO: this actually is doing nothing- // the threads are still spinning- }-#endif} break;case GGML_OP_MUL_MAT_ID:{@@ -16409,7 +16636,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {} break;case GGML_OP_SOFT_MAX:{- n_tasks = MIN(MIN(4, n_threads), ggml_nrows(node->src[0]));+ n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));} break;case GGML_OP_CONV_TRANSPOSE_1D:{@@ -16523,6 +16750,34 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {return n_tasks;}+static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_compute_state * state, const bool do_yield) {+ // wait for other threads to finish+ const int last_node_n = * node_n;++ while (true) {+ if (do_yield) {+ sched_yield();+ }++ * node_n = atomic_load(&state->shared->node_n);+ if (* node_n != last_node_n) break;+ }+}++static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_compute_state * state, const bool do_yield) {+ // wait for other threads to finish+ const int last_task_phase = * task_phase;++ while (true) {+ if (do_yield) {+ sched_yield();+ }++ * task_phase = atomic_load(&state->shared->node_task);+ if (* task_phase != last_task_phase) break;+ }+}+static thread_ret_t ggml_graph_compute_thread(void * data) {struct ggml_compute_state * state = (struct ggml_compute_state *) data;@@ -16533,7 +16788,8 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {set_numa_thread_affinity(state->ith, n_threads);- int node_n = -1;+ int node_n = -1;+ int task_phase = GGML_TASK_FINALIZE;while (true) {if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {@@ -16565,7 +16821,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {// distribute new work or execute it direct if 1Twhile (++node_n < cgraph->n_nodes) {GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes);-struct ggml_tensor * node = cgraph->nodes[node_n];const int n_tasks = ggml_get_n_tasks(node, n_threads);@@ -16574,13 +16829,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {params.nth = n_tasks;- /* INIT */- if (GGML_OP_HAS_INIT[node->op]) {- params.type = GGML_TASK_INIT;- ggml_compute_forward(¶ms, node);- }-if (n_tasks == 1) {+ /* INIT */+ if (GGML_OP_HAS_INIT[node->op]) {+ params.type = GGML_TASK_INIT;+ ggml_compute_forward(¶ms, node);+ }+// TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1,// they do something more efficient than spinning (?)params.type = GGML_TASK_COMPUTE;@@ -16601,38 +16856,24 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {}}- atomic_store(&state->shared->n_active, n_threads);- atomic_store(&state->shared->node_n, node_n);+ task_phase = GGML_TASK_INIT;+ atomic_store(&state->shared->n_active, n_threads);+ atomic_store(&state->shared->node_n, node_n);+ atomic_store(&state->shared->node_task, task_phase);} else {- // wait for other threads to finish- const int last = node_n;-- const bool do_yield = last < 0 || cgraph->nodes[last]->op == GGML_OP_MUL_MAT;-- while (true) {- // TODO: this sched_yield can have significant impact on the performance - either positive or negative- // depending on the workload and the operating system.- // since it is not clear what is the best approach, it should potentially become user-configurable- // ref: https://github.com/ggerganov/ggml/issues/291- // UPD: adding the do_yield flag seems to resolve the issue universally- if (do_yield) {- sched_yield();- }-- node_n = atomic_load(&state->shared->node_n);- if (node_n != last) break;- };+ ggml_graph_compute_thread_sync_node(&node_n, state, false);+ ggml_graph_compute_thread_sync_task(&task_phase, state, false);}// check if we should stopif (node_n >= cgraph->n_nodes) break;- /* COMPUTE */+ /* INIT & COMPUTE */struct ggml_tensor * node = cgraph->nodes[node_n];const int n_tasks = ggml_get_n_tasks(node, n_threads);struct ggml_compute_params params = {- /*.type =*/ GGML_TASK_COMPUTE,+ /*.type =*/ GGML_TASK_INIT,/*.ith =*/ state->ith,/*.nth =*/ n_tasks,/*.wsize =*/ cplan->work_size,@@ -16640,14 +16881,45 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {};if (state->ith < n_tasks) {+ if (GGML_OP_HAS_INIT[node->op]) {+ ggml_compute_forward(¶ms, node);+ }+ }++ if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {+ task_phase = GGML_TASK_COMPUTE;+ atomic_store(&state->shared->n_active, n_threads);+ atomic_store(&state->shared->node_task, task_phase);+ }+ else {+ // TODO: this sched_yield can have significant impact on the performance - either positive or negative+ // depending on the workload and the operating system.+ // since it is not clear what is the best approach, it should potentially become user-configurable+ // ref: https://github.com/ggerganov/ggml/issues/291+ // UPD: adding the do_yield flag seems to resolve the issue universally+ const bool do_yield = node_n < 0 || cgraph->nodes[node_n]->op == GGML_OP_MUL_MAT;+ ggml_graph_compute_thread_sync_task(&task_phase, state, do_yield);+ }++ if (state->ith < n_tasks) {+ params.type = GGML_TASK_COMPUTE;ggml_compute_forward(¶ms, node);}++ if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {+ task_phase = GGML_TASK_FINALIZE;+ atomic_store(&state->shared->n_active, n_threads);+ atomic_store(&state->shared->node_task, task_phase);+ }+ else {+ ggml_graph_compute_thread_sync_task(&task_phase, state, false);+ }}return GGML_EXIT_SUCCESS;}-struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {+struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threads) {if (n_threads <= 0) {n_threads = GGML_DEFAULT_N_THREADS;}@@ -16698,8 +16970,11 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)if (ggml_compute_forward_mul_mat_use_blas(node)) {if (node->src[0]->type != GGML_TYPE_F32) {- // here we need memory just for single 2D matrix from src0- cur = ggml_type_size(GGML_TYPE_F32)*(node->src[0]->ne[0]*node->src[0]->ne[1]);+ // here we need memory for fully dequantized matrix from src0+ // take into account that src0 can be broadcasted into src1[2,3]+ cur = ggml_type_size(GGML_TYPE_F32)+ * node->src[0]->ne[0]*node->src[0]->ne[1]+ * node->src[1]->ne[2]*node->src[1]->ne[3];}} else#endif@@ -16709,14 +16984,15 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {} break;case GGML_OP_MUL_MAT_ID:{+ cur = 0;const struct ggml_tensor * src0 = node->src[2];const struct ggml_tensor * src1 = node->src[1];const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type;if (src1->type != vec_dot_type) {- cur = ggml_row_size(vec_dot_type, ggml_nelements(src1));+ cur += ggml_row_size(vec_dot_type, ggml_nelements(src1));}const int n_as = ggml_get_op_params_i32(node, 1);- cur = GGML_PAD(cur, sizeof(int64_t)); // align+ cur += GGML_PAD(cur, sizeof(int64_t)); // aligncur += n_as * sizeof(int64_t); // matrix_row_countscur += n_as * src1->ne[1] * sizeof(int64_t); // matrix_rows} break;@@ -16727,6 +17003,7 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {}} break;case GGML_OP_SOFT_MAX:+ case GGML_OP_ROPE:{cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;} break;@@ -16851,6 +17128,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {/*.n_threads =*/ n_threads,/*.n_active =*/ n_threads,/*.node_n =*/ -1,+ /*.node_task =*/ GGML_TASK_FINALIZE,/*.abort_callback =*/ NULL,/*.abort_callback_data =*/ NULL,};@@ -18525,6 +18803,28 @@ enum ggml_opt_result ggml_opt_resume_g(////////////////////////////////////////////////////////////////////////////////+void ggml_quantize_init(enum ggml_type type) {+ ggml_critical_section_start();++ switch (type) {+ case GGML_TYPE_IQ2_XXS: iq2xs_init_impl(256); break;+ case GGML_TYPE_IQ2_XS: iq2xs_init_impl(512); break;+ default: // nothing+ break;+ }++ ggml_critical_section_end();+}++void ggml_quantize_free(void) {+ ggml_critical_section_start();++ iq2xs_free_impl(256);+ iq2xs_free_impl(512);++ ggml_critical_section_end();+}+size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist) {assert(k % QK4_0 == 0);const int nb = k / QK4_0;@@ -18652,32 +18952,53 @@ size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t *return (n/QK8_0*sizeof(block_q8_0));}-size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist) {+bool ggml_quantize_requires_imatrix(enum ggml_type type) {+ return+ type == GGML_TYPE_IQ2_XXS ||+ type == GGML_TYPE_IQ2_XS;+}++size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start,+ int nrows, int n_per_row, int64_t * hist, const float * imatrix) {+ ggml_quantize_init(type); // this is noop if already initializedsize_t result = 0;+ int n = nrows * n_per_row;switch (type) {case GGML_TYPE_Q4_0:{GGML_ASSERT(start % QK4_0 == 0);- block_q4_0 * block = (block_q4_0*)dst + start / QK4_0;- result = ggml_quantize_q4_0(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q4_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q4_1:{GGML_ASSERT(start % QK4_1 == 0);- block_q4_1 * block = (block_q4_1*)dst + start / QK4_1;- result = ggml_quantize_q4_1(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q4_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q5_0:{GGML_ASSERT(start % QK5_0 == 0);- block_q5_0 * block = (block_q5_0*)dst + start / QK5_0;- result = ggml_quantize_q5_0(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q5_0(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q5_1:{GGML_ASSERT(start % QK5_1 == 0);- block_q5_1 * block = (block_q5_1*)dst + start / QK5_1;- result = ggml_quantize_q5_1(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q5_1(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q8_0:{@@ -18688,48 +19009,77 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, icase GGML_TYPE_Q2_K:{GGML_ASSERT(start % QK_K == 0);- block_q2_K * block = (block_q2_K*)dst + start / QK_K;- result = ggml_quantize_q2_K(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q2_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q3_K:{GGML_ASSERT(start % QK_K == 0);- block_q3_K * block = (block_q3_K*)dst + start / QK_K;- result = ggml_quantize_q3_K(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q3_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q4_K:{GGML_ASSERT(start % QK_K == 0);- block_q4_K * block = (block_q4_K*)dst + start / QK_K;- result = ggml_quantize_q4_K(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q4_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q5_K:{GGML_ASSERT(start % QK_K == 0);- block_q5_K * block = (block_q5_K*)dst + start / QK_K;- result = ggml_quantize_q5_K(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q5_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_Q6_K:{GGML_ASSERT(start % QK_K == 0);- block_q6_K * block = (block_q6_K*)dst + start / QK_K;- result = ggml_quantize_q6_K(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_q6_K(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_IQ2_XXS:{GGML_ASSERT(start % QK_K == 0);- block_iq2_xxs * block = (block_iq2_xxs*)dst + start / QK_K;- result = ggml_quantize_iq2_xxs(src + start, block, n, n, hist);+ GGML_ASSERT(start % n_per_row == 0);+ GGML_ASSERT(imatrix);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_iq2_xxs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);+ } break;+ case GGML_TYPE_IQ2_XS:+ {+ GGML_ASSERT(start % QK_K == 0);+ GGML_ASSERT(start % n_per_row == 0);+ GGML_ASSERT(imatrix);+ size_t start_row = start / n_per_row;+ size_t row_size = ggml_row_size(type, n_per_row);+ result = quantize_iq2_xs(src + start, (char *)dst + start_row * row_size, nrows, n_per_row, hist, imatrix);+ GGML_ASSERT(result == row_size * nrows);} break;case GGML_TYPE_F16:{- int elemsize = sizeof(ggml_fp16_t);+ size_t elemsize = sizeof(ggml_fp16_t);ggml_fp32_to_fp16_row(src + start, (ggml_fp16_t *)dst + start, n);result = n * elemsize;} break;case GGML_TYPE_F32:{- int elemsize = sizeof(float);+ size_t elemsize = sizeof(float);result = n * elemsize;memcpy((uint8_t *)dst + start * elemsize, src + start, result);} break;@@ -19071,8 +19421,8 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i(int64_t) info->ne[3];if (ne % ggml_blck_size(info->type) != 0) {- fprintf(stderr, "%s: tensor '%s' number of elements (%" PRId64 ") is not a multiple of block size (%d)\n",- __func__, info->name.data, ne, ggml_blck_size(info->type));+ fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%d)\n",+ __func__, info->name.data, (int)info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));gguf_free(ctx);return NULL;}@@ -19173,7 +19523,7 @@ void gguf_free(struct gguf_context * ctx) {if (ctx->kv) {// free string memory - not great..- for (uint32_t i = 0; i < ctx->header.n_kv; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_kv; ++i) {struct gguf_kv * kv = &ctx->kv[i];if (kv->key.data) {@@ -19189,7 +19539,7 @@ void gguf_free(struct gguf_context * ctx) {if (kv->type == GGUF_TYPE_ARRAY) {if (kv->value.arr.data) {if (kv->value.arr.type == GGUF_TYPE_STRING) {- for (uint32_t j = 0; j < kv->value.arr.n; ++j) {+ for (uint64_t j = 0; j < kv->value.arr.n; ++j) {struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[j];if (str->data) {free(str->data);@@ -19205,7 +19555,7 @@ void gguf_free(struct gguf_context * ctx) {}if (ctx->infos) {- for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) {+ for (uint64_t i = 0; i < ctx->header.n_tensors; ++i) {struct gguf_tensor_info * info = &ctx->infos[i];if (info->name.data) {@@ -19876,7 +20226,7 @@ int ggml_cpu_has_arm_fma(void) {}int ggml_cpu_has_metal(void) {- return ggml_metal_supported();+ return llamafile_has_metal();}int ggml_cpu_has_f16c(void) {diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 16febaf3..bda99c91 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -232,7 +232,9 @@#define GGML_MAX_PARAMS 2048#define GGML_MAX_CONTEXTS 64#define GGML_MAX_SRC 10+#ifndef GGML_MAX_NAME#define GGML_MAX_NAME 64+#endif#define GGML_MAX_OP_PARAMS 64#define GGML_DEFAULT_N_THREADS 4#define GGML_DEFAULT_GRAPH_SIZE 2048@@ -353,6 +355,7 @@ extern "C" {GGML_TYPE_Q6_K = 14,GGML_TYPE_Q8_K = 15,GGML_TYPE_IQ2_XXS = 16,+ GGML_TYPE_IQ2_XS = 17,GGML_TYPE_I8,GGML_TYPE_I16,GGML_TYPE_I32,@@ -388,6 +391,7 @@ extern "C" {GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensorsGGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors};// available tensor operations:@@ -488,6 +492,8 @@ extern "C" {GGML_UNARY_OP_GELU,GGML_UNARY_OP_GELU_QUICK,GGML_UNARY_OP_SILU,+ GGML_UNARY_OP_HARDSWISH,+ GGML_UNARY_OP_HARDSIGMOID,GGML_UNARY_OP_COUNT,};@@ -658,41 +664,41 @@ extern "C" {GGML_API void ggml_print_object (const struct ggml_object * obj);GGML_API void ggml_print_objects(const struct ggml_context * ctx);- GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor) GGML_CALL;- GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor) GGML_CALL;- GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor) GGML_CALL;- GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN+ GGML_API GGML_CALL int64_t ggml_nelements (const struct ggml_tensor * tensor);+ GGML_API GGML_CALL int64_t ggml_nrows (const struct ggml_tensor * tensor);+ GGML_API GGML_CALL size_t ggml_nbytes (const struct ggml_tensor * tensor);+ GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN- GGML_API int ggml_blck_size(enum ggml_type type) GGML_CALL;- GGML_API size_t ggml_type_size(enum ggml_type type) GGML_CALL; // size in bytes for all elements in a block- GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne) GGML_CALL; // size in bytes for all elements in a row+ GGML_API GGML_CALL int ggml_blck_size(enum ggml_type type);+ GGML_API GGML_CALL size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block+ GGML_API GGML_CALL size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a rowGGML_DEPRECATED(GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float"use ggml_row_size() instead");- GGML_API const char * ggml_type_name(enum ggml_type type) GGML_CALL;- GGML_API const char * ggml_op_name (enum ggml_op op) GGML_CALL;- GGML_API const char * ggml_op_symbol(enum ggml_op op);+ GGML_API GGML_CALL const char * ggml_type_name(enum ggml_type type);+ GGML_API GGML_CALL const char * ggml_op_name (enum ggml_op op);+ GGML_API const char * ggml_op_symbol(enum ggml_op op);- GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);- GGML_API const char * ggml_op_desc(const struct ggml_tensor * t) GGML_CALL; // unary or op name+ GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);+ GGML_API GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name- GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor) GGML_CALL;+ GGML_API GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor);- GGML_API bool ggml_is_quantized(enum ggml_type type) GGML_CALL;+ GGML_API GGML_CALL bool ggml_is_quantized(enum ggml_type type);// TODO: temporary until model loading of ggml examples is refactoredGGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);- GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor) GGML_CALL;- GGML_API bool ggml_is_contiguous(const struct ggml_tensor * tensor) GGML_CALL;- GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor) GGML_CALL;- GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);- GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);- GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);- GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);- GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars+ GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor);+ GGML_API GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor);+ GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);+ GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalarsGGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);@@ -779,7 +785,7 @@ extern "C" {GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);- GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) GGML_CALL;+ GGML_API GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);@@ -1031,6 +1037,16 @@ extern "C" {struct ggml_tensor * a,struct ggml_tensor * b);+ // hardswish(x) = x * relu6(x + 3) / 6+ GGML_API struct ggml_tensor * ggml_hardswish(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // hardsigmoid(x) = relu6(x + 3) / 6+ GGML_API struct ggml_tensor * ggml_hardsigmoid(+ struct ggml_context * ctx,+ struct ggml_tensor * a);+// normalize along rowsGGML_API struct ggml_tensor * ggml_norm(struct ggml_context * ctx,@@ -1174,22 +1190,16 @@ extern "C" {struct ggml_tensor * a,struct ggml_tensor * b);- // a -> b, in-place, return view(b)- GGML_API struct ggml_tensor * ggml_cpy_inplace(+ GGML_API struct ggml_tensor * ggml_cast(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b);+ enum ggml_type type);// make contiguousGGML_API struct ggml_tensor * ggml_cont(struct ggml_context * ctx,struct ggml_tensor * a);- // make contiguous, in-place- GGML_API struct ggml_tensor * ggml_cont_inplace(- struct ggml_context * ctx,- struct ggml_tensor * a);-// make contiguous, with new shapeGGML_API struct ggml_tensor * ggml_cont_1d(struct ggml_context * ctx,@@ -1428,8 +1438,8 @@ extern "C" {float beta_slow);// compute correction dims for YaRN RoPE scaling- void ggml_rope_yarn_corr_dims(- int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]) GGML_CALL;+ GGML_CALL void ggml_rope_yarn_corr_dims(+ int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]);// xPos RoPE, in-place, returns view(a)GGML_API struct ggml_tensor * ggml_rope_xpos_inplace(@@ -1488,6 +1498,17 @@ extern "C" {int d1,bool is_2D);+ GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int s0,+ int s1,+ int p0,+ int p1,+ int d0,+ int d1);+GGML_API struct ggml_tensor * ggml_conv_1d(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1862,8 +1883,8 @@ extern "C" {// ggml_graph_plan() has to be called before ggml_graph_compute()// when plan.work_size > 0, caller must allocate memory for plan.work_data- GGML_API struct ggml_cplan ggml_graph_plan (struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);- GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);+ GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);+ GGML_API int ggml_graph_compute( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);// same as ggml_graph_compute() but the work data is allocated as a part of the context// note: the drawback of this API is that you must have ensured that the context has enough memory for the work data@@ -2070,6 +2091,18 @@ extern "C" {// quantization//+ // - ggml_quantize_init can be called multiple times with the same type+ // it will only initialize the quantization tables for the first call or after ggml_quantize_free+ // automatically called by ggml_quantize_chunk for convenience+ //+ // - ggml_quantize_free will free any memory allocated by ggml_quantize_init+ // call this at the end of the program to avoid memory leaks+ //+ // note: these are thread-safe+ //+ GGML_API void ggml_quantize_init(enum ggml_type type);+ GGML_API void ggml_quantize_free(void);+// TODO: these would probably get removed in favor of the more general ggml_quantize_chunkGGML_API size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);@@ -2082,9 +2115,13 @@ extern "C" {GGML_API size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);GGML_API size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);GGML_API size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist);- GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist);+ // some quantization type cannot be used without an importance matrix+ GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);++ // calls ggml_quantize_init internally (i.e. can allocate memory)+ GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst,+ int start, int nrows, int n_per_row, int64_t * hist, const float * imatrix);//// ggufdiff --git a/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.inc b/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.incnew file mode 100644index 00000000..fd7eca9f--- /dev/null+++ b/llama.cpp/ggml_vec_dot_iq2_xs_q8_K.inc@@ -0,0 +1,157 @@+static void ggml_vec_dot_iq2_xs_q8_K(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) {+ assert(n % QK_K == 0);++ const block_iq2_xs * restrict x = vx;+ const block_q8_K * restrict y = vy;++ const int nb = n / QK_K;++#if defined(__ARM_NEON)++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ ggml_int8x16x4_t q2u;+ ggml_int8x16x4_t q2s;+ ggml_int8x16x4_t q8b;++ int32x4x4_t scales32;++ float sumf = 0;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;+ const uint8x8_t scales8 = vld1_u8(x[i].scales);+ const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));+ const uint8x8_t scales_h = vshr_n_u8(scales8, 4);+ uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));+ scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));+ const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));+ const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));+ scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));+ scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));+ scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));+ scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));+ int32x4_t sumi = vdupq_n_s32(0);+ for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {+ q8b = ggml_vld1q_s8_x4(q8); q8 += 64;+ q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));+ q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));+ q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));+ q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));+ q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));+ q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));+ q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));+ q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));+ q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);+ q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);+ q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);+ q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);+ const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);+ const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);+ const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);+ const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);+ const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));+ sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);+ q2 += 8;+ }+ sumf += d*vaddvq_s32(sumi);+ }+ *s = 0.125f * sumf;++#elif defined(__AVX2__)++ const __m128i m4 = _mm_set1_epi8(0xf);+ const __m128i m1 = _mm_set1_epi8(1);+ const __m128i m511 = _mm_set1_epi16(511);+ const __m128i m127 = _mm_set1_epi16(127);++ const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;++ uint64_t aux64;++ // somewhat hacky, but gives a significant boost in performance+ __m128i aux_gindex, aux_sindex;+ const uint16_t * gindex = (const uint16_t *)&aux_gindex;+ const uint16_t * sindex = (const uint16_t *)&aux_sindex;++ __m256 accumf = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32__F16C(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const int8_t * restrict q8 = y[i].qs;++ memcpy(&aux64, x[i].scales, 8);+ __m128i stmp = _mm_set1_epi64x(aux64);+ stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));+ const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);++ __m256i sumi1 = _mm256_setzero_si256();+ __m256i sumi2 = _mm256_setzero_si256();+ for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {+ const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;+ const __m128i q2_data = _mm_loadu_si128((const __m128i*)q2); q2 += 8;+ aux_gindex = _mm_and_si128(q2_data, m511);+ aux_sindex = _mm_and_si128(_mm_srli_epi16(q2_data, 9), m127);+ const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]], iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);+ const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]], iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);+ const __m256i s2_1 = _mm256_set_epi64x(signs64[sindex[3]], signs64[sindex[2]], signs64[sindex[1]], signs64[sindex[0]]);+ const __m256i s2_2 = _mm256_set_epi64x(signs64[sindex[7]], signs64[sindex[6]], signs64[sindex[5]], signs64[sindex[4]]);+ const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);+ const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);+ const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1);+ const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2);++ const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));+ const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));++ sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));+ sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));+ }++ accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);++ }++ *s = 0.125f * hsum_float_8(accumf);++#else++ float sumf = 0.f;+ for (int i = 0; i < nb; ++i) {+ const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d;+ const uint16_t * restrict q2 = x[i].qs;+ const uint8_t * restrict sc = x[i].scales;+ const int8_t * restrict q8 = y[i].qs;+ int32_t bsum = 0;+ for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {+ const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;+ const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;+ int32_t sumi = 0;+ for (int l = 0; l < 2; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls1;+ sumi = 0;+ for (int l = 2; l < 4; ++l) {+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));+ const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];+ for (int j = 0; j < 8; ++j) {+ sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);+ }+ q8 += 8;+ }+ bsum += sumi * ls2;+ q2 += 4;+ }+ sumf += d * bsum;+ }+ *s = 0.125f * sumf;+#endif+}diff --git a/llama.cpp/imatrix/BUILD.mk b/llama.cpp/imatrix/BUILD.mknew file mode 100644index 00000000..d869b801--- /dev/null+++ b/llama.cpp/imatrix/BUILD.mk@@ -0,0 +1,18 @@+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘++PKGS += LLAMA_CPP_IMATRIX++LLAMA_CPP_IMATRIX_FILES := $(wildcard llama.cpp/imatrix/*)+LLAMA_CPP_IMATRIX_HDRS = $(filter %.h,$(LLAMA_CPP_IMATRIX_FILES))+LLAMA_CPP_IMATRIX_SRCS = $(filter %.cpp,$(LLAMA_CPP_IMATRIX_FILES))+LLAMA_CPP_IMATRIX_OBJS = $(LLAMA_CPP_IMATRIX_SRCS:%.cpp=o/$(MODE)/%.o)++o/$(MODE)/llama.cpp/imatrix/imatrix: \+ o/$(MODE)/llama.cpp/imatrix/imatrix.o \+ o/$(MODE)/llama.cpp/imatrix/imatrix.1.asc.zip.o \+ o/$(MODE)/llama.cpp/llama.cpp.a++.PHONY: o/$(MODE)/llama.cpp/imatrix+o/$(MODE)/llama.cpp/imatrix: \+ o/$(MODE)/llama.cpp/imatrix/imatrixdiff --git a/llama.cpp/imatrix/README.md b/llama.cpp/imatrix/README.mdnew file mode 100644index 00000000..d6657172--- /dev/null+++ b/llama.cpp/imatrix/README.md@@ -0,0 +1,50 @@+# llama.cpp/examples/imatrix++Compute an importance matrix for a model and given text dataset. Can be+used during quantization to enchance the quality of the quantum models.++More information is available here:+https://github.com/ggerganov/llama.cpp/pull/4861++## Usage++```+./imatrix -m <some_fp_model> -f <some_training_data> [-o <output_file>] [--verbosity <verbosity_level>]+ [-ofreq num_chunks] [-ow <0 or 1>] [other common params]+```++Here `-m` with a model name and `-f` with a file containing training+data (such as e.g. `wiki.train.raw`) are mandatory. The parameters in+square brackets are optional and have the following meaning:++* `-o` (or `--output-file`) specifies the name of the file where the+ computed data will be stored. If missing `imatrix.dat` is used.++* `--verbosity` specifies the verbosity level. If set to `0`, no output+ other than the perplexity of the processed chunks will be generated.+ If set to `1`, each time the results are saved a message is written to+ `stderr`. If `>=2`, a message is output each time data is collected+ for any tensor. Default verbosity level is `1`.++* `-ofreq` (or `--output-frequency`) specifies how often the so far+ computed result is saved to disk. Default is 10 (i.e., every 10+ chunks)++* `-ow` (or `--output-weight`) specifies if data will be collected for+ the `output.weight` tensor. My experience is that it is better to not+ utilize the importance matrix when quantizing `output.weight`, so this+ is set to `false` by default.++For faster computation, make sure to use GPU offloading via the `-ngl` argument++## Example++```bash+LLAMA_CUBLAS=1 make -j++# generate importance matrix (imatrix.dat)+./imatrix -m ggml-model-f16.gguf -f train-data.txt -ngl 99++# use the imatrix to perform a Q4_K_M quantization+./quantize --imatrix imatrix.dat ggml-model-f16.gguf ./ggml-model-q4_k_m.gguf q4_k_m+```diff --git a/llama.cpp/imatrix/imatrix.1 b/llama.cpp/imatrix/imatrix.1new file mode 100644index 00000000..f4d995b6--- /dev/null+++ b/llama.cpp/imatrix/imatrix.1@@ -0,0 +1,56 @@+.Dd December 5, 2023+.Dt LLAMAFILE-IMATRIX 1+.Os Llamafile Manual+.Sh NAME+.Nm llamafile-imatrix+.Nd importance matrix builder+.Sh SYNOPSIS+.Nm+.Op flags...+.Fl m Ar model.gguf+.Fl f Ar training.data+.Op Fl o Ar imatrix.dat+.Sh DESCRIPTION+.Nm+Compute an importance matrix for a model and given text dataset. Can be+used during quantization to enchance the quality of the quantum models.+More information is available here:+https://github.com/ggerganov/llama.cpp/pull/4861+.Sh OPTIONS+The following options are available:+.Bl -tag -width indent+.It Fl Fl version+Print version and exit.+.It Fl h , Fl Fl help+Show help message and exit.+.It Fl m Ar FNAME , Fl Fl model Ar FNAME+Model path in the GGUF file format.+.Pp+Default:+.Pa models/7B/ggml-model-f16.gguf+.It Fl f Ar FNAME , Fl Fl file Ar FNAME+Mandatory path of file containing training data, e.g.+.Pa wiki.train.raw+.It Fl o Ar FNAME , Fl Fl output-file Ar FNAME+The name of the file where the computed data will be stored. If this+flag is missing then+.Pa imatrix.dat+is used.+.It Fl ofreq , Fl Fl output-frequency+Specifies how often the so far computed result is saved to disk. The+default is is 10 (i.e., every 10 chunks).+.It Fl ow , Fl Fl output-weight+Specifies if data will be collected for the+.Pa output.weight+tensor. Experience indicates that it is better to not utilize the+importance matrix when quantizing+.Pa output.weight ,+so this is set to false by default.+.El+.Sh PROTIPS+For faster computation, pass the+.Fl ngl Ar 9999+flag for GPU offloading.+.Sh SEE ALSO+.Xr llamafile 1 ,+.Xr llamafile-quantize 1diff --git a/llama.cpp/imatrix/imatrix.1.asc b/llama.cpp/imatrix/imatrix.1.ascnew file mode 100644index 00000000..6e0cbac2--- /dev/null+++ b/llama.cpp/imatrix/imatrix.1.asc@@ -0,0 +1,30 @@+LLAMAFILE-IMATRIX(1) BSD General Commands Manual LLAMAFILE-IMATRIX(1)++NNAAMMEE+ llllaammaaffiillee--iimmaattrriixx — importance matrix builder++SSYYNNOOPPSSIISS+ llllaammaaffiillee--iimmaattrriixx [flags...]++DDEESSCCRRIIPPTTIIOONN+ llllaammaaffiillee--iimmaattrriixx Compute an importance matrix for a model and given text+ dataset. Can be used during quantization to enchance the quality of the+ quantum models.++ More information is available here:+ https://github.com/ggerganov/llama.cpp/pull/4861++OOPPTTIIOONNSS+ The following options are available:++ ----vveerrssiioonn+ Print version and exit.++ --hh, ----hheellpp+ Show help message and exit.++SSEEEE AALLSSOO+ llamafile(1), llamafile-quantize(1), llamafile-perplexity(1),+ llava-quantize(1), zipalign(1), unzip(1)++Llamafile Manual December 5, 2023 Llamafile Manualdiff --git a/llama.cpp/imatrix/imatrix.cpp b/llama.cpp/imatrix/imatrix.cppnew file mode 100644index 00000000..8eebefc7--- /dev/null+++ b/llama.cpp/imatrix/imatrix.cpp@@ -0,0 +1,532 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi++#include "llama.cpp/common.h"+#include "llamafile/llamafile.h"+#include "llama.cpp/llama.h"++#include <cmath>+#include <cstdio>+#include <cstring>+#include <ctime>+#include <sstream>+#include <thread>+#include <mutex>+#include <vector>+#include <fstream>+#include <unordered_map>+#include <algorithm>++#if defined(_MSC_VER)+#pragma warning(disable: 4244 4267) // possible loss of data+#endif++struct Stats {+ std::vector<float> values;+ int ncall = 0;+};++struct StatParams {+ std::string ofile = "imatrix.dat";+ int n_output_frequency = 10;+ int verbosity = 1;+ int keep_every = 0;+ bool collect_output_weight = false;+};++class IMatrixCollector {+public:+ IMatrixCollector() = default;+ void set_parameters(StatParams&& params) { m_params = std::move(params); }+ bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data);+ void save_imatrix() const;+private:+ std::unordered_map<std::string, Stats> m_stats;+ StatParams m_params;+ std::mutex m_mutex;+ int m_last_call = 0;+ std::vector<float> m_src1_data;+ std::vector<int> m_ids; // the expert ids from ggml_mul_mat_id+ //+ void save_imatrix(const char * file_name) const;+ void keep_imatrix(int ncall) const;+};++bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) {+ GGML_UNUSED(user_data);++ const struct ggml_tensor * src0 = t->src[0];+ const struct ggml_tensor * src1 = t->src[1];++ // when ask is true, the scheduler wants to know if we are interested in data from this tensor+ // if we return true, a follow-up call will be made with ask=false in which we can do the actual collection+ if (ask) {+ if (t->op == GGML_OP_MUL_MAT_ID) return true; // collect all indirect matrix multiplications+ if (t->op != GGML_OP_MUL_MAT) return false;+ if (src1->ne[1] < 16 || src1->type != GGML_TYPE_F32) return false;+ if (!(strncmp(src0->name, "blk.", 4) == 0 || (m_params.collect_output_weight && strcmp(src0->name, "output.weight") == 0))) return false;+ return true;+ }++ std::lock_guard<std::mutex> lock(m_mutex);++ // copy the data from the GPU memory if needed+ const bool is_host = ggml_backend_buffer_is_host(src1->buffer);++ if (!is_host) {+ m_src1_data.resize(ggml_nelements(src1));+ ggml_backend_tensor_get(src1, m_src1_data.data(), 0, ggml_nbytes(src1));+ }++ const float * data = is_host ? (const float *) src1->data : m_src1_data.data();++ if (t->op == GGML_OP_MUL_MAT_ID) {+ const int idx = ((int32_t *) t->op_params)[0];+ const int n_as = ((int32_t *) t->op_params)[1];++ // the top-k selected expert ids are stored in the src0 tensor+ // for simplicity, always copy src0 to host, because it is small+ // take into account that src0 is not contiguous!+ GGML_ASSERT(src0->ne[1] == src1->ne[1]);+ GGML_ASSERT(n_as*ggml_nrows(src0)*sizeof(int) == GGML_PAD(ggml_nbytes(src0), n_as*sizeof(int)));+ m_ids.resize(ggml_nbytes(src0)/sizeof(int));+ ggml_backend_tensor_get(src0, m_ids.data(), 0, ggml_nbytes(src0));++ // loop over all possible experts, regardless if they are used or not in the batch+ // this is necessary to guarantee equal number of "ncall" for each tensor+ for (int ex = 0; ex < n_as; ++ex) {+ src0 = t->src[2 + ex];+ auto& e = m_stats[src0->name];+ if (e.values.empty()) {+ e.values.resize(src1->ne[0], 0);+ }+ else if (e.values.size() != (size_t)src1->ne[0]) {+ fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]);+ exit(1); //GGML_ASSERT(false);+ }+ // NOTE: since we select top-k experts, the number of calls for the expert tensors will be k times larger+ // using the following line, we can correct for that if needed+ //if (idx == t->src[0]->ne[0] - 1) ++e.ncall;+ ++e.ncall;+ if (m_params.verbosity > 1) {+ printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);+ }+ for (int row = 0; row < (int)src1->ne[1]; ++row) {+ const int excur = m_ids[row*n_as + idx];+ GGML_ASSERT(excur >= 0 && excur < n_as); // sanity check+ if (excur != ex) continue;+ const float * x = data + row * src1->ne[0];+ for (int j = 0; j < (int)src1->ne[0]; ++j) {+ e.values[j] += x[j]*x[j];+ }+ }+ if (e.ncall > m_last_call) {+ m_last_call = e.ncall;+ if (m_last_call % m_params.n_output_frequency == 0) {+ save_imatrix();+ }+ if (m_params.keep_every > 0 && m_last_call%m_params.keep_every == 0) {+ keep_imatrix(m_last_call);+ }+ }+ }+ } else {+ auto& e = m_stats[src0->name];+ if (e.values.empty()) {+ e.values.resize(src1->ne[0], 0);+ }+ else if (e.values.size() != (size_t)src1->ne[0]) {+ fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", src0->name, (int)e.values.size(), (int)src1->ne[0]);+ exit(1); //GGML_ASSERT(false);+ }+ ++e.ncall;+ if (m_params.verbosity > 1) {+ printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, src0->name, ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);+ }+ for (int row = 0; row < (int)src1->ne[1]; ++row) {+ const float * x = data + row * src1->ne[0];+ for (int j = 0; j < (int)src1->ne[0]; ++j) {+ e.values[j] += x[j]*x[j];+ }+ }+ if (e.ncall > m_last_call) {+ m_last_call = e.ncall;+ if (m_last_call % m_params.n_output_frequency == 0) {+ save_imatrix();+ }+ if (m_params.keep_every > 0 && m_last_call%m_params.keep_every == 0) {+ keep_imatrix(m_last_call);+ }+ }+ }++ return true;+}++void IMatrixCollector::save_imatrix() const {+ save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str());+}++void IMatrixCollector::keep_imatrix(int ncall) const {+ auto file_name = m_params.ofile;+ if (file_name.empty()) file_name = "imatrix.dat";+ file_name += ".at_";+ file_name += std::to_string(ncall);+ save_imatrix(file_name.c_str());+}++void IMatrixCollector::save_imatrix(const char * fname) const {+ std::ofstream out(fname, std::ios::binary);+ int n_entries = m_stats.size();+ out.write((const char*)&n_entries, sizeof(n_entries));+ for (auto& p : m_stats) {+ int len = p.first.size();+ out.write((const char*)&len, sizeof(len));+ out.write(p.first.c_str(), len);+ out.write((const char*)&p.second.ncall, sizeof(p.second.ncall));+ int nval = p.second.values.size();+ out.write((const char*)&nval, sizeof(nval));+ if (nval > 0) out.write((const char*)p.second.values.data(), nval*sizeof(float));+ }+ if (m_params.verbosity > 0) {+ fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n",__func__,m_last_call,fname);+ }+}++static IMatrixCollector g_collector;++static bool ik_collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) {+ return g_collector.collect_imatrix(t, ask, user_data);+}+++struct results_log_softmax {+ double log_softmax;+ float logit;+ float prob;+};++static std::vector<float> softmax(const std::vector<float>& logits) {+ std::vector<float> probs(logits.size());+ float max_logit = logits[0];+ for (float v : logits) {+ max_logit = std::max(max_logit, v);+ }+ double sum_exp = 0.0;+ for (size_t i = 0; i < logits.size(); i++) {+ // Subtract the maximum logit value from the current logit value for numerical stability+ const float logit = logits[i] - max_logit;+ const float exp_logit = expf(logit);+ sum_exp += exp_logit;+ probs[i] = exp_logit;+ }+ for (size_t i = 0; i < probs.size(); i++) {+ probs[i] /= sum_exp;+ }+ return probs;+}++static results_log_softmax log_softmax(int n_vocab, const float * logits, int tok) {+ float max_logit = logits[0];+ for (int i = 1; i < n_vocab; ++i) {+ max_logit = std::max(max_logit, logits[i]);+ }+ double sum_exp = 0.0;+ for (int i = 0; i < n_vocab; ++i) {+ sum_exp += expf(logits[i] - max_logit);+ }+ return {logits[tok] - max_logit - log(sum_exp), logits[tok], expf(logits[tok] - max_logit) / (float) sum_exp};+}++static void process_logits(+ int n_vocab, const float * logits, const int * tokens, int n_token, std::vector<std::thread> & workers,+ double & nll, double & nll2, float * logit_history, float * prob_history+) {+ std::mutex mutex;+ int counter = 0;+ auto compute = [&mutex, &counter, &nll, &nll2, logit_history, prob_history, n_vocab, logits, tokens, n_token] () {+ double local_nll = 0;+ double local_nll2 = 0;+ while (true) {+ std::unique_lock<std::mutex> lock(mutex);+ int i = counter++;+ if (i >= n_token) {+ nll += local_nll; nll2 += local_nll2;+ break;+ }+ lock.unlock();+ const results_log_softmax results = log_softmax(n_vocab, logits + i*n_vocab, tokens[i+1]);+ const double v = -results.log_softmax;+ local_nll += v;+ local_nll2 += v*v;++ logit_history[i] = results.logit;+ prob_history[i] = results.prob;+ }+ };+ for (auto & w : workers) {+ w = std::thread(compute);+ }+ compute();+ for (auto & w : workers) {+ w.join();+ }+}++static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl) {++ const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));+ const int n_ctx = llama_n_ctx(ctx);++ auto tim1 = std::chrono::high_resolution_clock::now();+ fprintf(stderr, "%s: tokenizing the input ..\n", __func__);++ std::vector<llama_token> tokens = ::llama_tokenize(ctx, params.prompt, add_bos);++ auto tim2 = std::chrono::high_resolution_clock::now();+ fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());++ if (int(tokens.size()) < 2*n_ctx) {+ fprintf(stderr, "%s: you need at least %d tokens for a context of %d tokens\n",__func__,2*n_ctx,+ n_ctx);+ fprintf(stderr, "%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size());+ return false;+ }++ std::vector<float> logit_history;+ std::vector<float> prob_history;++ if (compute_ppl) {+ logit_history.resize(tokens.size());+ prob_history.resize(tokens.size());+ }++ const int n_chunk_max = tokens.size() / n_ctx;++ const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max);+ const int n_vocab = llama_n_vocab(llama_get_model(ctx));+ const int n_batch = params.n_batch;++ int count = 0;+ double nll = 0.0;+ double nll2 = 0.0;++ fprintf(stderr, "%s: computing over %d chunks with batch_size %d\n", __func__, n_chunk, n_batch);++ std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);++ const int num_batches = (n_ctx + n_batch - 1) / n_batch;++ std::vector<float> logits;+ if (compute_ppl && num_batches > 1) {+ logits.reserve((size_t)n_ctx * n_vocab);+ }++ for (int i = 0; i < n_chunk; ++i) {+ const int start = i * n_ctx;+ const int end = start + n_ctx;++ std::vector<float> logits;++ const auto t_start = std::chrono::high_resolution_clock::now();++ // clear the KV cache+ llama_kv_cache_clear(ctx);++ for (int j = 0; j < num_batches; ++j) {+ const int batch_start = start + j * n_batch;+ const int batch_size = std::min(end - batch_start, n_batch);++ // save original token and restore it after eval+ const auto token_org = tokens[batch_start];++ // add BOS token for the first batch of each chunk+ if (add_bos && j == 0) {+ tokens[batch_start] = llama_token_bos(llama_get_model(ctx));+ }++ if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {+ fprintf(stderr, "%s : failed to eval\n", __func__);+ return false;+ }++ // restore the original token in case it was set to BOS+ tokens[batch_start] = token_org;++ if (compute_ppl && num_batches > 1) {+ const auto * batch_logits = llama_get_logits(ctx);+ logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);+ }+ }++ const auto t_end = std::chrono::high_resolution_clock::now();++ if (i == 0) {+ const float t_total = std::chrono::duration<float>(t_end - t_start).count();+ fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total);+ int total_seconds = (int)(t_total * n_chunk);+ if (total_seconds >= 60*60) {+ fprintf(stderr, "%d hours ", total_seconds / (60*60));+ total_seconds = total_seconds % (60*60);+ }+ fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);+ }++ if (compute_ppl) {+ const int first = n_ctx/2;+ const auto all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);+ process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,+ workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first);+ count += n_ctx - first - 1;++ printf("[%d]%.4lf,", i + 1, std::exp(nll / count));+ fflush(stdout);++ logits.clear();+ }+ }+ printf("\n");++ if (compute_ppl) {+ nll2 /= count;+ nll /= count;+ const double ppl = exp(nll);+ nll2 -= nll * nll;+ if (nll2 > 0) {+ nll2 = sqrt(nll2/(count-1));+ printf("Final estimate: PPL = %.4lf +/- %.5lf\n", ppl, nll2*ppl);+ } else {+ printf("Unexpected negative standard deviation of log(prob)\n");+ }+ }++ return true;+}++int main(int argc, char ** argv) {++ if (llamafile_has(argv, "--version")) {+ puts("llamafile-imatrix v" LLAMAFILE_VERSION_STRING);+ return 0;+ }++ if (llamafile_has(argv, "-h") ||+ llamafile_has(argv, "-help") ||+ llamafile_has(argv, "--help")) {+ llamafile_help("/zip/llama.cpp/imatrix/imatrix.1.asc");+ __builtin_unreachable();+ }++ llamafile_init();+ llamafile_check_cpu();++ StatParams sparams;+ bool compute_ppl = true;+ std::vector<char*> args;+ args.push_back(argv[0]);+ int iarg = 1;+ for (; iarg < argc-1; ++iarg) {+ std::string arg{argv[iarg]};+ if (arg == "-o" || arg == "--output-file") {+ sparams.ofile = argv[++iarg];+ }+ else if (arg == "-ofreq" || arg == "--output-frequency") {+ sparams.n_output_frequency = std::stoi(argv[++iarg]);+ }+ else if (arg == "-ow" || arg == "--output-weight") {+ sparams.collect_output_weight = std::stoi(argv[++iarg]);+ }+ else if (arg == "--verbosity") {+ sparams.verbosity = std::stoi(argv[++iarg]);+ } else if (arg == "--no-ppl") {+ compute_ppl = false;+ } else if (arg == "--keep-imatrix") {+ sparams.keep_every = std::stoi(argv[++iarg]);+ } else {+ args.push_back(argv[iarg]);+ }+ }+ if (iarg < argc) {+ std::string arg{argv[iarg]};+ if (arg == "--no-ppl") {+ compute_ppl = false;+ } else {+ args.push_back(argv[iarg]);+ }+ }++ gpt_params params;+ params.n_batch = 512;+ if (!gpt_params_parse(args.size(), args.data(), params)) {+ return 1;+ }++ g_collector.set_parameters(std::move(sparams));++ params.logits_all = true;+ params.n_batch = std::min(params.n_batch, params.n_ctx);++ print_build_info();++ if (params.seed == LLAMA_DEFAULT_SEED) {+ params.seed = time(NULL);+ }++ fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);++ std::mt19937 rng(params.seed);+ if (params.random_prompt) {+ params.prompt = gpt_random_prompt(rng);+ }++ llama_backend_init(params.numa);++ llama_model_params mparams = llama_model_params_from_gpt_params(params);++ llama_model * model = llama_load_model_from_file(params.model.c_str(), mparams);+ if (model == NULL) {+ fprintf(stderr, "%s: error: unable to load model\n", __func__);+ return 1;+ }++ llama_context_params cparams = llama_context_params_from_gpt_params(params);++ // pass the callback to the backend scheduler+ // it will be executed for each node during the graph computation+ cparams.cb_eval = ik_collect_imatrix;+ cparams.cb_eval_user_data = NULL;++ llama_context * ctx = llama_new_context_with_model(model, cparams);+ if (ctx == NULL) {+ fprintf(stderr, "%s: error: unable to create context\n", __func__);+ return 1;+ }++ const int n_ctx_train = llama_n_ctx_train(model);+ if (params.n_ctx > n_ctx_train) {+ fprintf(stderr, "%s: warning: model was trained on only %d context tokens (%d specified)\n",+ __func__, n_ctx_train, params.n_ctx);+ }++ // print system information+ {+ fprintf(stderr, "\n");+ fprintf(stderr, "%s\n", get_system_info(params).c_str());+ }++ bool OK = compute_imatrix(ctx, params, compute_ppl);+ if (!OK) {+ return 1;+ }++ g_collector.save_imatrix();++ llama_print_timings(ctx);++ llama_free(ctx);+ llama_free_model(model);++ llama_backend_free();++ return 0;+}diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex c32d38ae..1c72a52f 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -1,27 +1,16 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#define LLAMA_API_INTERNAL-//#define LLAMA_GGML_BACKEND_CUDA_TEST // for testing only - enables ggml-cuda through ggml-backend, disables partial offloading#include "llama.h"#include "unicode.h"#include "ggml.h"-#include "ggml-alloc.h"+#include "ggml-backend.h"#include "ggml-metal.h"#include "ggml-cuda.h"-#include "llamafile/log.h"--#if defined(GGML_USE_CLBLAST)-# include "ggml-opencl.h"-#endif--#ifdef GGML_USE_MPI-# include "ggml-mpi.h"-#endif#ifndef QK_K# ifdef GGML_QKK_64# define QK_K 64@@ -60,6 +49,7 @@#include <thread>#include <type_traits>#include <unordered_map>+#include <fcntl.h>#include <unistd.h>#include <sys/mman.h>#include <sys/resource.h>@@ -125,10 +115,6 @@ static bool is_float_close(float a, float b, float abs_tol) {return std::fabs(b - a) <= abs_tol;}-#ifdef GGML_USE_CPU_HBM-#include <hbwmalloc.h>-#endif-static void zeros(std::ofstream & file, size_t n) {char zero = 0;for (size_t i = 0; i < n; ++i) {@@ -170,8 +156,10 @@ enum llm_arch {LLM_ARCH_BLOOM,LLM_ARCH_STABLELM,LLM_ARCH_QWEN,+ LLM_ARCH_QWEN2,LLM_ARCH_PHI2,LLM_ARCH_PLAMO,+ LLM_ARCH_CODESHELL,LLM_ARCH_UNKNOWN,};@@ -189,8 +177,10 @@ static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {{ LLM_ARCH_BLOOM, "bloom" },{ LLM_ARCH_STABLELM, "stablelm" },{ LLM_ARCH_QWEN, "qwen" },+ { LLM_ARCH_QWEN2, "qwen2" },{ LLM_ARCH_PHI2, "phi2" },{ LLM_ARCH_PLAMO, "plamo" },+ { LLM_ARCH_CODESHELL, "codeshell" },};enum llm_kv {@@ -544,6 +534,23 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_QWEN2,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_PHI2,{@@ -552,6 +559,9 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_OUTPUT, "output" },{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },{ LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },@@ -575,6 +585,26 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_CODESHELL,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,@@ -801,7 +831,7 @@ struct llama_mmap {// therefore it's already mappedis_owned = false;addr = llamafile_content(file->file);- if (!llamafile_gpu_supported()) {+ if (!llamafile_has_gpu()) {llamafile_schlep(addr, size);}return;@@ -839,7 +869,7 @@ struct llama_mmap {// report terminal progress of loading weights off the disk into// the cpu. if we're using gpu inference, then don't even bother- if (!llamafile_gpu_supported()) {+ if (!llamafile_has_gpu()) {llamafile_schlep(addr, size);}@@ -992,24 +1022,18 @@ struct llama_mlock {suggest = false;}- fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",- size, this->size, errmsg, suggest ? get_mlock_suggestion() : "");+ LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",+ size, this->size, errmsg, suggest ? get_mlock_suggestion() : "");return false;}static void raw_unlock(void * addr, size_t size) {if (munlock(addr, size)) {- fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));+ LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));}}};-typedef void (*GGML_CALL offload_func_t)(struct ggml_tensor * tensor);--GGML_CALL static void ggml_offload_nop(struct ggml_tensor * tensor) {- (void) tensor;-}-static std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) {std::vector<char> result(8, 0);const int n_tokens = llama_token_to_piece(llama_get_model(ctx), token, result.data(), result.size());@@ -1025,30 +1049,49 @@ static std::string llama_token_to_piece(const struct llama_context * ctx, llama_return std::string(result.data(), result.size());}-static ggml_backend_buffer_type_t llama_default_buffer_type(int n_gpu_layers) {+static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer) {ggml_backend_buffer_type_t buft = nullptr;- if (n_gpu_layers > 0) {- buft = ggml_backend_metal_buffer_type();+ // host buffers should only be used when data is expected to be copied to/from the GPU+ if (host_buffer) {+ buft = ggml_backend_cuda_host_buffer_type();}if (buft == nullptr) {-#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (n_gpu_layers > 0) {- buft = ggml_backend_cuda_buffer_type(0);- }-#else- buft = ggml_backend_cuda_host_buffer_type();-#endif+ buft = ggml_backend_cpu_buffer_type();}+ return buft;++ GGML_UNUSED(host_buffer);+}++static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) {+ ggml_backend_buffer_type_t buft = nullptr;++ if (!buft) buft = ggml_backend_metal_buffer_type();+ if (!buft) buft = ggml_backend_cuda_buffer_type(gpu);if (buft == nullptr) {- buft = ggml_backend_cpu_buffer_type();+ buft = llama_default_buffer_type_cpu(true);+ }+ return buft;++ GGML_UNUSED(gpu);+}++static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gpu, const float * tensor_split) {+ ggml_backend_buffer_type_t buft = nullptr;++ if (ggml_backend_cuda_get_device_count() > 1) {+ buft = ggml_backend_cuda_split_buffer_type(tensor_split);}+ if (buft == nullptr) {+ buft = llama_default_buffer_type_offload(fallback_gpu);+ }return buft;- GGML_UNUSED(n_gpu_layers);+ GGML_UNUSED(tensor_split);}//@@ -1071,8 +1114,10 @@ static llama_state g_state;// available llama modelsenum e_model {MODEL_UNKNOWN,+ MODEL_0_5B,MODEL_1B,MODEL_3B,+ MODEL_4B,MODEL_7B,MODEL_8B,MODEL_13B,@@ -1180,6 +1225,9 @@ struct llama_cparams {bool mul_mat_q;bool offload_kqv;++ ggml_backend_sched_eval_callback cb_eval;+ void * cb_eval_user_data;};struct llama_layer {@@ -1258,24 +1306,24 @@ struct llama_kv_cache {std::vector<struct ggml_tensor *> k_l; // per layerstd::vector<struct ggml_tensor *> v_l;- struct ggml_context * ctx = NULL;+ std::vector<struct ggml_context *> ctxs;+ std::vector<ggml_backend_buffer_t> bufs;- ggml_backend_buffer_t buf = NULL;+ size_t total_size() const {+ size_t size = 0;+ for (ggml_backend_buffer_t buf : bufs) {+ size += ggml_backend_buffer_get_size(buf);+ }+ return size;+ }~llama_kv_cache() {-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {- for (size_t i = 0; i < k_l.size(); ++i) {- ggml_cuda_free_data(k_l[i]);- ggml_cuda_free_data(v_l[i]);- }- }-#endif- if (ctx) {+ for (struct ggml_context * ctx : ctxs) {ggml_free(ctx);}-- ggml_backend_buffer_free(buf);+ for (ggml_backend_buffer_t buf : bufs) {+ ggml_backend_buffer_free(buf);+ }}};@@ -1352,22 +1400,38 @@ struct llama_model {std::vector<llama_layer> layers;+ llama_split_mode split_mode;+ int main_gpu;int n_gpu_layers;// gguf metadatastd::unordered_map<std::string, std::string> gguf_kv;- // context- struct ggml_context * ctx = NULL;+ // layer -> buffer type mapping+ struct layer_buft {+ layer_buft() : buft_matrix(nullptr), buft(nullptr) {}+ layer_buft(ggml_backend_buffer_type_t matrix) : buft_matrix(matrix), buft(matrix) {}+ layer_buft(ggml_backend_buffer_type_t matrix, ggml_backend_buffer_type_t other) : buft_matrix(matrix), buft(other) {}- // the model memory buffer- ggml_backend_buffer_t buf = NULL;+ ggml_backend_buffer_type_t buft_matrix; // matrices only - used by split buffers and backends that support only matrix multiplication+ ggml_backend_buffer_type_t buft; // everything else+ };++ layer_buft buft_input;+ layer_buft buft_output;+ std::vector<layer_buft> buft_layer;++ // contexts where the model tensors metadata is stored+ std::vector<struct ggml_context *> ctxs;++ // the model memory buffers for the tensor data+ std::vector<ggml_backend_buffer_t> bufs;// model memory mapped filestd::unique_ptr<llama_mmap> mapping;// objects representing data potentially being locked in memory- llama_mlock mlock_buf;+ std::vector<std::unique_ptr<llama_mlock>> mlock_bufs;llama_mlock mlock_mmap;// for quantize-stats only@@ -1377,39 +1441,33 @@ struct llama_model {int64_t t_start_us = 0;~llama_model() {-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {- for (size_t i = 0; i < tensors_by_name.size(); ++i) {- ggml_cuda_free_data(tensors_by_name[i].second);- }- ggml_cuda_free_scratch();- }-#endif--#if defined(GGML_USE_CLBLAST)- for (size_t i = 0; i < tensors_by_name.size(); ++i) {- ggml_cl_free_data(tensors_by_name[i].second);- }-#endif- if (ctx) {+ for (struct ggml_context * ctx : ctxs) {ggml_free(ctx);}-- ggml_backend_buffer_free(buf);+ for (ggml_backend_buffer_t buf : bufs) {+ ggml_backend_buffer_free(buf);+ }}};struct llama_context {llama_context(const llama_model & model) : model(model), t_start_us(model.t_start_us), t_load_us(model.t_load_us) {}~llama_context() {- ggml_allocr_free(alloc);- ggml_backend_buffer_free(buf_alloc);- ggml_backend_free(backend);+ ggml_backend_sched_free(sched);++ for (ggml_backend_t backend : backends) {+ ggml_backend_free(backend);+ }++ ggml_backend_buffer_free(buf_input);+ ggml_free(ctx_input);}llama_cparams cparams;- ggml_backend_t backend = nullptr;+ std::vector<ggml_backend_t> backends;+ ggml_backend_t backend_metal = nullptr;+ ggml_backend_t backend_cpu = nullptr;const llama_model & model;@@ -1443,11 +1501,18 @@ struct llama_context {// memory buffers used to evaluate the modelstd::vector<uint8_t> buf_compute_meta;- ggml_backend_buffer_t buf_alloc = NULL;- ggml_allocr * alloc = NULL;-- // temporary buffer for copying data to/from the backend- std::vector<no_init<uint8_t>> buf_copy;+ ggml_backend_sched_t sched = nullptr;+ // allocator for the input tensors+ ggml_tallocr * alloc = nullptr;++ // input tensors+ ggml_backend_buffer_t buf_input = nullptr;+ ggml_context * ctx_input = nullptr;+ struct ggml_tensor * inp_tokens; // I32 [n_batch]+ struct ggml_tensor * inp_embd; // F32 [n_embd, n_batch]+ struct ggml_tensor * inp_pos; // I32 [n_batch]+ struct ggml_tensor * inp_KQ_mask; // F32 [n_ctx, n_batch]+ struct ggml_tensor * inp_K_shift; // I32 [n_ctx]#ifdef GGML_USE_MPIggml_mpi_context * ctx_mpi = NULL;@@ -1459,16 +1524,17 @@ struct llama_context {//static bool llama_kv_cache_init(- const struct llama_hparams & hparams,struct llama_kv_cache & cache,+ const llama_model & model,ggml_type ktype,ggml_type vtype,uint32_t n_ctx,- int n_gpu_layers,bool offload) {+ const struct llama_hparams & hparams = model.hparams;+const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();- const uint32_t n_layer = hparams.n_layer;+ const int64_t n_layer = hparams.n_layer;cache.has_shift = false;@@ -1479,62 +1545,65 @@ static bool llama_kv_cache_init(cache.cells.clear();cache.cells.resize(n_ctx);- struct ggml_init_params params;- params.mem_size = 2u*n_layer*ggml_tensor_overhead();- params.mem_buffer = NULL;- params.no_alloc = true;-- cache.ctx = ggml_init(params);+#ifdef GGML_USE_CLBLAST+ offload = false;+#endif- size_t vram_kv_cache = 0;+ // count used buffer types+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;+ if (offload) {+ for (int64_t i = 0; i < n_layer; ++i) {+ buft_layer_count[model.buft_layer[i].buft]++;+ }+ } else {+ buft_layer_count[llama_default_buffer_type_cpu(true)] = n_layer;+ }- if (!cache.ctx) {- LLAMA_LOG_ERROR("%s: failed to allocate memory for kv cache\n", __func__);- return false;+ // create a context for each buffer type+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;+ for (auto & it : buft_layer_count) {+ int n_layers = it.second;+ struct ggml_init_params params = {+ /*.mem_size =*/ 2u*n_layers*ggml_tensor_overhead(),+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,+ };+ ggml_context * ctx = ggml_init(params);+ if (!ctx) {+ LLAMA_LOG_ERROR("%s: failed to allocate context for kv cache\n", __func__);+ return false;+ }+ ctx_map[it.first] = ctx;+ cache.ctxs.push_back(ctx);}cache.k_l.reserve(n_layer);cache.v_l.reserve(n_layer);- const int i_gpu_start = (int) n_layer - n_gpu_layers;-for (int i = 0; i < (int) n_layer; i++) {- ggml_tensor * k = ggml_new_tensor_1d(cache.ctx, ktype, n_embd_k_gqa*n_ctx);- ggml_tensor * v = ggml_new_tensor_1d(cache.ctx, vtype, n_embd_v_gqa*n_ctx);+ struct ggml_context * ctx = offload ? ctx_map.at(model.buft_layer[i].buft) : cache.ctxs.front();+ ggml_tensor * k = ggml_new_tensor_1d(ctx, ktype, n_embd_k_gqa*n_ctx);+ ggml_tensor * v = ggml_new_tensor_1d(ctx, vtype, n_embd_v_gqa*n_ctx);ggml_format_name(k, "cache_k_l%d", i);ggml_format_name(v, "cache_v_l%d", i);cache.k_l.push_back(k);cache.v_l.push_back(v);-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (i >= i_gpu_start) {- if (offload) {- ggml_cuda_assign_buffers_no_scratch(k);- ggml_cuda_assign_buffers_no_scratch(v);- vram_kv_cache += ggml_nbytes(k);- vram_kv_cache += ggml_nbytes(v);- // HACK: mark tensor as allocated- k->data = v->data = (void *)(uintptr_t)1;- }- }-#endif // GGML_USE_CUBLAS- }-- // allocate tensors- cache.buf = ggml_backend_alloc_ctx_tensors_from_buft(cache.ctx, llama_default_buffer_type(n_gpu_layers));-- // buf may be NULL with full offload- if (cache.buf) {- // initialize the buffer to avoid NaNs in the padding- ggml_backend_buffer_clear(cache.buf, 0);}- if (vram_kv_cache > 0) {- LLAMA_LOG_INFO("%s: VRAM kv self = %.2f MB\n", __func__, vram_kv_cache / 1024.0 / 1024.0);+ // allocate tensors and initialize the buffers to avoid NaNs in the padding+ for (auto it : ctx_map) {+ ggml_backend_buffer_type_t buft = it.first;+ ggml_context * ctx = it.second;+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);+ if (!buf) {+ LLAMA_LOG_ERROR("%s: failed to allocate buffer for kv cache\n", __func__);+ return false;+ }+ ggml_backend_buffer_clear(buf, 0);+ LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);+ cache.bufs.push_back(buf);}- GGML_UNUSED(i_gpu_start);- GGML_UNUSED(offload);-return true;}@@ -1858,13 +1927,13 @@ namespace GGUFMeta {__func__, override_type_to_str(override->tag), override->key);switch (override->tag) {case LLAMA_KV_OVERRIDE_BOOL: {- printf("%s\n", override->bool_value ? "true" : "false");+ LLAMA_LOG_INFO("%s\n", override->bool_value ? "true" : "false");} break;case LLAMA_KV_OVERRIDE_INT: {- printf("%" PRId64 "\n", override->int_value);+ LLAMA_LOG_INFO("%" PRId64 "\n", override->int_value);} break;case LLAMA_KV_OVERRIDE_FLOAT: {- printf("%.6f\n", override->float_value);+ LLAMA_LOG_INFO("%.6f\n", override->float_value);} break;default:// Shouldn't be possible to end up here, but just in case...@@ -1963,6 +2032,11 @@ struct llama_model_loader {LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);llama_model_loader(const std::string & fname, bool use_mmap, const struct llama_model_kv_override * param_overrides_p) : file(fname.c_str(), "rbe") {+ int trace = 0;+ if (getenv("LLAMA_TRACE")) {+ trace = atoi(getenv("LLAMA_TRACE"));+ }+struct gguf_init_params params = {/*.no_alloc = */ true,/*.ctx = */ &ctx_meta,@@ -2016,27 +2090,27 @@ struct llama_model_loader {type_max = type;}- // TODO: make runtime configurable-#if 0- struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i));- LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, ggml_get_name(meta), ggml_type_name(type), llama_format_tensor_shape(meta).c_str());-#endif+ if (trace > 0) {+ struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i));+ LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, ggml_get_name(meta), ggml_type_name(type), llama_format_tensor_shape(meta).c_str());+ }}switch (type_max) {- case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;- case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;- case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;- case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;- case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;- case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;- case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;- case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;- case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;- case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;- case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;- case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;+ case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;+ case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;+ case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;+ case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;+ case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;+ case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;+ case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;+ case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;+ case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;+ case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;+ case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;+ case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;+ case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;default:{LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));@@ -2167,9 +2241,8 @@ struct llama_model_loader {return get_tensor_meta(get_tensor_name(i));}- struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta, ggml_backend_type backend) {+ struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta) {struct ggml_tensor * tensor = ggml_dup_tensor(ctx, meta);- tensor->backend = backend; // TODO: ggml_set_backendggml_set_name(tensor, ggml_get_name(meta));n_created++;@@ -2177,7 +2250,7 @@ struct llama_model_loader {return tensor;}- struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector<int64_t> & ne, ggml_backend_type backend, bool required = true) {+ struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector<int64_t> & ne, bool required = true) {struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, name.c_str());if (cur == NULL) {@@ -2187,12 +2260,6 @@ struct llama_model_loader {throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));}- if (backend == GGML_BACKEND_GPU_SPLIT) {- if (ne.size() == 1) {- throw std::runtime_error(format("%s: 1-dimensional tensor '%s' cannot be split on the GPU", __func__, name.c_str()));- }- }-{bool is_ok = true;for (size_t i = 0; i < ne.size(); ++i) {@@ -2210,7 +2277,7 @@ struct llama_model_loader {}}- return create_tensor_for(ctx, cur, backend);+ return create_tensor_for(ctx, cur);}void done_getting_tensors() const {@@ -2229,25 +2296,35 @@ struct llama_model_loader {return gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, idx);}- void init_mapping(bool prefetch = true) {- /*- // prefetch only CPU tensors+ void init_mapping(bool prefetch = true, llama_mlock * lmlock = nullptr) {+ // prefetch the whole file - all the data is needed anywayif (use_mmap) {- size_t size_pref = 0; // prefetch+ mapping.reset(new llama_mmap(&file, prefetch ? -1 : 0, ggml_is_numa()));+ }- for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {- struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));- if (cur->backend == GGML_BACKEND_CPU) {- size_t tensor_end = gguf_get_tensor_offset(ctx_gguf, i) + ggml_nbytes(cur);- size_pref = std::max(size_pref, tensor_end);- }+ // compute the total size of all tensors for progress reporting+ for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {+ struct ggml_tensor * cur = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i));+ size_data += ggml_nbytes(cur);+ }++ if (use_mmap && mapping) {+ if (lmlock) {+ lmlock->init(mapping->addr);}- mapping.reset(new llama_mmap(&file, gguf_get_data_offset(ctx_gguf) + size_pref, ggml_is_numa()));+ mmap_used_first = mapping->size;}- */- // prefetch the whole file - all the data is needed anyway- if (use_mmap) {- mapping.reset(new llama_mmap(&file, prefetch ? -1 : 0, ggml_is_numa()));+ }++ void get_mapping_range(size_t * first, size_t * last, ggml_context * ctx) const {+ GGML_ASSERT(mapping);++ *first = mapping->size;+ *last = 0;+ for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {+ const size_t offs = file_offset(ggml_get_name(tensor));+ *first = std::min(*first, offs);+ *last = std::max(*last, offs + ggml_nbytes(tensor));}}@@ -2256,8 +2333,11 @@ struct llama_model_loader {const size_t offs = file_offset(ggml_get_name(cur));if (use_mmap && mapping) {- GGML_ASSERT(cur->data == nullptr);- cur->data = (uint8_t *)mapping->addr + offs;+ if (cur->data == nullptr) {+ cur->data = (uint8_t *)mapping->addr + offs;+ } else {+ memcpy(cur->data, (uint8_t *)mapping->addr + offs, ggml_nbytes(cur));+ }} else {GGML_ASSERT(cur->data != nullptr);file.seek(offs, SEEK_SET);@@ -2265,37 +2345,23 @@ struct llama_model_loader {}}- // Returns false if cancelled by progress_callback- bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) const {- size_t size_data = 0;-- for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {- struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));- size_data += ggml_nbytes(cur);- }-- if (use_mmap && buf_mmap) {- if (lmlock) {- lmlock->init(mapping->addr);- }- }+ size_t size_done = 0;+ size_t size_data = 0;+ size_t mmap_used_first = -1;+ size_t mmap_used_last = 0;-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- const bool legacy_offload = ggml_cublas_loaded();-#else- const bool legacy_offload = false;-#endif+ // Returns false if cancelled by progress_callback+ bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) {+ GGML_ASSERT(size_data != 0 && "call init_mapping() first");std::vector<no_init<uint8_t>> read_buf;- size_t size_done = 0;-- size_t mmap_first = -1;- size_t mmap_last = 0;-for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));- GGML_ASSERT(cur); // unused tensors should have been caught by load_data already+ if (!cur) {+ // some tensors may be allocated in a different context+ continue;+ }if (progress_callback) {if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {@@ -2305,67 +2371,48 @@ struct llama_model_loader {const size_t offs = file_offset(ggml_get_name(cur));- if (!legacy_offload || cur->backend == GGML_BACKEND_CPU) {- if (use_mmap && mapping) {- if (buf_mmap) {- ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs);- if (lmlock) {- lmlock->grow_to(offs + ggml_nbytes(cur));- }- mmap_first = std::min(mmap_first, offs);- mmap_last = std::max(mmap_last, offs + ggml_nbytes(cur));- } else {- ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur));+ if (use_mmap && mapping) {+ if (buf_mmap && cur->data == nullptr) {+ ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs);+ if (lmlock) {+ lmlock->grow_to(offs + ggml_nbytes(cur));}+ mmap_used_first = std::min(mmap_used_first, offs);+ mmap_used_last = std::max(mmap_used_last, offs + ggml_nbytes(cur));} else {- if (ggml_backend_buffer_is_host(cur->buffer)) {- file.seek(offs, SEEK_SET);- file.read_raw(cur->data, ggml_nbytes(cur));- } else {- read_buf.resize(ggml_nbytes(cur));- file.seek(offs, SEEK_SET);- file.read_raw(read_buf.data(), ggml_nbytes(cur));- ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur));- }+ ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur));}} else {- // HACK: mark tensor as allocated- cur->data = (void *)(uintptr_t)1;- void * data;- if (use_mmap && mapping) {- data = (uint8_t *) mapping->addr + offs;+ if (ggml_backend_buffer_is_host(cur->buffer)) {+ file.seek(offs, SEEK_SET);+ file.read_raw(cur->data, ggml_nbytes(cur));} else {read_buf.resize(ggml_nbytes(cur));file.seek(offs, SEEK_SET);file.read_raw(read_buf.data(), ggml_nbytes(cur));- data = read_buf.data();+ ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur));}--#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- ggml_cuda_transform_tensor(data, cur);-#elif defined(GGML_USE_CLBLAST)- GGML_ASSERT(cur->backend == GGML_BACKEND_GPU);- ggml_cl_transform_tensor(data, cur);-#else- GGML_ASSERT(!"GPU tensor without a GPU backend");- GGML_UNUSED(data);-#endif}size_done += ggml_nbytes(cur);}- // unmap offloaded tensors and metadata- if (use_mmap && mapping) {- mapping->unmap_fragment(0, mmap_first);- mapping->unmap_fragment(mmap_last, mapping->size);+ // check if this is the last call and do final cleanup+ if (size_done >= size_data) {+ // unmap offloaded tensors and metadata+ if (use_mmap && mapping) {+ mapping->unmap_fragment(0, mmap_used_first);+ if (mmap_used_last != 0) {+ mapping->unmap_fragment(mmap_used_last, mapping->size);+ }+ }+ if (progress_callback) {+ // Even though the model is done loading, we still honor+ // cancellation since we need to free allocations.+ return progress_callback(1.0f, progress_callback_user_data);+ }}- if (progress_callback) {- // Even though the model is done loading, we still honor- // cancellation since we need to free allocations.- return progress_callback(1.0f, progress_callback_user_data);- }return true;}};@@ -2399,7 +2446,8 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";// K-quants- case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K";+ case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";+ case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";@@ -2409,6 +2457,8 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";case LLAMA_FTYPE_MOSTLY_IQ2_XXS:return "IQ2_XSS - 2.0625 bpw";+ case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";+ case LLAMA_FTYPE_MOSTLY_Q3_K_XS:return "Q3_K - Extra small";default: return "unknown, may not work";}@@ -2624,6 +2674,7 @@ static void llm_load_hparams(ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {+ case 24: model.type = e_model::MODEL_1B; break;case 32: model.type = e_model::MODEL_3B; break;default: model.type = e_model::MODEL_UNKNOWN;}@@ -2638,11 +2689,23 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_QWEN2:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ switch (hparams.n_layer) {+ case 24: model.type = hparams.n_embd == 1024 ? e_model::MODEL_0_5B : e_model::MODEL_1B; break;+ case 32: model.type = e_model::MODEL_7B; break;+ case 40: model.type = hparams.n_head == 20 ? e_model::MODEL_4B : e_model::MODEL_13B; break;+ case 80: model.type = e_model::MODEL_70B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;case LLM_ARCH_PHI2:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);switch (hparams.n_layer) {+ case 24: model.type = e_model::MODEL_1B; break;case 32: model.type = e_model::MODEL_3B; break;default: model.type = e_model::MODEL_UNKNOWN;}@@ -2667,6 +2730,14 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_CODESHELL:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ switch (hparams.n_layer) {+ case 42: model.type = e_model::MODEL_SMALL; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}@@ -2959,7 +3030,15 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown");LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());- LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9);+ if (ml.n_elements >= 1e12) {+ LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, ml.n_elements*1e-12);+ } else if (ml.n_elements >= 1e9) {+ LLAMA_LOG_INFO("%s: model params = %.2f B\n", __func__, ml.n_elements*1e-9);+ } else if (ml.n_elements >= 1e6) {+ LLAMA_LOG_INFO("%s: model params = %.2f M\n", __func__, ml.n_elements*1e-6);+ } else {+ LLAMA_LOG_INFO("%s: model params = %.2f K\n", __func__, ml.n_elements*1e-3);+ }if (ml.n_bytes < GiB) {LLAMA_LOG_INFO("%s: model size = %.2f MiB (%.2f BPW) \n", __func__, ml.n_bytes/1024.0/1024.0, ml.n_bytes*8.0/ml.n_elements);} else {@@ -2983,6 +3062,7 @@ static bool llm_load_tensors(llama_model_loader & ml,llama_model & model,int n_gpu_layers,+ enum llama_split_mode split_mode,int main_gpu,const float * tensor_split,bool use_mlock,@@ -2990,702 +3070,652 @@ static bool llm_load_tensors(void * progress_callback_user_data) {model.t_start_us = ggml_time_us();- auto & ctx = model.ctx;auto & hparams = model.hparams;+ model.split_mode = split_mode;+ model.main_gpu = main_gpu;model.n_gpu_layers = n_gpu_layers;- size_t ctx_size = ggml_tensor_overhead() * ml.n_tensors;+ const int64_t n_layer = hparams.n_layer;+ const int64_t i_gpu_start = std::max((int64_t) hparams.n_layer - n_gpu_layers, (int64_t) 0);- LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, ctx_size/1024.0/1024.0);+ // there is very little benefit to offloading the input layer, so always keep it on the CPU+ model.buft_input = llama_default_buffer_type_cpu(true);- // create the ggml context- {+ model.buft_layer.resize(n_layer);++ // assign cpu layers+ for (int64_t i = 0; i < i_gpu_start; ++i) {+ model.buft_layer[i] = llama_default_buffer_type_cpu(true);+ }++ if (split_mode == LLAMA_SPLIT_LAYER && llamafile_has_cuda()) {+ // calculate the split points+ int device_count = ggml_backend_cuda_get_device_count();+ bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; });+ float splits[GGML_CUDA_MAX_DEVICES];+ if (all_zero) {+ // default split, by free memory+ for (int i = 0; i < device_count; ++i) {+ size_t total;+ size_t free;+ ggml_backend_cuda_get_device_memory(i, &total, &free);+ splits[i] = free;+ }+ } else {+ std::copy(tensor_split, tensor_split + device_count, splits);+ }++ // sum and normalize the splits to get the split points+ float split_sum = 0.0f;+ for (int i = 0; i < device_count; ++i) {+ split_sum += splits[i];+ splits[i] = split_sum;+ }+ for (int i = 0; i < device_count; ++i) {+ splits[i] /= split_sum;+ }++ // assign the repeating layers to the devices according to the splits+ int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1);+ for (int64_t i = i_gpu_start; i < n_layer; ++i) {+ int layer_gpu = std::upper_bound(splits, splits + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits;+ model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu);+ }+ // assign the output layer+ if (n_gpu_layers > n_layer) {+ int layer_gpu = std::upper_bound(splits, splits + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits;+ model.buft_output = llama_default_buffer_type_offload(layer_gpu);+ } else {+ model.buft_output = llama_default_buffer_type_cpu(true);+ }+ } else {+ ggml_backend_buffer_type_t split_buft;+ if (split_mode == LLAMA_SPLIT_ROW) {+ split_buft = llama_default_buffer_type_split(main_gpu, tensor_split);+ } else {+ // LLAMA_SPLIT_NONE or LLAMA_SPLIT_LAYER in backends where it is not supported+ split_buft = llama_default_buffer_type_offload(main_gpu);+ }+ // assign the repeating layers+ for (int64_t i = i_gpu_start; i < n_layer; ++i) {+ model.buft_layer[i] = {+ split_buft,+ llama_default_buffer_type_offload(main_gpu)+ };+ }+ // assign the output layer+ if (n_gpu_layers > n_layer) {+ model.buft_output = {+ split_buft,+ llama_default_buffer_type_offload(main_gpu)+ };+ } else {+ model.buft_output = llama_default_buffer_type_cpu(true);+ }+ }++ // count used buffer types+ std::map<ggml_backend_buffer_type_t, int> buft_layer_count;+ buft_layer_count[model.buft_input.buft]++;+ buft_layer_count[model.buft_input.buft_matrix]++;+ buft_layer_count[model.buft_output.buft]++;+ buft_layer_count[model.buft_output.buft_matrix]++;+ for (int64_t i = 0; i < n_layer; ++i) {+ buft_layer_count[model.buft_layer[i].buft]++;+ buft_layer_count[model.buft_layer[i].buft_matrix]++;+ }++ // create one context per buffer type+ size_t ctx_size = ggml_tensor_overhead()*ml.n_tensors;+ std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;+ for (auto & it : buft_layer_count) {struct ggml_init_params params = {/*.mem_size =*/ ctx_size,/*.mem_buffer =*/ NULL,/*.no_alloc =*/ true,};-- model.ctx = ggml_init(params);- if (!model.ctx) {- throw std::runtime_error(format("ggml_init() failed"));+ ggml_context * ctx = ggml_init(params);+ if (!ctx) {+ throw std::runtime_error(format("failed to create context"));}+ ctx_map[it.first] = ctx;+ model.ctxs.push_back(ctx);}- (void) main_gpu;-- enum ggml_backend_type llama_backend_offload = GGML_BACKEND_CPU;- enum ggml_backend_type llama_backend_offload_split = GGML_BACKEND_CPU;--#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {- LLAMA_LOG_INFO("%s: using " GGML_CUDA_NAME " for GPU acceleration\n", __func__);- ggml_cuda_set_main_device(main_gpu);-- llama_backend_offload = GGML_BACKEND_GPU;- llama_backend_offload_split = GGML_BACKEND_GPU_SPLIT;- }-#elif defined(GGML_USE_CLBLAST)- LLAMA_LOG_INFO("%s: using OpenCL for GPU acceleration\n", __func__);- llama_backend_offload = GGML_BACKEND_GPU;- llama_backend_offload_split = GGML_BACKEND_GPU;-#endif+ LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, model.ctxs.size()*ctx_size/1024.0/1024.0);// create tensors for the weights{const int64_t n_embd = hparams.n_embd;const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();- const int64_t n_layer = hparams.n_layer;+ const int64_t n_embd_gqa = n_embd_v_gqa;const int64_t n_vocab = hparams.n_vocab;+ const int64_t n_ff = hparams.n_ff;++ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);++ ggml_context * ctx_input = ctx_map.at(model.buft_input.buft);+ ggml_context * ctx_output = ctx_map.at(model.buft_output.buft);+ ggml_context * ctx_output_split = ctx_map.at(model.buft_output.buft_matrix);+ auto ctx_for_layer = [&](int i) { return ctx_map.at(model.buft_layer[i].buft); };+ auto ctx_for_layer_split = [&](int i) { return ctx_map.at(model.buft_layer[i].buft_matrix); };++ model.layers.resize(n_layer);const auto tn = LLM_TN(model.arch);switch (model.arch) {case LLM_ARCH_LLAMA:case LLM_ARCH_REFACT:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split);- layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});// optional bias tensors- layer.bq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, backend, false);- layer.bk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, backend, false);- layer.bv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, backend, false);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend, false);+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, false);+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, false);+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, false);+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, false);- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});- layer.ffn_gate_inp = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}, backend, false);+ layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd}, false);if (layer.ffn_gate_inp == nullptr) {GGML_ASSERT(hparams.n_expert == 0);GGML_ASSERT(hparams.n_expert_used == 0);- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});} else {GGML_ASSERT(hparams.n_expert > 0);GGML_ASSERT(hparams.n_expert_used > 0);// MoE branchfor (uint32_t x = 0; x < hparams.n_expert; ++x) {- layer.ffn_gate_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff}, backend_split);- layer.ffn_down_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd}, backend_split);- layer.ffn_up_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}, backend_split);+ layer.ffn_gate_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXP, "weight", i, x), {n_embd, n_ff});+ layer.ffn_down_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXP, "weight", i, x), { n_ff, n_embd});+ layer.ffn_up_exp[x] = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff});}}}} break;case LLM_ARCH_BAICHUAN:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split);- layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});}} break;case LLM_ARCH_FALCON:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_OUTPUT, "weight").c_str()) >= 0) {+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});} else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU+ ml.n_created--; // artificial tensor}-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i).c_str()) >= 0) {- layer.attn_norm_2 = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, backend);- layer.attn_norm_2_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, backend);+ layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd});+ layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd});}- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});}} break;case LLM_ARCH_STARCODER:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);- model.pos_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);- layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;case LLM_ARCH_PERSIMMON:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- const int i_gpu_start = n_layer - n_gpu_layers;- model.layers.resize(n_layer);- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload;- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split;auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);- layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);- layer.attn_q_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64}, backend);- layer.attn_q_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64}, backend);- layer.attn_k_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64}, backend);- layer.attn_k_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64}, backend);++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});++ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});++ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});++ layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64});+ layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64});++ layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64});+ layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64});}} break;case LLM_ARCH_BLOOM:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);- model.tok_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, GGML_BACKEND_CPU);- model.tok_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ model.tok_norm = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd});+ model.tok_norm_b = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);- layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;case LLM_ARCH_MPT:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});// AWQ ScaleActivation layer- layer.ffn_act = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, backend, false);+ layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, false);}} break;case LLM_ARCH_STABLELM:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- /*- llama_model_loader: - tensor 4: blk.0.attn_output.weight f16 [ 2560, 2560, 1, 1 ]- */- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split);- layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);+ // optional bias tensors, present in Stable LM 2 1.6B+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, false);+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, false);+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, false);- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});++ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});}} break;case LLM_ARCH_QWEN:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);- }+ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- const uint32_t n_ff = hparams.n_ff / 2;+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd*3});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd*3});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- const int i_gpu_start = n_layer - n_gpu_layers;+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});- model.layers.resize(n_layer);+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff/2});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff/2, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff/2});+ }+ } break;+ case LLM_ARCH_QWEN2:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd * 3}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd * 3}, backend);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ // optional bias tensors+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});}} break;case LLM_ARCH_PHI2:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);- model.output_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ model.output_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- const int i_gpu_start = n_layer - n_gpu_layers;+ auto & layer = model.layers[i];- model.layers.resize(n_layer);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, false);+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, false);- auto & layer = model.layers[i];+ if (layer.wqkv == nullptr) {+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd});- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa});- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa});+ }- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);- layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;case LLM_ARCH_PLAMO:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;-- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }-- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});}- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);-- const int i_gpu_start = n_layer - n_gpu_layers;-- model.layers.resize(n_layer);-- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});- layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split);- layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split);- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});- layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});}} break;case LLM_ARCH_GPT2:{- model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);- model.pos_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU);+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});+ model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train});// output{- ggml_backend_type backend_norm;- ggml_backend_type backend_output;+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }- if (n_gpu_layers > int(n_layer)) {- backend_norm = llama_backend_offload;- backend_output = llama_backend_offload_split;- } else {- backend_norm = GGML_BACKEND_CPU;- backend_output = GGML_BACKEND_CPU;- }+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);- model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);- model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);- model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);- }+ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- const uint32_t n_ff = hparams.n_ff;- const int64_t n_embd_gqa = n_embd_v_gqa;- GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());- GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});- const int i_gpu_start = n_layer - n_gpu_layers;+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});- model.layers.resize(n_layer);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});++ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});+ }+ } break;+ case LLM_ARCH_CODESHELL:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }- for (uint32_t i = 0; i < n_layer; ++i) {- const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT- const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT+ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);auto & layer = model.layers[i];- layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);- layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});- layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);- layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);+ layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa});+ layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa});- layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);- layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd});- layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);- layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});- layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);- layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});- layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff});}} break;default:@@ -3695,82 +3725,51 @@ static bool llm_load_tensors(ml.done_getting_tensors();- ml.init_mapping();+ ml.init_mapping(true, use_mlock ? &model.mlock_mmap : nullptr);- // allocate tensors- size_t vram_weights = 0;- size_t buf_size = 0;+ // create the backend buffers+ std::vector<std::pair<ggml_context *, ggml_backend_buffer_t>> ctx_bufs;- ggml_backend_buffer_type_t buft = llama_default_buffer_type(n_gpu_layers);+ for (auto & it : ctx_map) {+ ggml_backend_buffer_type_t buft = it.first;+ ggml_context * ctx = it.second;+ ggml_backend_buffer_t buf = nullptr;- for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {- // GGML_BACKEND_GPU tensors are for CUDA and OpenCL only, which are handled separately without ggml-backend- if (t->backend == GGML_BACKEND_CPU) {- buf_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), ggml_backend_buft_get_alignment(buft));- } else {- vram_weights += ggml_nbytes(t);+ // only the mmap region containing the tensors in the model is mapped to the backend buffer+ // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer, then we could just use metal for all layers+ // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size+ if (ml.use_mmap && buft == llama_default_buffer_type_cpu(true)) {+ size_t first, last;+ ml.get_mapping_range(&first, &last, ctx);+ buf = ggml_backend_cpu_buffer_from_ptr((char *) ml.mapping->addr + first, last - first);}- }-- // create backend buffer- ggml_backend_buffer_t buf_mmap = nullptr;-- if (ggml_metal_supported()) {- if (n_gpu_layers > 0) {- if (ml.use_mmap) {+ else if (ml.use_mmap && buft == ggml_backend_metal_buffer_type()) {const size_t max_size = ggml_get_max_tensor_size(ctx);- model.buf = ggml_backend_metal_buffer_from_ptr(ml.mapping->addr, ml.mapping->size, max_size);- buf_mmap = model.buf;- } else {- model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_metal_buffer_type());+ size_t first, last;+ ml.get_mapping_range(&first, &last, ctx);+ buf = ggml_backend_metal_buffer_from_ptr((char *) ml.mapping->addr + first, last - first, max_size);}- }- }-#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {- // for testing only- if (n_gpu_layers > 0) {- model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_cuda_buffer_type(0));- }- }-#endif-- if (model.buf == nullptr) {- // CPU backend, and indirectly CUDA and OpenCL- if (ml.use_mmap) {- model.buf = ggml_backend_cpu_buffer_from_ptr(ml.mapping->addr, ml.mapping->size);- buf_mmap = model.buf;- } else {- // allocate only CPU tensors- model.buf = ggml_backend_buft_alloc_buffer(buft, buf_size);- ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(model.buf);- for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {- if (t->backend == GGML_BACKEND_CPU) {- ggml_tallocr_alloc(alloc, t);- }+ else {+ buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);+ if (buf != nullptr && use_mlock && ggml_backend_buffer_is_host(buf)) {+ model.mlock_bufs.emplace_back(new llama_mlock);+ auto & mlock_buf = model.mlock_bufs.back();+ mlock_buf->init (ggml_backend_buffer_get_base(buf));+ mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));}- ggml_tallocr_free(alloc);}- }-- if (use_mlock && ggml_backend_buffer_is_host(model.buf)) {- model.mlock_buf.init (ggml_backend_buffer_get_base(model.buf));- model.mlock_buf.grow_to(ggml_backend_buffer_get_size(model.buf));+ if (buf == nullptr) {+ throw std::runtime_error("failed to allocate buffer");+ }+ // indicate that this buffer contains weights+ // this is used by ggml_backend_sched to improve op scheduling -> ops that use a weight are preferably scheduled to the backend that contains the weight+ ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);+ model.bufs.push_back(buf);+ ctx_bufs.emplace_back(ctx, buf);}// print memory requirements{- size_t sys_mem_required = ctx_size + buf_size;-- if (sys_mem_required > 0) {- LLAMA_LOG_INFO("%s: system memory used = %7.2f MiB\n", __func__, sys_mem_required / 1024.0 / 1024.0);- }- if (vram_weights > 0) {- LLAMA_LOG_INFO("%s: VRAM used = %7.2f MiB\n", __func__, vram_weights / 1024.0 / 1024.0);- }--#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu);@@ -3782,24 +3781,26 @@ static bool llm_load_tensors(const int max_offloadable_layers = hparams.n_layer + 1;LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);++ for (ggml_backend_buffer_t buf : model.bufs) {+ LLAMA_LOG_INFO("%s: %10s buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);}-#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)}-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- ggml_cuda_set_tensor_split(tensor_split);-#else- GGML_UNUSED(tensor_split);-#endif // GGML_USE_CUBLAS-// populate tensors_by_name- for (int i = 0; i < ml.n_tensors; ++i) {- struct ggml_tensor * cur = ggml_get_tensor(ctx, ml.get_tensor_name(i));- model.tensors_by_name.emplace_back(ggml_get_name(cur), cur);+ for (ggml_context * ctx : model.ctxs) {+ for (auto * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {+ model.tensors_by_name.emplace_back(ggml_get_name(cur), cur);+ }}- if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf_mmap, use_mlock ? &model.mlock_mmap : NULL)) {- return false;+ // load tensor data+ for (auto & it : ctx_bufs) {+ ggml_context * ctx = it.first;+ ggml_backend_buffer_t buf = it.second;+ if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf, use_mlock ? &model.mlock_mmap : NULL)) {+ return false;+ }}model.mapping = std::move(ml.mapping);@@ -3833,13 +3834,13 @@ static int llama_model_load(const std::string & fname, llama_model & model, cons}if (!llm_load_tensors(- ml, model, params.n_gpu_layers, params.main_gpu, params.tensor_split, params.use_mlock,+ ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock,params.progress_callback, params.progress_callback_user_data)) {return -2;}} catch (const std::exception & err) {- LLAMA_LOG_ERROR("error loading model: %s\n", err.what());+ LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());return -1;}@@ -3880,22 +3881,24 @@ static struct ggml_tensor * llm_build_inp_embd(const llama_hparams & hparams,const llama_batch & batch,struct ggml_tensor * tok_embd,+ struct ggml_tensor * inp_tokens,+ struct ggml_tensor * inp_embd,const llm_build_cb & cb) {const int64_t n_embd = hparams.n_embd;struct ggml_tensor * inpL;if (batch.token) {- struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, batch.n_tokens);+ struct ggml_tensor * inp_tokens_v = ggml_view_1d(ctx, inp_tokens, batch.n_tokens, 0);cb(inp_tokens, "inp_tokens", -1);- inpL = ggml_get_rows(ctx, tok_embd, inp_tokens);+ inpL = ggml_get_rows(ctx, tok_embd, inp_tokens_v);} else {#ifdef GGML_USE_MPIGGML_ASSERT(false && "not implemented");#endif- inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens);+ inpL = ggml_view_2d(ctx, inp_embd, n_embd, batch.n_tokens, inp_embd->nb[1], 0);}return inpL;@@ -3909,9 +3912,9 @@ static void llm_build_k_shift(const llama_cparams & cparams,const llama_kv_cache & kv,struct ggml_cgraph * graph,+ struct ggml_tensor * K_shift,llm_rope_type type,int64_t n_ctx,- int n_rot,float freq_base,float freq_scale,const llm_build_cb & cb) {@@ -3919,17 +3922,13 @@ static void llm_build_k_shift(const int64_t n_head_kv = hparams.n_head_kv;const int64_t n_embd_head_k = hparams.n_embd_head_k;const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int32_t n_rot = hparams.n_rot;const int32_t n_orig_ctx = cparams.n_yarn_orig_ctx;const float ext_factor = cparams.yarn_ext_factor;const float attn_factor = cparams.yarn_attn_factor;const float beta_fast = cparams.yarn_beta_fast;const float beta_slow = cparams.yarn_beta_slow;- GGML_ASSERT(n_embd_head_k % n_rot == 0);-- struct ggml_tensor * K_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_ctx);- cb(K_shift, "K_shift", -1);-int rope_type = 0;switch (type) {@@ -4117,6 +4116,7 @@ static struct ggml_tensor * llm_build_kqv(const llama_model & model,const llama_hparams & hparams,const llama_kv_cache & kv,+ struct ggml_cgraph * graph,struct ggml_tensor * wo,struct ggml_tensor * wo_b,struct ggml_tensor * q_cur,@@ -4195,6 +4195,8 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);cb(cur, "kqv_merged_cont", il);+ ggml_build_forward_expand(graph, cur);+cur = ggml_mul_mat(ctx, wo, cur);if (wo_b) {cb(cur, "kqv_wo", il);@@ -4207,8 +4209,47 @@ static struct ggml_tensor * llm_build_kqv(return cur;}+static struct ggml_tensor * llm_build_kv(+ struct ggml_context * ctx,+ const llama_model & model,+ const llama_hparams & hparams,+ const llama_kv_cache & kv,+ struct ggml_cgraph * graph,+ struct ggml_tensor * wo,+ struct ggml_tensor * wo_b,+ struct ggml_tensor * k_cur,+ struct ggml_tensor * v_cur,+ struct ggml_tensor * q_cur,+ struct ggml_tensor * kq_mask,+ int64_t n_ctx,+ int32_t n_tokens,+ int32_t kv_head,+ int32_t n_kv,+ float max_alibi_bias,+ float kq_scale,+ const llm_build_cb & cb,+ int il) {++ // these nodes are added to the graph together so that they are not reordered+ // by doing so, the number of splits in the graph is reduced+ ggml_build_forward_expand(graph, q_cur);+ ggml_build_forward_expand(graph, k_cur);+ ggml_build_forward_expand(graph, v_cur);++ llm_build_kv_store(ctx, hparams, kv, graph, k_cur, v_cur, n_ctx, n_tokens, kv_head, cb, il);++ struct ggml_tensor * cur;+ cur = llm_build_kqv(ctx, model, hparams, kv, graph,+ wo, wo_b,+ q_cur, kq_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, kq_scale, cb, il);+ cb(cur, "kqv_out", il);++ return cur;+}+struct llm_build_context {const llama_model & model;+ const llama_context & lctx;const llama_hparams & hparams;const llama_cparams & cparams;const llama_batch & batch;@@ -4255,6 +4296,7 @@ struct llm_build_context {const llm_build_cb & cb,bool worst_case) :model (lctx.model),+ lctx (lctx),hparams (model.hparams),cparams (lctx.cparams),batch (batch),@@ -4285,8 +4327,6 @@ struct llm_build_context {do_rope_shift (worst_case || kv_self.has_shift),cb (cb),buf_compute_meta (lctx.buf_compute_meta) {- GGML_ASSERT(!!kv_self.ctx);-// all initializations should be done in init()}@@ -4317,20 +4357,20 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -4368,23 +4408,21 @@ struct llm_build_context {Qcur = ggml_rope_custom(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,- n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);Kcur = ggml_rope_custom(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,- n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4498,24 +4536,25 @@ struct llm_build_context {const int64_t n_embd_head = hparams.n_embd_head_v;GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -4541,12 +4580,12 @@ struct llm_build_context {case MODEL_7B:Qcur = ggml_rope_custom(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,- n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);Kcur = ggml_rope_custom(ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,- n_embd_head, 0, 0, n_orig_ctx, freq_base, freq_scale,+ hparams.n_rot, 0, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);break;@@ -4560,14 +4599,13 @@ struct llm_build_context {cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);// apply ALiBi for 13B modelconst float max_alibi_bias = model.type == MODEL_13B ? 8.0f : -1.0f;- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4619,24 +4657,25 @@ struct llm_build_context {const int64_t n_embd_head = hparams.n_embd_head_v;const int64_t n_embd_gqa = hparams.n_embd_v_gqa();GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -4677,22 +4716,20 @@ struct llm_build_context {// using mode = 2 for neox modeQcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4747,15 +4784,15 @@ struct llm_build_context {struct ggml_tensor * pos;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);@@ -4789,11 +4826,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4840,26 +4875,25 @@ struct llm_build_context {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);const int64_t n_embd_head = hparams.n_embd_head_v;- GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);-- const int64_t n_rot = n_embd_head_k / 2;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head/2 == hparams.n_rot);struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);- cb(inpL, "imp_embd", -1);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);if (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -4919,7 +4953,7 @@ struct llm_build_context {// RoPE the first n_rot of q/k, pass the other half, and concat.struct ggml_tensor * qrot = ggml_view_3d(- ctx0, tmpq, n_rot, n_head, n_tokens,+ ctx0, tmpq, hparams.n_rot, n_head, n_tokens,ggml_element_size(tmpq) * n_embd_head,ggml_element_size(tmpq) * n_embd_head * n_head,0@@ -4927,7 +4961,7 @@ struct llm_build_context {cb(qrot, "qrot", il);struct ggml_tensor * krot = ggml_view_3d(- ctx0, tmpk, n_rot, n_head, n_tokens,+ ctx0, tmpk, hparams.n_rot, n_head, n_tokens,ggml_element_size(tmpk) * n_embd_head,ggml_element_size(tmpk) * n_embd_head * n_head,0@@ -4936,29 +4970,29 @@ struct llm_build_context {// get the second half of tmpq, e.g tmpq[n_rot:, :, :]struct ggml_tensor * qpass = ggml_view_3d(- ctx0, tmpq, n_rot, n_head, n_tokens,+ ctx0, tmpq, hparams.n_rot, n_head, n_tokens,ggml_element_size(tmpq) * n_embd_head,ggml_element_size(tmpq) * n_embd_head * n_head,- ggml_element_size(tmpq) * n_rot+ ggml_element_size(tmpq) * hparams.n_rot);cb(qpass, "qpass", il);struct ggml_tensor * kpass = ggml_view_3d(- ctx0, tmpk, n_rot, n_head, n_tokens,+ ctx0, tmpk, hparams.n_rot, n_head, n_tokens,ggml_element_size(tmpk) * n_embd_head,ggml_element_size(tmpk) * n_embd_head * n_head,- ggml_element_size(tmpk) * n_rot+ ggml_element_size(tmpk) * hparams.n_rot);cb(kpass, "kpass", il);struct ggml_tensor * qrotated = ggml_rope_custom(- ctx0, qrot, inp_pos, n_rot, 2, 0, n_orig_ctx,+ ctx0, qrot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(qrotated, "qrotated", il);struct ggml_tensor * krotated = ggml_rope_custom(- ctx0, krot, inp_pos, n_rot, 2, 0, n_orig_ctx,+ ctx0, krot, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(krotated, "krotated", il);@@ -4997,12 +5031,9 @@ struct llm_build_context {);cb(Vcur, "Vcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- // TODO: not tested, could be broken- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Q, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Q, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5057,11 +5088,11 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);for (int il = 0; il < n_layer; ++il) {@@ -5089,11 +5120,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);cb(Qcur, "Qcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5149,11 +5178,11 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);inpL = llm_build_norm(ctx0, inpL, hparams,@@ -5187,11 +5216,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5244,11 +5271,11 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);for (int il = 0; il < n_layer; ++il) {@@ -5282,11 +5309,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5342,20 +5367,20 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, hparams.n_rot, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -5373,12 +5398,24 @@ struct llm_build_context {// compute Q and K and RoPE themstruct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }Qcur = ggml_rope_custom(ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,@@ -5394,11 +5431,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5455,20 +5490,20 @@ struct llm_build_context {struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -5500,22 +5535,20 @@ struct llm_build_context {// using mode = 2 for neox modeQcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, n_embd_head, 2, 0, n_orig_ctx,+ ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5560,6 +5593,126 @@ struct llm_build_context {return gf;}++ struct ggml_cgraph * build_qwen2() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);++ // these nodes are added to the graph together so that they are not reordered+ // by doing so, the number of splits in the graph is reduced+ ggml_build_forward_expand(gf, Qcur);+ ggml_build_forward_expand(gf, Kcur);+ ggml_build_forward_expand(gf, Vcur);++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }+struct ggml_cgraph * build_phi2() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);@@ -5572,20 +5725,20 @@ struct llm_build_context {struct ggml_tensor * ffn_output;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE_NEOX, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -5597,15 +5750,25 @@ struct llm_build_context {// self-attention{- cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output);- cb(cur, "wqkv", il);+ struct ggml_tensor * Qcur = nullptr;+ struct ggml_tensor * Kcur = nullptr;+ struct ggml_tensor * Vcur = nullptr;- cur = ggml_add(ctx0, cur, model.layers[il].bqkv);- cb(cur, "bqkv", il);+ if (model.layers[il].wqkv) {+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output);+ cb(cur, "wqkv", il);- struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));- struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));- struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);+ cb(cur, "bqkv", il);++ Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));+ } else {+ Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, attn_norm_output), model.layers[il].bq);+ Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, attn_norm_output), model.layers[il].bk);+ Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, attn_norm_output), model.layers[il].bv);+ }cb(Qcur, "Qcur", il);cb(Kcur, "Kcur", il);@@ -5631,11 +5794,9 @@ struct llm_build_context {);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f, cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f, cb, il);cb(cur, "kqv_out", il);}@@ -5681,24 +5842,25 @@ struct llm_build_context {const int64_t n_embd_head = hparams.n_embd_head_v;GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);struct ggml_tensor * cur;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);// shift the entire K-cache if neededif (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);}for (int il = 0; il < n_layer; ++il) {@@ -5724,22 +5886,20 @@ struct llm_build_context {cb(Vcur, "Vcur", il);Qcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ ctx0, ggml_reshape_3d(ctx0, Qcur, hparams.n_rot, n_head, n_tokens), inp_pos,n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Qcur, "Qcur", il);Kcur = ggml_rope_custom(- ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ ctx0, ggml_reshape_3d(ctx0, Kcur, hparams.n_rot, n_head_kv, n_tokens), inp_pos,n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);cb(Kcur, "Kcur", il);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, NULL,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}struct ggml_tensor * sa_out = cur;@@ -5794,15 +5954,15 @@ struct llm_build_context {struct ggml_tensor * pos;struct ggml_tensor * inpL;- inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);cb(inpL, "inp_embd", -1);// inp_pos - contains the positions- struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);cb(inp_pos, "inp_pos", -1);// KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);cb(KQ_mask, "KQ_mask", -1);pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);@@ -5836,11 +5996,9 @@ struct llm_build_context {Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);-- cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5882,193 +6040,116 @@ struct llm_build_context {return gf;}-};-//-// tensor offloading helpers-//-// TODO: will be removed with backend v2--enum llm_offload_func_e {- OFFLOAD_FUNC_NOP,- OFFLOAD_FUNC,- OFFLOAD_FUNC_FRC, // force offload- OFFLOAD_FUNC_KQV,- OFFLOAD_FUNC_NR,- OFFLOAD_FUNC_EMB, // embeddings- OFFLOAD_FUNC_OUT,-};+ struct ggml_cgraph * build_codeshell() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);-// TODO: will be removed with backend v2-struct llm_offload_trie {- struct node {- ~node() {- for (int i = 0; i < 256; ++i) {- if (children[i]) {- delete children[i];- }- }- }+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);- node * children[256] = { nullptr };- llm_offload_func_e func = OFFLOAD_FUNC_NOP;- };+ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;- llm_offload_trie() {- root = new node;- }+ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, lctx.inp_tokens, lctx.inp_embd, cb);+ cb(inpL, "inp_embd", -1);- llm_offload_trie(const std::unordered_map<const char *, llm_offload_func_e> & map) {- root = new node;+ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_view_1d(ctx0, lctx.inp_pos, n_tokens, 0);+ cb(inp_pos, "inp_pos", -1);- for (const auto & kv : map) {- add(kv.first, kv.second);+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_view_2d(ctx0, lctx.inp_KQ_mask, n_kv, n_tokens, n_kv*ggml_type_size(lctx.inp_KQ_mask->type), 0);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, lctx.inp_K_shift, LLM_ROPE, n_ctx, freq_base, freq_scale, cb);}- }- ~llm_offload_trie() {- delete root;- }+ for (int il = 0; il < n_layer; ++il) {+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm,+ model.layers[il].attn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "attn_norm", il);- void add(const char * name, llm_offload_func_e func) {- node * cur = root;+ // self-attention+ {+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);+ cb(cur, "wqkv", il);- for (int i = 0; ; ++i) {- const uint8_t c = name[i];+ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);+ cb(cur, "bqkv", il);- if (!c) {- break;- }+ struct ggml_tensor * tmpq = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ struct ggml_tensor * tmpk = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));- if (!cur->children[c]) {- cur->children[c] = new node;- }+ cb(tmpq, "tmpq", il);+ cb(tmpk, "tmpk", il);+ cb(Vcur, "Vcur", il);- cur = cur->children[c];- }+ struct ggml_tensor * Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);- cur->func = func;- }+ struct ggml_tensor * Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos,+ hparams.n_rot, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);- llm_offload_func_e find(const char * name) const {- const node * cur = root;+ cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, n_ctx, n_tokens, kv_head, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }- for (int i = 0; ; ++i) {- const uint8_t c = name[i];+ // add the input+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);+ cb(ffn_inp, "ffn_inp", il);- if (!c) {- break;- }+ // FF+ {+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm,+ model.layers[il].ffn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "ffn_norm", il);- if (!cur->children[c]) {- return OFFLOAD_FUNC_NOP;+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b,+ NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);+ cb(cur, "ffn_out", il);}- cur = cur->children[c];+ inpL = ggml_add(ctx0, cur, ffn_inp);+ cb(inpL, "l_out", il);}- return cur->func;- }+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.output_norm,+ model.output_norm_b,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);- node * root = nullptr;-};+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);-// TODO: will be removed with backend v2-static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map = {- //{ "inp_tokens", OFFLOAD_FUNC_NR }, // TODO: missing K-quants get_rows kernel- //{ "inp_embd", OFFLOAD_FUNC_NR }, // TODO: missing K-quants get_rows kernel- { "pos_embd", OFFLOAD_FUNC_NR },-- { "inp_pos", OFFLOAD_FUNC_FRC }, // this is often used for KQ ops (e.g. rope)- { "KQ_mask", OFFLOAD_FUNC_FRC },- { "K_shift", OFFLOAD_FUNC_FRC },-- { "K_shifted", OFFLOAD_FUNC },-- { "inp_norm", OFFLOAD_FUNC_NR },- { "inp_norm_w", OFFLOAD_FUNC_NR },- { "inp_norm_wb", OFFLOAD_FUNC_NR },-- { "norm", OFFLOAD_FUNC },- { "norm_w", OFFLOAD_FUNC },- { "norm_wb", OFFLOAD_FUNC },-- { "attn_norm", OFFLOAD_FUNC },- { "attn_norm_2", OFFLOAD_FUNC },-- { "wqkv", OFFLOAD_FUNC_KQV },- { "bqkv", OFFLOAD_FUNC_KQV },- { "wqkv_clamped", OFFLOAD_FUNC_KQV },-- { "tmpk", OFFLOAD_FUNC_KQV },- { "tmpq", OFFLOAD_FUNC_KQV },- { "tmpv", OFFLOAD_FUNC_KQV },- { "Kcur", OFFLOAD_FUNC_KQV },- { "Qcur", OFFLOAD_FUNC_KQV },- { "Vcur", OFFLOAD_FUNC_KQV },-- { "krot", OFFLOAD_FUNC_KQV },- { "qrot", OFFLOAD_FUNC_KQV },- { "kpass", OFFLOAD_FUNC_KQV },- { "qpass", OFFLOAD_FUNC_KQV },- { "krotated", OFFLOAD_FUNC_KQV },- { "qrotated", OFFLOAD_FUNC_KQV },-- { "q", OFFLOAD_FUNC_KQV },- { "k", OFFLOAD_FUNC_KQV },- { "kq", OFFLOAD_FUNC_KQV },- { "kq_scaled", OFFLOAD_FUNC_KQV },- { "kq_scaled_alibi", OFFLOAD_FUNC_KQV },- { "kq_masked", OFFLOAD_FUNC_KQV },- { "kq_soft_max", OFFLOAD_FUNC_KQV },- { "kq_soft_max_ext", OFFLOAD_FUNC_KQV },- { "v", OFFLOAD_FUNC_KQV },- { "kqv", OFFLOAD_FUNC_KQV },- { "kqv_merged", OFFLOAD_FUNC_KQV },- { "kqv_merged_cont", OFFLOAD_FUNC_KQV },- { "kqv_wo", OFFLOAD_FUNC_KQV },- { "kqv_out", OFFLOAD_FUNC_KQV },-- { "ffn_inp", OFFLOAD_FUNC },- { "ffn_norm", OFFLOAD_FUNC },-- { "ffn_up", OFFLOAD_FUNC },- { "ffn_up_b", OFFLOAD_FUNC },- { "ffn_gate", OFFLOAD_FUNC },- { "ffn_gate_b", OFFLOAD_FUNC },- { "ffn_gate_par", OFFLOAD_FUNC },- { "ffn_act", OFFLOAD_FUNC },- { "ffn_down", OFFLOAD_FUNC },- { "ffn_down_b", OFFLOAD_FUNC },- { "ffn_out", OFFLOAD_FUNC },-- { "ffn_silu", OFFLOAD_FUNC },- { "ffn_gelu", OFFLOAD_FUNC },- { "ffn_relu", OFFLOAD_FUNC },- { "ffn_sqr(relu)", OFFLOAD_FUNC },-- { "ffn_moe_logits", OFFLOAD_FUNC },- { "ffn_moe_probs", OFFLOAD_FUNC },- { "ffn_moe_argsort", OFFLOAD_FUNC },- { "ffn_moe_weights", OFFLOAD_FUNC },- { "ffn_moe_weights_sum", OFFLOAD_FUNC },- { "ffn_moe_weights_norm", OFFLOAD_FUNC },- { "ffn_moe_weighted", OFFLOAD_FUNC },- { "ffn_moe_up", OFFLOAD_FUNC },- { "ffn_moe_gate", OFFLOAD_FUNC },- { "ffn_moe_silu", OFFLOAD_FUNC },- { "ffn_moe_gate_par", OFFLOAD_FUNC },- { "ffn_moe_down", OFFLOAD_FUNC },- { "ffn_moe_out", OFFLOAD_FUNC },-- { "l_out", OFFLOAD_FUNC },-- { "result_norm", OFFLOAD_FUNC_EMB },- { "result_output_no_bias", OFFLOAD_FUNC_EMB },- { "result_output", OFFLOAD_FUNC_OUT },-};+ ggml_build_forward_expand(gf, cur);-static llm_offload_trie k_offload_func_trie(k_offload_map);+ return gf;+ }+};static struct ggml_cgraph * llama_build_graph(llama_context & lctx,@@ -6076,25 +6157,9 @@ static struct ggml_cgraph * llama_build_graph(const auto & model = lctx.model;// check if we should build the worst-case graph (for memory measurement)- const bool worst_case = ggml_allocr_is_measure(lctx.alloc);-- // keep track of the input that has already been allocated- bool alloc_inp_tokens = false;- bool alloc_inp_embd = false;- bool alloc_inp_pos = false;- bool alloc_inp_KQ_mask = false;- bool alloc_inp_K_shift = false;--#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- const bool do_offload = true;-#else- const bool do_offload = true; // TODO: set to false after finishing refactoring-#endif-- int n_non_view = 0; // number of non-view tensors that have been processed by the callback+ const bool worst_case = ggml_tallocr_is_measure(lctx.alloc);// this callback allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.)- // TODO: will be removed with backend v2llm_build_cb cb = [&](struct ggml_tensor * cur, const char * name, int il) {if (il >= 0) {ggml_format_name(cur, "%s-%d", name, il);@@ -6102,249 +6167,78 @@ static struct ggml_cgraph * llama_build_graph(ggml_set_name(cur, name);}- //- // allocate input tensors and set input data- //- // TODO: will be removed with backend v2-- if (!alloc_inp_tokens && strcmp(name, "inp_tokens") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc) && batch.token) {- const int64_t n_tokens = cur->ne[0];-- ggml_backend_tensor_set(cur, batch.token, 0, n_tokens*ggml_element_size(cur));- }-- alloc_inp_tokens = true;- }-- if (!alloc_inp_embd && strcmp(name, "inp_embd") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc) && batch.embd) {- const int64_t n_embd = cur->ne[0];- const int64_t n_tokens = cur->ne[1];-- ggml_backend_tensor_set(cur, batch.embd, 0, n_tokens*n_embd*ggml_element_size(cur));- }-- alloc_inp_embd = true;- }-- if (!alloc_inp_pos && strcmp(name, "inp_pos") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc) && batch.pos) {- const int64_t n_tokens = cur->ne[0];-- static_assert(std::is_same<llama_pos, int32_t>::value, "llama_pos must be int32_t");- ggml_backend_tensor_set(cur, batch.pos, 0, n_tokens*ggml_element_size(cur));+ if (!lctx.cparams.offload_kqv) {+ if (strcmp(name, "kqv_merged_cont") == 0) {+ // all nodes between the KV store and the attention output are run on the CPU+ ggml_backend_sched_set_node_backend(lctx.sched, cur, lctx.backend_cpu);}-- alloc_inp_pos = true;}+ };- if (!alloc_inp_KQ_mask && strcmp(name, "KQ_mask") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc)) {- const int64_t n_kv = cur->ne[0];- const int64_t n_tokens = cur->ne[1];-- float * data;- if (ggml_backend_buffer_is_host(cur->buffer)) {- data = (float *) cur->data;- } else {- lctx.buf_copy.resize(ggml_nbytes(cur));- data = (float *) lctx.buf_copy.data();- }-- for (int h = 0; h < 1; ++h) {- for (int j = 0; j < n_tokens; ++j) {- const llama_pos pos = batch.pos[j];- const llama_seq_id seq_id = batch.seq_id[j][0];-- for (int i = 0; i < n_kv; ++i) {- float f;- if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {- f = -INFINITY;- } else {- f = 0;- }- data[h*(n_kv*n_tokens) + j*n_kv + i] = f;- }- }- }-- if (data != cur->data) {- ggml_backend_tensor_set(cur, data, 0, ggml_nbytes(cur));- }- }-- alloc_inp_KQ_mask = true;- }-- if (!alloc_inp_K_shift && strcmp(name, "K_shift") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc)) {- const int64_t n_ctx = cur->ne[0];+ struct ggml_cgraph * result = NULL;- int32_t * data;- if (ggml_backend_buffer_is_host(cur->buffer)) {- data = (int32_t *) cur->data;- } else {- lctx.buf_copy.resize(ggml_nbytes(cur));- data = (int32_t *) lctx.buf_copy.data();- }+ struct llm_build_context llm(lctx, batch, cb, worst_case);- for (int i = 0; i < n_ctx; ++i) {- data[i] = lctx.kv_self.cells[i].delta;- }+ //+ // set input data+ //- if (data != cur->data) {- ggml_backend_tensor_set(cur, data, 0, ggml_nbytes(cur));- }- }+ if (!ggml_tallocr_is_measure(lctx.alloc)) {+ if (batch.token) {+ const int64_t n_tokens = batch.n_tokens;- alloc_inp_K_shift = true;+ ggml_backend_tensor_set(lctx.inp_tokens, batch.token, 0, n_tokens*ggml_element_size(lctx.inp_tokens));}- // view tensors are not processed further- if (cur->view_src != nullptr) {- return;- }+ if (batch.embd) {+ const int64_t n_embd = llm.n_embd;+ const int64_t n_tokens = batch.n_tokens;- if (cur->op != GGML_OP_NONE) {- n_non_view++;+ ggml_backend_tensor_set(lctx.inp_embd, batch.embd, 0, n_tokens*n_embd*ggml_element_size(lctx.inp_embd));}- //- // offload layers- //- // TODO: will be removed with backend v2--//#define LLAMA_OFFLOAD_DEBUG+ if (batch.pos) {+ const int64_t n_tokens = batch.n_tokens;- if (!do_offload) {- return;+ ggml_backend_tensor_set(lctx.inp_pos, batch.pos, 0, n_tokens*ggml_element_size(lctx.inp_pos));}- const int n_layer = model.hparams.n_layer;-- const int n_gpu_layers = model.n_gpu_layers;- const int i_gpu_start = n_layer - n_gpu_layers;-- // should we offload the final norm? yes if we are not computing embeddings- const bool offload_emb = lctx.embedding.empty();-- static const std::unordered_map<llm_offload_func_e, std::string, std::hash<int>> k_offload_func_name = {- { OFFLOAD_FUNC_NOP, "CPU" },- { OFFLOAD_FUNC_OUT, "CPU" },-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- { OFFLOAD_FUNC, "GPU (CUDA)" },- { OFFLOAD_FUNC_FRC, "GPU (CUDA) FRC" },- { OFFLOAD_FUNC_KQV, "GPU (CUDA) KQV" },- { OFFLOAD_FUNC_NR, "GPU (CUDA) NR" },- { OFFLOAD_FUNC_EMB, "GPU (CUDA) EMB" },-#else- { OFFLOAD_FUNC, "CPU" },- { OFFLOAD_FUNC_FRC, "CPU" },- { OFFLOAD_FUNC_KQV, "CPU" },- { OFFLOAD_FUNC_NR, "CPU" },- { OFFLOAD_FUNC_EMB, "CPU" },-#endif // GGML_USE_CUBLAS- };+ {+ const int64_t n_kv = llm.n_kv;+ const int64_t n_tokens = batch.n_tokens;- // check the global map for what offload function to use for this tensor- llm_offload_func_e func_e = k_offload_func_trie.find(name);+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer));+ float * data = (float *) lctx.inp_KQ_mask->data;- if (func_e == OFFLOAD_FUNC_NOP) {-#ifdef LLAMA_OFFLOAD_DEBUG- // if a tensor hasn't been offloaded, we warn the user- if (worst_case) {- LLAMA_LOG_WARN("%s: %32s: not offloaded (ref: %s)\n", __func__,- cur->name, "https://github.com/ggerganov/llama.cpp/pull/3837");- }-#endif+ for (int h = 0; h < 1; ++h) {+ for (int j = 0; j < n_tokens; ++j) {+ const llama_pos pos = batch.pos[j];+ const llama_seq_id seq_id = batch.seq_id[j][0];- return;- }-- // count the number of layers and respect the provided n_gpu_layers- switch (func_e) {- case OFFLOAD_FUNC_NOP:- case OFFLOAD_FUNC_OUT:- break;- case OFFLOAD_FUNC:- if (n_gpu_layers < n_layer) {- if (il < i_gpu_start) {- func_e = OFFLOAD_FUNC_NOP;- }- }- break;- case OFFLOAD_FUNC_FRC:- if (!lctx.cparams.offload_kqv) {- func_e = OFFLOAD_FUNC_NOP;- } break;- case OFFLOAD_FUNC_KQV:- if (!lctx.cparams.offload_kqv) {- func_e = OFFLOAD_FUNC_NOP;- } else {- if (n_gpu_layers < n_layer) {- if (il < i_gpu_start) {- func_e = OFFLOAD_FUNC_NOP;+ for (int i = 0; i < n_kv; ++i) {+ float f;+ if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {+ f = -INFINITY;+ } else {+ f = 0;}+ data[h*(n_kv*n_tokens) + j*n_kv + i] = f;}}- break;- case OFFLOAD_FUNC_NR:- if (n_gpu_layers <= n_layer + 0) {- func_e = OFFLOAD_FUNC_NOP;- }- break;- case OFFLOAD_FUNC_EMB:- if (!offload_emb || n_gpu_layers < n_layer) {- func_e = OFFLOAD_FUNC_NOP;- }- break;- default: GGML_ASSERT(false);+ }}- offload_func_t func = ggml_offload_nop;-- // this is needed for compatibility with Metal for example-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- static offload_func_t ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc;-#else- static offload_func_t ggml_offload_gpu = ggml_offload_nop;-#endif-- switch (func_e) {- case OFFLOAD_FUNC_NOP:- case OFFLOAD_FUNC_OUT: func = ggml_offload_nop; break;- case OFFLOAD_FUNC:- case OFFLOAD_FUNC_KQV:- case OFFLOAD_FUNC_FRC:- case OFFLOAD_FUNC_NR:- case OFFLOAD_FUNC_EMB: func = ggml_offload_gpu; break;- default: GGML_ASSERT(false);- }+ if (llm.do_rope_shift) {+ const int64_t n_ctx = llm.n_ctx;- // apply offload function to the tensor- func(cur);+ GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_K_shift->buffer));+ int32_t * data = (int32_t *) lctx.inp_K_shift->data;-#ifdef LLAMA_OFFLOAD_DEBUG- if (worst_case) {- LLAMA_LOG_INFO("%s: %32s: %s\n", __func__, cur->name, k_offload_func_name.at(func_e).c_str());+ for (int i = 0; i < n_ctx; ++i) {+ data[i] = lctx.kv_self.cells[i].delta;+ }}-#endif- };-- struct ggml_cgraph * result = NULL;-- struct llm_build_context llm(lctx, batch, cb, worst_case);+ }llm.init();@@ -6389,6 +6283,10 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_qwen();} break;+ case LLM_ARCH_QWEN2:+ {+ result = llm.build_qwen2();+ } break;case LLM_ARCH_PHI2:{result = llm.build_phi2();@@ -6401,33 +6299,16 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_gpt2();} break;+ case LLM_ARCH_CODESHELL:+ {+ result = llm.build_codeshell();+ } break;default:GGML_ASSERT(false);}llm.free();- if (worst_case) {- int n_non_view_total = 0;-- for (int i = 0; i < result->n_nodes; ++i) {- if (result->nodes[i]->view_src == nullptr) {- n_non_view_total++;- }- }-- LLAMA_LOG_INFO("%s: non-view tensors processed: %d/%d\n", __func__, n_non_view, n_non_view_total);-- if (n_non_view != n_non_view_total) {- LLAMA_LOG_WARN("%s: ****************************************************************\n", __func__);- LLAMA_LOG_WARN("%s: not all non-view tensors have been processed with a callback\n", __func__);- LLAMA_LOG_WARN("%s: this can indicate an inefficiency in the graph implementation\n", __func__);- LLAMA_LOG_WARN("%s: build with LLAMA_OFFLOAD_DEBUG for more info\n", __func__);- LLAMA_LOG_WARN("%s: ref: https://github.com/ggerganov/llama.cpp/pull/3837\n", __func__);- LLAMA_LOG_WARN("%s: ****************************************************************\n", __func__);- }- }-return result;}@@ -6473,8 +6354,6 @@ static int llama_decode_internal(auto & kv_self = lctx.kv_self;- GGML_ASSERT(!!kv_self.ctx);-const int64_t n_embd = hparams.n_embd;const int64_t n_vocab = hparams.n_vocab;@@ -6528,12 +6407,11 @@ static int llama_decode_internal(//printf("kv_self.n = %5d, kv_self.used = %5d, kv_self.head = %5d\n", kv_self.n, kv_self.used, kv_self.head);- ggml_allocr_reset(lctx.alloc);+ ggml_backend_sched_reset(lctx.sched);+ ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);ggml_cgraph * gf = llama_build_graph(lctx, batch);- ggml_allocr_alloc_graph(lctx.alloc, gf);-// the output is always the last tensor in the graphstruct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];GGML_ASSERT(strcmp(res->name, "result_output") == 0);@@ -6545,30 +6423,6 @@ static int llama_decode_internal(GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);}-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- char * buf_alloc_base = (char *)ggml_backend_buffer_get_base(lctx.buf_alloc);- for (int i = 0; i < gf->n_leafs; i++) {- ggml_tensor * node = gf->leafs[i];- if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) {- ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base);- ggml_cuda_copy_to_device(node);- }- }-- for (int i = 0; i < gf->n_nodes; i++) {- ggml_tensor * node = gf->nodes[i];- if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) {- ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base);- }- }-- // HACK: ggml-alloc may change the tensor backend when reusing a parent, so force output to be on the CPU here if needed- if (!lctx.embedding.empty()) {- embeddings->backend = GGML_BACKEND_CPU;- }- res->backend = GGML_BACKEND_CPU;-#endif-// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);// for big prompts, if BLAS is enabled, it is better to use only one thread@@ -6590,12 +6444,16 @@ static int llama_decode_internal(ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);#endif- if (ggml_backend_is_metal(lctx.backend)) {- ggml_backend_metal_set_n_cb(lctx.backend, n_threads);- } else if (ggml_backend_is_cpu(lctx.backend)) {- ggml_backend_cpu_set_n_threads(lctx.backend, n_threads);+ if (ggml_backend_is_metal(lctx.backend_metal)) {+ ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads);+ }++ if (lctx.backend_cpu != nullptr) {+ ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);}- ggml_backend_graph_compute(lctx.backend, gf);+ ggml_backend_sched_graph_compute(lctx.sched, gf);++ // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));#ifdef GGML_USE_MPIggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer);@@ -6643,30 +6501,33 @@ static int llama_decode_internal(logits_out.clear();#endif+ ggml_backend_t res_backend = ggml_backend_sched_get_node_backend(lctx.sched, res);+ GGML_ASSERT(res_backend != nullptr);if (batch.logits) {logits_out.resize(n_vocab * n_tokens);for (uint32_t i = 0; i < n_tokens; i++) {if (batch.logits[i] == 0) {continue;}- ggml_backend_tensor_get(res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float));+ ggml_backend_tensor_get_async(res_backend, res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float));#ifndef NDEBUGlogits_valid[i] = true;#endif}} else if (lctx.logits_all) {logits_out.resize(n_vocab * n_tokens);- ggml_backend_tensor_get(res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float));+ ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float));#ifndef NDEBUGstd::fill(logits_valid.begin(), logits_valid.end(), true);#endif} else {logits_out.resize(n_vocab);- ggml_backend_tensor_get(res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float));+ ggml_backend_tensor_get_async(res_backend, res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float));#ifndef NDEBUGlogits_valid[0] = true;#endif}+ ggml_backend_synchronize(res_backend);}// extract embeddings@@ -6674,7 +6535,9 @@ static int llama_decode_internal(auto & embedding_out = lctx.embedding;embedding_out.resize(n_embd);- ggml_backend_tensor_get(embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float));+ ggml_backend_t embeddings_backend = ggml_backend_sched_get_node_backend(lctx.sched, embeddings);+ ggml_backend_tensor_get_async(embeddings_backend, embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float));+ ggml_backend_synchronize(embeddings_backend);}// measure the performance only for the single-token evals@@ -6745,15 +6608,15 @@ static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {static llama_token llama_byte_to_token(const llama_vocab & vocab, uint8_t ch) {static const char * hex = "0123456789ABCDEF";switch (llama_vocab_get_type(vocab)) {- case LLAMA_VOCAB_TYPE_SPM: {- const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };- return vocab.token_to_id.at(buf);- }- case LLAMA_VOCAB_TYPE_BPE: {- return vocab.token_to_id.at(bytes_to_unicode_bpe(ch));- }- default:- GGML_ASSERT(false);+ case LLAMA_VOCAB_TYPE_SPM: {+ const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };+ return vocab.token_to_id.at(buf);+ }+ case LLAMA_VOCAB_TYPE_BPE: {+ return vocab.token_to_id.at(bytes_to_unicode_bpe(ch));+ }+ default:+ GGML_ASSERT(false);}}@@ -7287,7 +7150,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break;#ifdef PRETOKENIZERDEBUG- fprintf(stderr, "FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());+ LLAMA_LOG_WARN("FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());#endifauto source = std::distance(buffer.begin(), it);@@ -7300,7 +7163,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<buffer.emplace_after(it, (*raw_text), left_reminder_offset, left_reminder_length);#ifdef PRETOKENIZERDEBUG- fprintf(stderr, "FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());+ LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());#endifit++;}@@ -7316,7 +7179,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<buffer.emplace_after(it, (*raw_text), right_reminder_offset, right_reminder_length);#ifdef PRETOKENIZERDEBUG- fprintf(stderr, "FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());+ LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());#endifit++;@@ -7332,7 +7195,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<raw_text_base_length = right_reminder_length;#ifdef PRETOKENIZERDEBUG- fprintf(stderr, "RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());+ LLAMA_LOG_WARN("RR: (%ld %ld) '%s'\n", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());#endif} else {if (source == 0) {@@ -7389,7 +7252,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &}#ifdef PRETOKENIZERDEBUG- fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());#endifllm_tokenizer_spm tokenizer(vocab);llama_escape_whitespace(raw_text);@@ -7410,7 +7273,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);#ifdef PRETOKENIZERDEBUG- fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());+ LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());#endifllm_tokenizer_bpe tokenizer(vocab);tokenizer.tokenize(raw_text, output);@@ -7829,7 +7692,7 @@ struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar)void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) {if (seed == LLAMA_DEFAULT_SEED) {- seed = _rand64();+ seed = time(NULL);}ctx->rng.seed(seed);}@@ -7874,10 +7737,57 @@ void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * canauto comp = [](const llama_token_data & a, const llama_token_data & b) {return a.logit > b.logit;};- if (k == (int) candidates->size) {- std::sort(candidates->data, candidates->data + candidates->size, comp);- } else {+ if (k <= 128) {std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp);+ } else {+ constexpr int nbuckets = 128;+ constexpr float bucket_low = -10.0f;+ constexpr float bucket_high = 10.0f;+ constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);+ constexpr float bucker_inter = -bucket_low * bucket_scale;++ std::vector<int> bucket_idx(candidates->size);+ std::vector<int> histo(nbuckets, 0);++ for (int i = 0; i < (int)candidates->size; ++i) {+ const float val = candidates->data[i].logit;+ int ib = int(bucket_scale * val + bucker_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);+ ib = std::max(0, std::min(nbuckets-1, ib));+ bucket_idx[i] = ib;+ ++histo[ib];+ }+ int nhave = 0;+ int ib = nbuckets - 1;+ for ( ; ib >= 0; --ib) {+ nhave += histo[ib];+ if (nhave >= k) break;+ }+ std::vector<llama_token_data> tmp_tokens(nhave);+ auto ptr = tmp_tokens.data();+ std::vector<llama_token_data*> bucket_ptrs;+ bucket_ptrs.reserve(nbuckets - ib);+ for (int j = nbuckets - 1; j >= ib; --j) {+ bucket_ptrs.push_back(ptr);+ ptr += histo[j];+ }+ for (int i = 0; i < (int)candidates->size; ++i) {+ int j = bucket_idx[i];+ if (j >= ib) {+ *bucket_ptrs[nbuckets-1-j]++ = candidates->data[i];+ }+ }++ ptr = tmp_tokens.data();+ int ndone = 0;+ for (int j = nbuckets-1; j > ib; --j) {+ std::sort(ptr, ptr + histo[j], comp);+ ptr += histo[j];+ ndone += histo[j];+ }+ std::partial_sort(ptr, ptr + k - ndone, ptr + histo[ib], comp);++ std::memcpy(candidates->data, tmp_tokens.data(), k*sizeof(llama_token_data));+}candidates->sorted = true;}@@ -8069,6 +7979,73 @@ void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * c}}+void llama_sample_entropy(struct llama_context * ctx, llama_token_data_array * candidates_p, float min_temp, float max_temp, float exponent_val) {+ const int64_t t_start_sample_us = ggml_time_us();++ // no need to do anything if there is only one (or zero) candidates+ if(candidates_p->size <= 1) {+ return;+ }++ // Calculate maximum possible entropy+ float max_entropy = -logf(1.0f / candidates_p->size);++ llama_sample_softmax(nullptr, candidates_p);++ // Calculate entropy of the softmax probabilities+ float entropy = 0.0f;+ for (size_t i = 0; i < candidates_p->size; ++i) {+ float prob = candidates_p->data[i].p;+ if (prob > 0.0f) { // Ensure no log(0)+ entropy -= prob * logf(prob);+ }+ }++ // Normalize the entropy (max_entropy cannot be 0 here because we checked candidates_p->size != 1 above)+ float normalized_entropy = entropy / max_entropy;++ // Map the normalized entropy to the desired temperature range using the power function+ float dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent_val);++#ifdef DEBUG+ LLAMA_LOG_INFO("Your text maxtemp value is: %f\n", max_temp);+ LLAMA_LOG_INFO("Entropy: %f\n", entropy);+ LLAMA_LOG_INFO("Max Possible Entropy: %f\n", max_entropy);+ LLAMA_LOG_INFO("Normalized Entropy: %f\n", normalized_entropy);+ LLAMA_LOG_INFO("Exponent: %f\n", exponent_val);+ LLAMA_LOG_INFO("Dynamic Temperature (dyn_temp): %f\n", dyn_temp);+#endif++ // Apply the dynamically calculated temperature scaling+ for (size_t i = 0; i < candidates_p->size; ++i) {+ candidates_p->data[i].logit /= dyn_temp;+ }++ // Re-compute softmax probabilities after scaling logits with dynamic temperature+ double max_l_double = candidates_p->data[0].logit;+ double cum_sum_double = 0.0;+ for (size_t i = 0; i < candidates_p->size; ++i) {+ double p = exp(candidates_p->data[i].logit - max_l_double);+ candidates_p->data[i].p = p; // Store the scaled probability+ cum_sum_double += p;+ }+ for (size_t i = 0; i < candidates_p->size; ++i) {+ candidates_p->data[i].p /= cum_sum_double; // Re-normalize the probabilities+ }++#ifdef DEBUG+ // Print the updated top 25 probabilities after temperature scaling+ LLAMA_LOG_INFO("\nUpdated Top 25 Probabilities After Dynamic Temperature Scaling (in percentages):\n");+ for (size_t i = 0; i < 25 && i < candidates_p->size; ++i) {+ LLAMA_LOG_INFO("Token %zu: %f%%\n", i + 1, candidates_p->data[i].p * 100.0f);+ }+#endif++ if (ctx) {+ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;+ }+}+void llama_sample_temp(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) {const int64_t t_start_sample_us = ggml_time_us();@@ -8188,39 +8165,59 @@ static void llama_log_softmax(float * array, size_t size) {}}+void llama_sample_apply_guidance(+ struct llama_context * ctx,+ float * logits,+ float * logits_guidance,+ float scale) {+ GGML_ASSERT(ctx);++ const auto t_start_sample_us = ggml_time_us();+ const auto n_vocab = llama_n_vocab(llama_get_model(ctx));++ llama_log_softmax(logits, n_vocab);+ llama_log_softmax(logits_guidance, n_vocab);++ for (int i = 0; i < n_vocab; ++i) {+ auto & l = logits[i];+ const auto & g = logits_guidance[i];++ l = scale * (l - g) + g;+ }++ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;+}+void llama_sample_classifier_free_guidance(struct llama_context * ctx,llama_token_data_array * candidates,struct llama_context * guidance_ctx,float scale) {- int64_t t_start_sample_us = ggml_time_us();-GGML_ASSERT(ctx);+ int64_t t_start_sample_us;- auto n_vocab = llama_n_vocab(llama_get_model(ctx));+ t_start_sample_us = ggml_time_us();+ const size_t n_vocab = llama_n_vocab(llama_get_model(ctx));- GGML_ASSERT(n_vocab == (int)candidates->size);+ GGML_ASSERT(n_vocab == candidates->size);GGML_ASSERT(!candidates->sorted);- std::vector<float> logits_base;- logits_base.reserve(candidates->size);- for (size_t i = 0; i < candidates->size; ++i) {- logits_base.push_back(candidates->data[i].logit);+ std::vector<float> logits_base(n_vocab);+ for (size_t i = 0; i < n_vocab; ++i) {+ logits_base[i] = candidates->data[i].logit;}- llama_log_softmax(logits_base.data(), candidates->size);- float* logits_guidance = llama_get_logits(guidance_ctx);- llama_log_softmax(logits_guidance, n_vocab);+ float * logits_guidance = llama_get_logits(guidance_ctx);- for (int i = 0; i < n_vocab; ++i) {- float logit_guidance = logits_guidance[i];- float logit_base = logits_base[i];- candidates->data[i].logit = scale * (logit_base - logit_guidance) + logit_guidance;- }+ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;+ llama_sample_apply_guidance(ctx, logits_base.data(), logits_guidance, scale);+ t_start_sample_us = ggml_time_us();- if (ctx) {- ctx->t_sample_us += ggml_time_us() - t_start_sample_us;+ for (size_t i = 0; i < n_vocab; ++i) {+ candidates->data[i].logit = logits_base[i];}++ ctx->t_sample_us += ggml_time_us() - t_start_sample_us;}llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {@@ -8637,13 +8634,19 @@ struct quantize_state_internal {const llama_model_quantize_params * params;int n_attention_wv = 0;- int n_feed_forward_w2 = 0;+ int n_ffn_down = 0;+ int n_ffn_gate = 0;+ int n_ffn_up = 0;int i_attention_wv = 0;- int i_feed_forward_w2 = 0;+ int i_ffn_down = 0;+ int i_ffn_gate = 0;+ int i_ffn_up = 0;int n_k_quantized = 0;int n_fallback = 0;+ bool has_imatrix = false;+quantize_state_internal(const llama_model & model, const llama_model_quantize_params * params): model(model), params(params)@@ -8721,17 +8724,53 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyauto use_more_bits = [](int i_layer, int num_layers) -> bool {return i_layer < num_layers/8 || i_layer >= 7*num_layers/8 || (i_layer - num_layers/8)%3 == 2;};+ const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);+ auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {+ if (n_expert > 1) {+ // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly+ // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work+ // for getting the current layer as I initially thought, and we need to resort to parsing the+ // tensor name.+ n_layer /= n_expert;+ if (sscanf(name, "blk.%d.", &i_layer) != 1) {+ throw std::runtime_error(format("Failed to determine layer for tensor %s", name));+ }+ if (i_layer < 0 || i_layer >= n_layer) {+ throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));+ }+ }+ return std::make_pair(i_layer, n_layer);+ };if (name == tn(LLM_TENSOR_OUTPUT, "weight")) {int nx = tensor->ne[0];if (arch == LLM_ARCH_FALCON || nx % QK_K != 0) {new_type = GGML_TYPE_Q8_0;}+ else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) {+ new_type = GGML_TYPE_Q5_K;+ }else if (new_type != GGML_TYPE_Q8_0) {new_type = GGML_TYPE_Q6_K;}+ } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS) {+ if (name.find("attn_v.weight") != std::string::npos) {+ if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;+ else new_type = GGML_TYPE_Q2_K;+ ++qs.i_attention_wv;+ }+ else if (name.find("ffn_down") != std::string::npos) {+ if (qs.i_ffn_down < qs.n_ffn_down/8) new_type = GGML_TYPE_Q2_K;+ ++qs.i_ffn_down;+ }+ else if (name == "token_embd.weight") new_type = GGML_TYPE_Q2_K;} else if (name.find("attn_v.weight") != std::string::npos) {- if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {+ new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;+ }+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {+ new_type = GGML_TYPE_Q4_K;+ }else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;}@@ -8759,11 +8798,19 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty// TODO: explore better strategiesnew_type = GGML_TYPE_Q8_0;}- } else if (name.find("ffn_down.weight") != std::string::npos) {+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) {+ new_type = GGML_TYPE_Q2_K;+ }+ } else if (name.find("ffn_down") != std::string::npos) {+ auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str());+ int i_layer = info.first, n_layer = info.second;if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS) {+ if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;+ }else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {- new_type = qs.i_feed_forward_w2 < 2 ? GGML_TYPE_Q5_K- : arch != LLM_ARCH_FALCON || use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q4_K+ new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K+ : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K: GGML_TYPE_Q3_K;}else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {@@ -8771,22 +8818,37 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty}else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {if (arch == LLM_ARCH_FALCON) {- new_type = qs.i_feed_forward_w2 < 2 ? GGML_TYPE_Q6_K :- use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;+ new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K :+ use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;} else {- if (use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K;+ if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;}}- else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(qs.i_feed_forward_w2, qs.n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K;- else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && qs.i_feed_forward_w2 < 4) {+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {new_type = GGML_TYPE_Q5_K;}- ++qs.i_feed_forward_w2;+ else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || ftype == LLAMA_FTYPE_MOSTLY_Q5_0)+ && qs.has_imatrix && i_layer < n_layer/8) {+ // Guard against craziness in the first few ffn_down layers that can happen even with imatrix for Q4_0/Q5_0.+ // We only do it when an imatrix is provided because a) we want to make sure that one can always get the+ // same quantization as before imatrix stuff, and b) Q4_1/Q5_1 do go crazy on ffn_down without an imatrix.+ new_type = ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q5_1;+ }+ ++qs.i_ffn_down;} else if (name.find("attn_output.weight") != std::string::npos) {if (arch != LLM_ARCH_FALCON) {- if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;- else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K;- else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;+ if (qs.model.hparams.n_expert == 8) {+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS ||+ ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ||+ ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {+ new_type = GGML_TYPE_Q5_K;+ }+ } else {+ if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) new_type = GGML_TYPE_Q4_K;+ else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;+ }} else {if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K;}@@ -8796,9 +8858,28 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_tyelse if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K;else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K;}- else if (name.find("ffn_gate.weight") != std::string::npos || name.find("ffn_up.weight") != std::string::npos) {- if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;+ else if (name.find("ffn_gate") != std::string::npos) {+ auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str());+ int i_layer = info.first, n_layer = info.second;+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && !use_more_bits(i_layer, n_layer)) {+ new_type = GGML_TYPE_Q2_K;+ }+ ++qs.i_ffn_gate;+ }+ else if (name.find("ffn_up") != std::string::npos) {+ auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str());+ int i_layer = info.first, n_layer = info.second;+ if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_XS && !use_more_bits(i_layer, n_layer)) {+ new_type = GGML_TYPE_Q2_K;+ }+ ++qs.i_ffn_up;}+ // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;+ //}+ // IK: let's remove this, else Q2_K is almost the same as Q3_K_S+ //else if (name.find("ffn_gate") != std::string::npos || name.find("ffn_up") != std::string::npos) {+ // if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;+ //}// This can be used to reduce the size of the Q5_K_S model.// The associated PPL increase is fully in line with the size reduction//else {@@ -8806,7 +8887,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty//}bool convert_incompatible_tensor = false;if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K ||- new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) {+ new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K ||+ new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_XXS) {int nx = tensor->ne[0];int ny = tensor->ne[1];if (nx % QK_K != 0) {@@ -8818,6 +8900,8 @@ static ggml_type get_k_quant_type(quantize_state_internal & qs, ggml_type new_ty}if (convert_incompatible_tensor) {switch (new_type) {+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:case GGML_TYPE_Q2_K: new_type = GGML_TYPE_Q4_0; break;case GGML_TYPE_Q3_K: new_type = GGML_TYPE_Q4_1; break;case GGML_TYPE_Q4_K: new_type = GGML_TYPE_Q5_0; break;@@ -8846,7 +8930,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const scase LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break;// K-quants+ case LLAMA_FTYPE_MOSTLY_Q2_K_S:case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break;+ case LLAMA_FTYPE_MOSTLY_Q3_K_XS:case LLAMA_FTYPE_MOSTLY_Q3_K_S:case LLAMA_FTYPE_MOSTLY_Q3_K_M:case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break;@@ -8856,6 +8942,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const scase LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break;case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break;case LLAMA_FTYPE_MOSTLY_IQ2_XXS:quantized_type = GGML_TYPE_IQ2_XXS; break;+ case LLAMA_FTYPE_MOSTLY_IQ2_XS :quantized_type = GGML_TYPE_IQ2_XS; break;default: throw std::runtime_error(format("invalid output file type %d\n", ftype));}@@ -8886,6 +8973,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sif (params->only_copy) {ftype = model.ftype;}+ const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;+ if (params->imatrix) {+ imatrix_data = static_cast<const std::unordered_map<std::string, std::vector<float>>*>(params->imatrix);+ if (imatrix_data) {+ LLAMA_LOG_INFO("================================ Have weights data with %d entries\n",int(imatrix_data->size()));+ qs.has_imatrix = true;+ }+ }const size_t align = GGUF_DEFAULT_ALIGNMENT;struct gguf_context * ctx_out = gguf_init_empty();@@ -8904,13 +8999,19 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sif (name.find("attn_v.weight") != std::string::npos || name.find("attn_qkv.weight") != std::string::npos) {++qs.n_attention_wv;}- else if (name.find("ffn_down.weight") != std::string::npos) {- ++qs.n_feed_forward_w2;+ else if (name.find("ffn_down") != std::string::npos) {+ ++qs.n_ffn_down;+ }+ else if (name.find("ffn_gate") != std::string::npos) {+ ++qs.n_ffn_gate;+ }+ else if (name.find("ffn_up") != std::string::npos) {+ ++qs.n_ffn_up;}}- if (qs.n_attention_wv != qs.n_feed_forward_w2 || (uint32_t)qs.n_attention_wv != model.hparams.n_layer) {- LLAMA_LOG_WARN("%s ============ Strange model: n_attention_wv = %d, n_feed_forward_w2 = %d, hparams.n_layer = %d\n",- __func__, qs.n_attention_wv, qs.n_feed_forward_w2, model.hparams.n_layer);+ if (qs.n_attention_wv != qs.n_ffn_down || (uint32_t)qs.n_attention_wv != model.hparams.n_layer) {+ LLAMA_LOG_WARN("%s ============ Strange model: n_attention_wv = %d, n_ffn_down = %d, hparams.n_layer = %d\n",+ __func__, qs.n_attention_wv, qs.n_ffn_down, model.hparams.n_layer);}size_t total_size_org = 0;@@ -8995,6 +9096,30 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s} else {const size_t nelements = ggml_nelements(tensor);+ const float * imatrix = nullptr;+ if (imatrix_data) {+ auto it = imatrix_data->find(tensor->name);+ if (it == imatrix_data->end()) {+ LLAMA_LOG_INFO("\n====== %s: did not find weights for %s\n", __func__, tensor->name);+ } else {+ if (it->second.size() == (size_t)tensor->ne[0]) {+ imatrix = it->second.data();+ } else {+ LLAMA_LOG_INFO("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__,+ int(it->second.size()), int(tensor->ne[0]), tensor->name);+ }+ }+ }+ if ((new_type == GGML_TYPE_IQ2_XXS ||+ new_type == GGML_TYPE_IQ2_XS ||+ (new_type == GGML_TYPE_Q2_K && params->ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && strcmp(tensor->name, "token_embd.weight") != 0)) && !imatrix) {+ LLAMA_LOG_ERROR("\n\n============================================================\n");+ LLAMA_LOG_ERROR("Missing importance matrix for tensor %s in a very low-bit quantization\n", tensor->name);+ LLAMA_LOG_ERROR("The result will be garbage, so bailing out\n");+ LLAMA_LOG_ERROR("============================================================\n\n");+ throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name));+ }+float * f32_data;if (tensor->type == GGML_TYPE_F32) {@@ -9015,21 +9140,28 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const snew_data = work.data();std::array<int64_t, 1 << 4> hist_cur = {};- static const int chunk_size = 32 * 512;+ const int n_per_row = tensor->ne[0];+ const int nrows = nelements / n_per_row;++ static const int min_chunk_size = 32 * 512;+ const int chunk_size = n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row);+const int nchunk = (nelements + chunk_size - 1)/chunk_size;const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1;if (nthread_use < 2) {- new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data());+ new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, hist_cur.data(), imatrix);} else {- size_t counter = 0;+ int counter = 0;new_size = 0;- auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements]() {+ auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, chunk_size,+ nrows, n_per_row, imatrix]() {std::array<int64_t, 1 << 4> local_hist = {};+ const int nrows_per_chunk = chunk_size / n_per_row;size_t local_size = 0;while (true) {std::unique_lock<std::mutex> lock(mutex);- size_t first = counter; counter += chunk_size;- if (first >= nelements) {+ int first_row = counter; counter += nrows_per_chunk;+ if (first_row >= nrows) {if (local_size > 0) {for (int j=0; j<int(local_hist.size()); ++j) {hist_cur[j] += local_hist[j];@@ -9039,8 +9171,9 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sbreak;}lock.unlock();- size_t last = std::min(nelements, first + chunk_size);- local_size += ggml_quantize_chunk(new_type, f32_data, new_data, first, last - first, local_hist.data());+ const int this_nrow = std::min(nrows - first_row, nrows_per_chunk);+ local_size += ggml_quantize_chunk(new_type, f32_data, new_data,+ first_row * n_per_row, this_nrow, n_per_row, local_hist.data(), imatrix);}};for (int it = 0; it < nthread_use - 1; ++it) {@@ -9051,7 +9184,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sworkers.clear();}- LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB | hist: ", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);+ LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0);int64_t tot_count = 0;for (size_t i = 0; i < hist_cur.size(); i++) {hist_all[i] += hist_cur[i];@@ -9059,6 +9192,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s}if (tot_count > 0) {+ LLAMA_LOG_INFO(" | hist: ");for (size_t i = 0; i < hist_cur.size(); i++) {LLAMA_LOG_INFO("%5.3f ", hist_cur[i] / float(nelements));}@@ -9144,48 +9278,23 @@ static int llama_apply_lora_from_file_internal(LLAMA_LOG_INFO("%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling);- // create a name -> tensor map of the model to accelerate lookups- // find the max tensor size to estimate the required temporary buffer size- size_t max_tensor_size = 0;- std::unordered_map<std::string, struct ggml_tensor*> model_tensors;- for (const auto & kv : model.tensors_by_name) {- model_tensors.insert(kv);- size_t f32_size = ggml_nelements(kv.second) * sizeof(float);- max_tensor_size = std::max(max_tensor_size, f32_size);- }-- // create a temporary ggml context to store the lora tensors- // TODO: use ggml-alloc- size_t lora_ctx_size = max_tensor_size * 3;- LLAMA_LOG_INFO("%s: allocating %.f MB for lora temporary buffer\n", __func__, lora_ctx_size / 1024.0 / 1024.0);- std::vector<uint8_t> lora_buf(lora_ctx_size);-- struct ggml_init_params params;- params.mem_size = lora_buf.size();- params.mem_buffer = lora_buf.data();- params.no_alloc = false;-- using unique_context = std::unique_ptr<ggml_context, decltype(&ggml_free)>;-- unique_context lora_ctx(nullptr, ggml_free);- lora_ctx.reset(ggml_init(params));- std::unordered_map<std::string, struct ggml_tensor *> lora_tensors;-// load base modelstd::unique_ptr<llama_model_loader> ml;-- if (path_base_model) {+ if (path_base_model) {LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr));- ml->init_mapping(false); // no prefetching+ ml->init_mapping(/*prefetch*/ false); // no prefetching}- // read tensors and apply- bool warned = false;- int n_tensors = 0;-- std::vector<uint8_t> work_buffer;+ struct tensor_meta {+ std::string name;+ ggml_type type;+ int32_t ne[2];+ size_t offset;+ };+ std::map<std::string, tensor_meta> tensor_meta_map;+ // load all tensor metawhile (true) {if (fin.tell() == llamafile_size(fin.file)) {// eof@@ -9198,7 +9307,7 @@ static int llama_apply_lora_from_file_internal(fin.read_raw(&n_dims, sizeof(n_dims));fin.read_raw(&name_len, sizeof(name_len));- fin.read_raw(&ftype, sizeof(ftype));+ fin.read_raw(&ftype, sizeof(ftype));if (n_dims != 1 && n_dims != 2) {LLAMA_LOG_ERROR("%s: unsupported tensor dimension %d\n", __func__, n_dims);@@ -9212,31 +9321,23 @@ static int llama_apply_lora_from_file_internal(std::string name;{- GGML_ASSERT(name_len <= 1024);- char buf[1024];+ GGML_ASSERT(name_len < GGML_MAX_NAME);+ char buf[GGML_MAX_NAME];fin.read_raw(buf, name_len);name = std::string(buf, name_len);}- // check for lora suffix and get the type of tensor- const std::string lora_suffix = ".lora";- size_t pos = name.rfind(lora_suffix);- if (pos == std::string::npos) {- LLAMA_LOG_ERROR("%s: error: '%s' is not a lora tensor\n", __func__, name.c_str());- return 1;+ // check for lora suffix+ std::string lora_suffix;+ if (name.length() > 6) {+ lora_suffix = name.substr(name.length() - 6);}-- std::string lora_type = name.substr(pos + lora_suffix.length());- std::string base_name = name;- base_name.erase(pos);- // LLAMA_LOG_INFO("%s: %s => %s (lora type %s) \n", __func__, name.c_str(), base_name.c_str(), lora_type.c_str());-- if (model_tensors.find(base_name) == model_tensors.end()) {- LLAMA_LOG_ERROR("%s: unknown tensor '%s' in lora adapter\n", __func__, name.data());+ if (lora_suffix != ".loraA" && lora_suffix != ".loraB") {+ LLAMA_LOG_ERROR("%s: error: '%s' is not a lora tensor\n", __func__, name.c_str());return 1;}- // create ggml tensor+ // tensor typeggml_type wtype;switch (ftype) {case 0: wtype = GGML_TYPE_F32; break;@@ -9248,122 +9349,177 @@ static int llama_apply_lora_from_file_internal(return false;}}- ggml_tensor * lora_tensor = ggml_new_tensor_2d(lora_ctx.get(), wtype, ne[0], ne[1]);- ggml_set_name(lora_tensor, name.c_str());- // load tensor data+ // data offsetsize_t offset = fin.tell();- size_t tensor_data_size = ggml_nbytes(lora_tensor);offset = (offset + 31) & -32;- fin.seek(offset, SEEK_SET);- fin.read_raw(lora_tensor->data, tensor_data_size);- lora_tensors[name] = lora_tensor;+ // skip tensor data+ fin.seek(offset + ggml_row_size(wtype, ne[0]) * ne[1], SEEK_SET);++ tensor_meta_map.emplace(name, tensor_meta{ name, wtype, { ne[0], ne[1] }, offset });+ }- // check if we have both A and B tensors and apply- if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() &&- lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) {+ bool warned = false;+ int n_tensors = 0;- ggml_tensor * dest_t = model_tensors[base_name];+ // apply+ ggml_backend_t backend_cpu = ggml_backend_cpu_init();+ if (backend_cpu == nullptr) {+ LLAMA_LOG_ERROR("%s: error: failed to initialize cpu backend\n", __func__);+ return 1;+ }+ ggml_backend_cpu_set_n_threads(backend_cpu, n_threads);- offload_func_t offload_func = ggml_offload_nop;- offload_func_t offload_func_force_inplace = ggml_offload_nop;+ std::vector<no_init<uint8_t>> read_buf;+ for (const auto & it : model.tensors_by_name) {+ const std::string & base_name = it.first;+ ggml_tensor * model_t = it.second;-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) {- if (dest_t->type != GGML_TYPE_F16) {- throw std::runtime_error(format(- "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models. dest_t->type: %d", __func__, dest_t->type));- }- offload_func = ggml_cuda_assign_buffers;- offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace;- }-#endif // GGML_USE_CUBLAS+ if (tensor_meta_map.find(base_name + ".loraA") == tensor_meta_map.end() ||+ tensor_meta_map.find(base_name + ".loraB") == tensor_meta_map.end()) {+ continue;+ }- ggml_tensor * base_t;- if (ml) {- struct gguf_context * ctx_gguf = ml->ctx_gguf;+ tensor_meta & metaA = tensor_meta_map.at(base_name + ".loraA");+ tensor_meta & metaB = tensor_meta_map.at(base_name + ".loraB");- // load from base model- if (gguf_find_tensor(ctx_gguf, base_name.c_str()) < 0) {- LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str());- return 1;- }+ ggml_init_params lora_init_params = {+ /* .mem_size */ ggml_tensor_overhead()*128 + ggml_graph_overhead(),+ /* .mem_buffer */ nullptr,+ /* .no_alloc */ true,+ };+ ggml_context * lora_ctx = ggml_init(lora_init_params);+ if (lora_ctx == nullptr) {+ LLAMA_LOG_ERROR("%s: error: failed to initialize lora context\n", __func__);+ ggml_backend_free(backend_cpu);+ return 1;+ }- base_t = ml->get_tensor_meta(base_name.c_str());- ml->load_data_for(base_t);- } else {- base_t = dest_t;- }+ // create tensors+ ggml_tensor * loraA = ggml_new_tensor_2d(lora_ctx, metaA.type, metaA.ne[0], metaA.ne[1]);+ ggml_tensor * loraB = ggml_new_tensor_2d(lora_ctx, metaB.type, metaB.ne[0], metaB.ne[1]);+ ggml_set_name(loraA, metaA.name.c_str());+ ggml_set_name(loraB, metaB.name.c_str());- if (ggml_is_quantized(base_t->type)) {- if (!warned) {- LLAMA_LOG_WARN("%s: warning: using a lora adapter with a quantized model may result in poor quality, "- "use a f16 or f32 base model with --lora-base\n", __func__);- warned = true;- }+ ggml_tensor * base_t;+ if (ml) {+ if (gguf_find_tensor(ml->ctx_gguf, base_name.c_str()) < 0) {+ LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str());+ return 1;}+ base_t = ggml_dup_tensor(lora_ctx, ml->get_tensor_meta(base_name.c_str()));+ } else {+ base_t = ggml_dup_tensor(lora_ctx, model_t);+ }+ ggml_set_name(base_t, base_name.c_str());- ggml_tensor * loraA = lora_tensors[base_name + ".loraA"];- GGML_ASSERT(loraA->type == GGML_TYPE_F32);- ggml_set_name(loraA, "loraA");+ // allocate in backend buffer+ ggml_backend_buffer_t lora_buf = ggml_backend_alloc_ctx_tensors_from_buft(lora_ctx, ggml_backend_cpu_buffer_type());+ if (lora_buf == nullptr) {+ LLAMA_LOG_ERROR("%s: error: failed to allocate lora tensors\n", __func__);+ return 1;+ }- ggml_tensor * loraB = lora_tensors[base_name + ".loraB"];- GGML_ASSERT(loraB->type == GGML_TYPE_F32);- ggml_set_name(loraB, "loraB");+ // load tensor data+ auto load_tensor = [&read_buf, &fin](const tensor_meta & tensor_meta, ggml_tensor * tensor) {+ read_buf.resize(ggml_nbytes(tensor));+ fin.seek(tensor_meta.offset, SEEK_SET);+ fin.read_raw(read_buf.data(), ggml_nbytes(tensor));+ ggml_backend_tensor_set(tensor, read_buf.data(), 0, read_buf.size());+ };+ load_tensor(metaA, loraA);+ load_tensor(metaB, loraB);- if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) {- LLAMA_LOG_ERROR("%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");"- " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]);- return 1;- }+ // load base model tensor data+ if (ml) {+ ml->load_data_for(base_t);+ } else {+ ggml_backend_tensor_copy(model_t, base_t);+ }++ if (ggml_is_quantized(base_t->type) && !warned) {+ LLAMA_LOG_WARN("%s: warning: using a lora adapter with a quantized model may result in poor quality, "+ "use a f16 or f32 base model with --lora-base\n", __func__);+ warned = true;+ }++ if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) {+ LLAMA_LOG_ERROR("%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");"+ " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]);+ ggml_free(lora_ctx);+ ggml_backend_buffer_free(lora_buf);+ ggml_backend_free(backend_cpu);+ return 1;+ }+ auto build_lora_graph = [&]() {// w = w + BA*s- ggml_tensor * BA = ggml_mul_mat(lora_ctx.get(), loraA, loraB);- offload_func(BA);+ ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB);ggml_set_name(BA, "BA");if (scaling != 1.0f) {- BA = ggml_scale_inplace(lora_ctx.get(), BA, scaling);- offload_func(BA);+ BA = ggml_scale(lora_ctx, BA, scaling);ggml_set_name(BA, "BA_scaled");}ggml_tensor * r;- if (base_t == dest_t) {- r = ggml_add_inplace(lora_ctx.get(), dest_t, BA);- offload_func_force_inplace(r);- ggml_set_name(r, "r_add_inplace");- }- else {- r = ggml_add(lora_ctx.get(), base_t, BA);- offload_func(r);- ggml_set_name(r, "r_add");+ r = ggml_add_inplace(lora_ctx, base_t, BA);+ ggml_set_name(r, "r_add");- r = ggml_cpy(lora_ctx.get(), r, dest_t);- offload_func(r);- ggml_set_name(r, "r_cpy");+ if (base_t->type != model_t->type) {+ // convert the result to the model type+ r = ggml_cast(lora_ctx, r, model_t->type);+ ggml_set_name(r, "r_cast");}- struct ggml_cgraph * gf = ggml_new_graph(lora_ctx.get());- ggml_build_forward_expand(gf, r);+ return r;+ };++ ggml_cgraph * gf = ggml_new_graph(lora_ctx);+ ggml_tensor * r = build_lora_graph();+ ggml_build_forward_expand(gf, r);- ggml_graph_compute_helper(work_buffer, gf, n_threads);+ ggml_backend_buffer_t graph_buf = ggml_backend_alloc_ctx_tensors_from_buft(lora_ctx, ggml_backend_cpu_buffer_type());+ if (graph_buf == nullptr) {+ LLAMA_LOG_ERROR("%s: error: failed to allocate graph tensors\n", __func__);+ ggml_free(lora_ctx);+ ggml_backend_buffer_free(lora_buf);+ ggml_backend_free(backend_cpu);+ return 1;+ }- // the tensors in the adapter must be sorted such that loraA and loraB of the same tensor are next to each other- GGML_ASSERT(lora_tensors.size() == 2);+ ggml_backend_graph_compute(backend_cpu, gf);- // we won't need these tensors again, reset the context to save memory- lora_ctx.reset(ggml_init(params));- lora_tensors.clear();+ ggml_backend_tensor_set(model_t, r->data, 0, ggml_nbytes(r));- n_tensors++;- if (n_tensors % 4 == 0) {- LLAMA_LOG_INFO(".");- }+#if 0+ // TODO: use scheduler with fallback to CPU for less copies between CPU and GPU+ //ggml_backend_sched_t sched = ggml_backend_sched_new(backends.data(), backends.size(), GGML_DEFAULT_GRAPH_SIZE);++ // sched compute+ ggml_build_forward_expand(gf, build_graph());+ ggml_backend_sched_init_measure(sched, gf);++ // create the graph again, since the previous one was destroyed by the measure+ ggml_graph_clear(gf);+ ggml_build_forward_expand(gf, build_graph());+ ggml_backend_sched_graph_compute(sched, gf);+ ggml_backend_sched_free(sched);+#endif++ ggml_backend_buffer_free(lora_buf);+ ggml_backend_buffer_free(graph_buf);+ ggml_free(lora_ctx);++ n_tensors++;+ if (n_tensors % 4 == 0) {+ LLAMA_LOG_INFO(".");}}+ ggml_backend_free(backend_cpu);+const int64_t t_lora_us = ggml_time_us() - t_start_lora_us;LLAMA_LOG_INFO(" done (%.2f ms)\n", t_lora_us / 1000.0);@@ -9376,6 +9532,7 @@ static int llama_apply_lora_from_file_internal(struct llama_model_params llama_model_default_params() {struct llama_model_params result = {/*.n_gpu_layers =*/ 0,+ /*.split_mode =*/ LLAMA_SPLIT_LAYER,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,@@ -9386,8 +9543,9 @@ struct llama_model_params llama_model_default_params() {/*.use_mlock =*/ false,};- if (ggml_metal_supported()) {- result.n_gpu_layers = 1;+ if (llamafile_has_metal()) {+ // note: we usually have plenty of VRAM, so by default offload all layers to the GPU+ result.n_gpu_layers = 999;}return result;@@ -9408,6 +9566,8 @@ struct llama_context_params llama_context_default_params() {/*.yarn_beta_fast =*/ 32.0f,/*.yarn_beta_slow =*/ 1.0f,/*.yarn_orig_ctx =*/ 0,+ /*.cb_eval =*/ nullptr,+ /*.cb_eval_user_data =*/ nullptr,/*.type_k =*/ GGML_TYPE_F16,/*.type_v =*/ GGML_TYPE_F16,/*.mul_mat_q =*/ true,@@ -9427,6 +9587,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {/*.quantize_output_tensor =*/ true,/*.only_copy =*/ false,/*.pure =*/ false,+ /*.imatrix =*/ nullptr,};return result;@@ -9467,6 +9628,7 @@ void llama_backend_free(void) {#ifdef GGML_USE_MPIggml_mpi_backend_free();#endif+ ggml_quantize_free();}int64_t llama_time_us(void) {@@ -9547,6 +9709,9 @@ struct llama_context * llama_new_context_with_model(hparams.n_yarn_orig_ctx != 0 ? hparams.n_yarn_orig_ctx :hparams.n_ctx_train;+ cparams.cb_eval = params.cb_eval;+ cparams.cb_eval_user_data = params.cb_eval_user_data;+auto rope_scaling_type = params.rope_scaling_type;if (rope_scaling_type == LLAMA_ROPE_SCALING_UNSPECIFIED) {rope_scaling_type = hparams.rope_scaling_type_train;@@ -9577,44 +9742,53 @@ struct llama_context * llama_new_context_with_model(GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0);GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0);- // reserve memory for context buffersif (!hparams.vocab_only) {- // initialize backend- if (ggml_metal_supported()) {+ // initialize backends+if (llamafile_has_metal()) {if (model->n_gpu_layers > 0) {- ctx->backend = ggml_backend_metal_init();- if (ctx->backend == nullptr) {+ ctx->backend_metal = ggml_backend_metal_init();+ if (ctx->backend_metal == nullptr) {LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__);+ llama_free(ctx);+ return nullptr;}+ ctx->backends.push_back(ctx->backend_metal);}- }-#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (ggml_cublas_loaded()) {- // for testing only+} else if (llamafile_has_cuda()) {if (model->n_gpu_layers > 0) {- ctx->backend = ggml_backend_cuda_init(0);- if (ctx->backend == nullptr) {- LLAMA_LOG_ERROR("%s: failed to initialize CUDA backend\n", __func__);- }- }- }-#endif-- if (ctx->backend == nullptr && ggml_backend_buffer_is_host(model->buf)) {- ctx->backend = ggml_backend_cpu_init();- if (ctx->backend == nullptr) {- LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);+ // with split_mode LLAMA_SPLIT_NONE or LLAMA_SPLIT_ROW, only the main GPU backend is used+ if (model->split_mode == LLAMA_SPLIT_NONE || model->split_mode == LLAMA_SPLIT_ROW) {+ ggml_backend_t backend = ggml_backend_cuda_init(model->main_gpu);+ if (backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, model->main_gpu);+ llama_free(ctx);+ return nullptr;+ }+ ctx->backends.push_back(backend);+ } else {+ // LLAMA_SPLIT_LAYER requires a backend for each GPU+ for (int device = 0; device < ggml_backend_cuda_get_device_count(); ++device) {+ ggml_backend_t backend = ggml_backend_cuda_init(device);+ if (backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA%d backend\n", __func__, device);+ llama_free(ctx);+ return nullptr;+ }+ ctx->backends.push_back(backend);+ }}}-- if (ctx->backend == nullptr) {- LLAMA_LOG_ERROR("%s: failed to initialize a backend\n", __func__);- delete ctx;+}+ ctx->backend_cpu = ggml_backend_cpu_init();+ if (ctx->backend_cpu == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);+ llama_free(ctx);return nullptr;}+ ctx->backends.push_back(ctx->backend_cpu);- if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, type_k, type_v,- cparams.n_ctx, model->n_gpu_layers, cparams.offload_kqv)) {+ if (!llama_kv_cache_init(ctx->kv_self, ctx->model, type_k, type_v,+ cparams.n_ctx, cparams.offload_kqv)) {LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);llama_free(ctx);return nullptr;@@ -9638,74 +9812,80 @@ struct llama_context * llama_new_context_with_model(ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));}- // resized during inference- if (params.logits_all) {- ctx->logits.reserve(cparams.n_ctx*hparams.n_vocab);- } else {- ctx->logits.reserve(hparams.n_vocab);- }+ // resized during inference, reserve maximum+ ctx->logits.reserve(hparams.n_vocab*cparams.n_batch);if (params.embedding){ctx->embedding.resize(hparams.n_embd);}+ // graph inputs{- // the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data- ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());+ ggml_init_params init_params = {+ /* .mem_size */ ggml_tensor_overhead()*5,+ /* .mem_buffer */ nullptr,+ /* .no_alloc */ true,+ };+ ctx->ctx_input = ggml_init(init_params);- // create measure allocator- ctx->alloc = ggml_allocr_new_measure_from_backend(ctx->backend);+ ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);+ ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);+ ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);+ ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);+ ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);- // build worst-case graph- int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch);- int n_past = cparams.n_ctx - n_tokens;- llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph- ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0));+ ggml_set_name(ctx->inp_tokens, "inp_tokens");+ ggml_set_name(ctx->inp_embd, "inp_embd");+ ggml_set_name(ctx->inp_pos, "inp_pos");+ ggml_set_name(ctx->inp_KQ_mask, "inp_KQ_mask");+ ggml_set_name(ctx->inp_K_shift, "inp_K_shift");- // measure memory requirements for the graph- size_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf);+ ctx->buf_input = ggml_backend_alloc_ctx_tensors_from_buft(ctx->ctx_input, llama_default_buffer_type_cpu(true));- LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MiB\n", __func__, (ctx->buf_compute_meta.size() + alloc_size) / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: %10s input buffer size = %8.2f MiB\n", __func__,+ ggml_backend_buffer_name(ctx->buf_input),+ ggml_backend_buffer_get_size(ctx->buf_input) / 1024.0 / 1024.0);+ }- // create allocator again with exact memory requirements- ggml_allocr_free(ctx->alloc);+ // scheduler and compute buffers+ {+ // buffer types used for the compute buffer of each backend+ std::vector<ggml_backend_buffer_type_t> backend_buft;+ for (auto * backend : ctx->backends) {+ if (ggml_backend_is_cpu(backend)) {+ // use host buffers for the CPU backend compute buffer+ backend_buft.push_back(llama_default_buffer_type_cpu(true));+ } else {+ backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));+ }+ }- ctx->buf_alloc = ggml_backend_alloc_buffer(ctx->backend, alloc_size);- ctx->alloc = ggml_allocr_new_from_buffer(ctx->buf_alloc);-#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)- if (model->n_gpu_layers > 0) {- // the CPU buffer adds this padding in case the malloc buffer is not aligned, so we need to do the same for the GPU buffer, since we use the same offsets- ggml_cuda_set_scratch_size(alloc_size + 64);- LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MiB\n", __func__, alloc_size / 1024.0 / 1024.0);+ // buffer used to store the computation graph and the tensor meta data+ ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());- // calculate total VRAM usage- auto add_tensor = [](const ggml_tensor * t, size_t & size) {- if (t->backend == GGML_BACKEND_GPU || t->backend == GGML_BACKEND_GPU_SPLIT) {- size += ggml_nbytes(t);- }- };- size_t model_vram_size = 0;- for (const auto & kv : model->tensors_by_name) {- add_tensor(kv.second, model_vram_size);- }+ ctx->sched = ggml_backend_sched_new(ctx->backends.data(), backend_buft.data(), ctx->backends.size(), LLAMA_MAX_NODES);+ ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu);- size_t kv_vram_size = 0;- for (auto & k : ctx->kv_self.k_l) {- add_tensor(k, kv_vram_size);- }- for (auto & v : ctx->kv_self.v_l) {- add_tensor(v, kv_vram_size);- }+ // build worst-case graph+ int n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch);+ int n_past = cparams.n_ctx - n_tokens;+ llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph+ ggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0));- size_t ctx_vram_size = alloc_size + kv_vram_size;- size_t total_vram_size = model_vram_size + ctx_vram_size;+ // initialize scheduler with the worst-case graph+ ggml_backend_sched_init_measure(ctx->sched, gf);+ ctx->alloc = ggml_backend_sched_get_tallocr(ctx->sched, ctx->backend_cpu);- LLAMA_LOG_INFO("%s: total VRAM used: %.2f MiB (model: %.2f MiB, context: %.2f MiB)\n", __func__,- total_vram_size / 1024.0 / 1024.0,- model_vram_size / 1024.0 / 1024.0,- ctx_vram_size / 1024.0 / 1024.0);+ for (ggml_backend_t backend : ctx->backends) {+ ggml_backend_buffer_t buf = ggml_backend_sched_get_buffer(ctx->sched, backend);+ LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,+ ggml_backend_buffer_name(buf),+ ggml_backend_buffer_get_size(buf) / 1024.0 / 1024.0);}-#endif++ // note: the number of splits during measure is higher than during inference due to the kv shift+ int n_splits = ggml_backend_sched_get_n_splits(ctx->sched);+ LLAMA_LOG_INFO("%s: graph splits (measure): %d\n", __func__, n_splits);}}@@ -9802,9 +9982,8 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3}int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {- return snprintf(buf, buf_size, "%s %s%s %s",+ return snprintf(buf, buf_size, "%s %s %s",llama_model_arch_name(model->arch).c_str(),- model->hparams.n_expert > 0 ? (std::to_string(model->hparams.n_expert) + "x").c_str() : "",llama_model_type_name(model->type),llama_model_ftype_name(model->ftype).c_str());}@@ -9826,7 +10005,14 @@ uint64_t llama_model_n_params(const struct llama_model * model) {}struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name) {- return ggml_get_tensor(model->ctx, name);+ auto it = std::find_if(model->tensors_by_name.begin(), model->tensors_by_name.end(),+ [name](const std::pair<std::string, struct ggml_tensor *> & it) {+ return it.first == name;+ });+ if (it == model->tensors_by_name.end()) {+ return nullptr;+ }+ return it->second;}uint32_t llama_model_quantize(@@ -10004,19 +10190,18 @@ size_t llama_get_state_size(const struct llama_context * ctx) {// for reference, std::mt19937(1337) serializes to 6701 bytes.const size_t s_rng_size = sizeof(size_t);const size_t s_rng = LLAMA_MAX_RNG_STATE;- const size_t s_logits_capacity = sizeof(size_t);const size_t s_logits_size = sizeof(size_t);+ // assume worst case for logits although only currently set ones are serializedconst size_t s_logits = ctx->logits.capacity() * sizeof(float);const size_t s_embedding_size = sizeof(size_t);const size_t s_embedding = ctx->embedding.size() * sizeof(float);const size_t s_kv_size = sizeof(size_t);const size_t s_kv_ntok = sizeof(int);- const size_t s_kv = ggml_backend_buffer_get_size(ctx->kv_self.buf);+ const size_t s_kv = ctx->kv_self.total_size();const size_t s_total = (+ s_rng_size+ s_rng- + s_logits_capacity+ s_logits_size+ s_logits+ s_embedding_size@@ -10085,37 +10270,27 @@ struct llama_data_file_context : llama_data_context {static void llama_copy_state_data_internal(struct llama_context * ctx, llama_data_context * data_ctx) {// copy rng{- std::stringstream rng_ss;+ std::ostringstream rng_ss;rng_ss << ctx->rng;- const size_t rng_size = rng_ss.str().size();- char rng_buf[LLAMA_MAX_RNG_STATE];+ const std::string & rng_str = rng_ss.str();+ const size_t rng_size = rng_str.size();- memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE);- memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size());+ GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE);- data_ctx->write(&rng_size, sizeof(rng_size));- data_ctx->write(&rng_buf[0], LLAMA_MAX_RNG_STATE);+ data_ctx->write(&rng_size, sizeof(rng_size));+ data_ctx->write(rng_str.data(), rng_size);}// copy logits{- const size_t logits_cap = ctx->logits.capacity();const size_t logits_size = ctx->logits.size();- data_ctx->write(&logits_cap, sizeof(logits_cap));data_ctx->write(&logits_size, sizeof(logits_size));if (logits_size) {data_ctx->write(ctx->logits.data(), logits_size * sizeof(float));}-- // If there is a gap between the size and the capacity, write padding- size_t padding_size = (logits_cap - logits_size) * sizeof(float);- if (padding_size > 0) {- std::vector<uint8_t> padding(padding_size, 0); // Create a buffer filled with zeros- data_ctx->write(padding.data(), padding_size);- }}// copy embeddings@@ -10140,7 +10315,7 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datconst auto n_embd_v_gqa = hparams.n_embd_v_gqa();const auto n_ctx = cparams.n_ctx;- const size_t kv_buf_size = ggml_backend_buffer_get_size(kv_self.buf);+ const size_t kv_buf_size = kv_self.total_size();const uint32_t kv_head = kv_self.head;const uint32_t kv_size = kv_self.size;const uint32_t kv_used = kv_self.used;@@ -10153,46 +10328,19 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datif (kv_buf_size) {const size_t elt_size = ggml_element_size(kv_self.k_l[0]);- ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });- ggml_cgraph * gf = ggml_new_graph(cpy_ctx);-- std::vector<struct ggml_tensor *> kout2d(n_layer);- std::vector<struct ggml_tensor *> vout2d(n_layer);-- for (int il = 0; il < (int) n_layer; ++il) {- kout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head);- vout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa);-- ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],- n_embd_k_gqa, kv_head,- elt_size*n_embd_k_gqa, 0);-- ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],- kv_head, n_embd_v_gqa,- elt_size*n_ctx, 0);-- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, k2d, kout2d[il]));- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, v2d, vout2d[il]));- }-- ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend);-- ggml_backend_graph_compute(ctx->backend, gf);-std::vector<uint8_t> tmp_buf;for (int il = 0; il < (int) n_layer; ++il) {- tmp_buf.resize(ggml_nbytes(kout2d[il]));- ggml_backend_tensor_get(kout2d[il], tmp_buf.data(), 0, tmp_buf.size());+ tmp_buf.resize(elt_size*n_embd_k_gqa*kv_head);+ ggml_backend_tensor_get(kv_self.k_l[il], tmp_buf.data(), 0, tmp_buf.size());data_ctx->write(tmp_buf.data(), tmp_buf.size());- tmp_buf.resize(ggml_nbytes(vout2d[il]));- ggml_backend_tensor_get(vout2d[il], tmp_buf.data(), 0, tmp_buf.size());- data_ctx->write(tmp_buf.data(), tmp_buf.size());+ // v is not contiguous, copy row by row+ tmp_buf.resize(elt_size*kv_head);+ for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {+ ggml_backend_tensor_get(kv_self.v_l[il], tmp_buf.data(), ir*elt_size*n_ctx, tmp_buf.size());+ data_ctx->write(tmp_buf.data(), tmp_buf.size());+ }}-- ggml_free(cpy_ctx);-- ggml_backend_buffer_free(buf);}for (uint32_t i = 0; i < kv_size; ++i) {@@ -10225,13 +10373,13 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {// set rng{size_t rng_size;- char rng_buf[LLAMA_MAX_RNG_STATE];+ memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size);- memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size);- memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE;+ GGML_ASSERT(rng_size <= LLAMA_MAX_RNG_STATE);- std::stringstream rng_ss;- rng_ss.str(std::string(&rng_buf[0], rng_size));+ std::string rng_str((char *)inp, rng_size); inp += rng_size;++ std::istringstream rng_ss(rng_str);rng_ss >> ctx->rng;GGML_ASSERT(!rng_ss.fail());@@ -10239,20 +10387,18 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {// set logits{- size_t logits_cap;size_t logits_size;- memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap);memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size);- GGML_ASSERT(ctx->logits.capacity() == logits_cap);+ GGML_ASSERT(ctx->logits.capacity() >= logits_size);if (logits_size) {ctx->logits.resize(logits_size);+memcpy(ctx->logits.data(), inp, logits_size * sizeof(float));+ inp += logits_size * sizeof(float);}-- inp += logits_cap * sizeof(float);}// set embeddings@@ -10291,48 +10437,22 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used);if (kv_buf_size) {- GGML_ASSERT(ggml_backend_buffer_get_size(kv_self.buf) == kv_buf_size);+ GGML_ASSERT(kv_self.total_size() == kv_buf_size);const size_t elt_size = ggml_element_size(kv_self.k_l[0]);- ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });- ggml_cgraph * gf = ggml_new_graph(cpy_ctx);-- std::vector<struct ggml_tensor *> kin2d(n_layer);- std::vector<struct ggml_tensor *> vin2d(n_layer);-- for (int il = 0; il < n_layer; ++il) {- kin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head);- vin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa);-- ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],- n_embd_k_gqa, kv_head,- elt_size*n_embd_k_gqa, 0);-- ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],- kv_head, n_embd_v_gqa,- elt_size*n_ctx, 0);-- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, kin2d[il], k2d));- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, vin2d[il], v2d));- }-- ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend);-- // load data into the tensors- for (int il = 0; il < n_layer; ++il) {- ggml_backend_tensor_set(kin2d[il], inp, 0, ggml_nbytes(kin2d[il]));- inp += ggml_nbytes(kin2d[il]);-- ggml_backend_tensor_set(vin2d[il], inp, 0, ggml_nbytes(vin2d[il]));- inp += ggml_nbytes(vin2d[il]);+ for (int il = 0; il < (int) n_layer; ++il) {+ size_t k_size = elt_size*n_embd_k_gqa*kv_head;+ ggml_backend_tensor_set(kv_self.k_l[il], inp, 0, k_size);+ inp += k_size;++ // v is not contiguous, copy row by row+ size_t v_row_size = elt_size*kv_head;+ for (int ir = 0; ir < (int) n_embd_v_gqa; ++ir) {+ ggml_backend_tensor_set(kv_self.v_l[il], inp, ir*elt_size*n_ctx, v_row_size);+ inp += v_row_size;+ }}-- ggml_backend_graph_compute(ctx->backend, gf);-- ggml_free(cpy_ctx);-- ggml_backend_buffer_free(buf);}ctx->kv_self.head = kv_head;@@ -10648,6 +10768,8 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token tokenif (0 <= token && token < llama_n_vocab(model)) {switch (llama_vocab_get_type(model->vocab)) {case LLAMA_VOCAB_TYPE_SPM: {+ // NOTE: we accept all unsupported token types,+ // suppressing them like CONTROL tokens.if (llama_is_normal_token(model->vocab, token)) {std::string result = model->vocab.id_to_token[token].text;llama_unescape_whitespace(result);@@ -10656,6 +10778,13 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token}memcpy(buf, result.c_str(), result.length());return result.length();+ } else if (llama_is_user_defined_token(model->vocab, token)) {+ std::string result = model->vocab.id_to_token[token].text;+ if (length < (int) result.length()) {+ return -result.length();+ }+ memcpy(buf, result.c_str(), result.length());+ return result.length();} else if (llama_is_unknown_token(model->vocab, token)) { // NOLINTif (length < 3) {return -3;@@ -10670,14 +10799,12 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token}buf[0] = llama_token_to_byte(model->vocab, token);return 1;- } else {- // TODO: for now we accept all unsupported token types,- // suppressing them like CONTROL tokens.- // GGML_ASSERT(false);}break;}case LLAMA_VOCAB_TYPE_BPE: {+ // NOTE: we accept all unsupported token types,+ // suppressing them like CONTROL tokens.if (llama_is_normal_token(model->vocab, token)) {std::string result = model->vocab.id_to_token[token].text;result = llama_decode_text(result);@@ -10686,12 +10813,15 @@ int32_t llama_token_to_piece(const struct llama_model * model, llama_token token}memcpy(buf, result.c_str(), result.length());return result.length();+ } else if (llama_is_user_defined_token(model->vocab, token)) {+ std::string result = model->vocab.id_to_token[token].text;+ if (length < (int) result.length()) {+ return -result.length();+ }+ memcpy(buf, result.c_str(), result.length());+ return result.length();} else if (llama_is_control_token(model->vocab, token)) {;- } else {- // TODO: for now we accept all unsupported token types,- // suppressing them like CONTROL tokens.- // GGML_ASSERT(false);}break;}@@ -10730,7 +10860,7 @@ void llama_print_timings(struct llama_context * ctx) {__func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval);LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",__func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval);- LLAMA_LOG_INFO("%s: total time = %10.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms));+ LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (timings.t_end_ms - timings.t_start_ms), (timings.n_p_eval + timings.n_eval));}void llama_reset_timings(struct llama_context * ctx) {@@ -10802,7 +10932,7 @@ const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internalvoid llama_log_set(ggml_log_callback log_callback, void * user_data) {g_state.log_callback = log_callback ? log_callback : llama_log_callback_default;g_state.log_callback_user_data = user_data;- ggml_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);+ ggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);}static void llama_log_internal_v(ggml_log_level level, const char * format, va_list args) {@@ -10824,7 +10954,6 @@ static void llama_log_internal_v(ggml_log_level level, const char * format, va_lstatic void llama_log_internal(ggml_log_level level, const char * format, ...) {va_list args;- if (FLAG_log_disable) return;va_start(args, format);llama_log_internal_v(level, format, args);va_end(args);diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 669d23cc..7c418b7a 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -41,9 +41,8 @@#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN-#define LLAMA_SESSION_VERSION 3+#define LLAMA_SESSION_VERSION 4-// Defined when llama.cpp is compiled with support for offloading model layers to GPU.#define LLAMA_SUPPORTS_GPU_OFFLOAD#ifdef __cplusplus@@ -100,6 +99,9 @@ extern "C" {LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensorsLLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensorsLLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_Q3_K_XS = 22, // except 1d tensorsLLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};@@ -112,6 +114,12 @@ extern "C" {LLAMA_ROPE_SCALING_MAX_VALUE = LLAMA_ROPE_SCALING_YARN,};+ enum llama_split_mode {+ LLAMA_SPLIT_NONE = 0, // single GPU+ LLAMA_SPLIT_LAYER = 1, // split layers and KV across GPUs+ LLAMA_SPLIT_ROW = 2, // split rows across GPUs+ };+typedef struct llama_token_data {llama_token id; // token idfloat logit; // log-odds of the token@@ -174,8 +182,16 @@ extern "C" {struct llama_model_params {int32_t n_gpu_layers; // number of layers to store in VRAM- int32_t main_gpu; // the GPU that is used for scratch and small tensors- const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES)+ enum llama_split_mode split_mode; // how to split the model across multiple GPUs++ // main_gpu interpretation depends on split_mode:+ // LLAMA_SPLIT_NONE: the GPU that is used for the entire model+ // LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results+ // LLAMA_SPLIT_LAYER: ignored+ int32_t main_gpu;++ // proportion of the model (layers or rows) to offload to each GPU, size: LLAMA_MAX_DEVICES+ const float * tensor_split;// Called with a progress value between 0.0 and 1.0. Pass NULL to disable.// If the provided progress_callback returns true, model loading continues.@@ -211,6 +227,9 @@ extern "C" {float yarn_beta_slow; // YaRN high correction dimuint32_t yarn_orig_ctx; // YaRN original context size+ ggml_backend_sched_eval_callback cb_eval;+ void * cb_eval_user_data;+enum ggml_type type_k; // data type for K cacheenum ggml_type type_v; // data type for V cache@@ -229,6 +248,7 @@ extern "C" {bool quantize_output_tensor; // quantize output.weightbool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignoredbool pure; // disable k-quant mixtures and quantize all tensors to the same type+ void * imatrix; // pointer to importance matrix data} llama_model_quantize_params;// grammar types@@ -693,14 +713,21 @@ extern "C" {float penalty_present);/// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806- /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted.- /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.- /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.- LLAMA_API void llama_sample_classifier_free_guidance(+ /// @param logits Logits extracted from the original generation context.+ /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.+ /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.+ LLAMA_API void llama_sample_apply_guidance(+ struct llama_context * ctx,+ float * logits,+ float * logits_guidance,+ float scale);++ LLAMA_API DEPRECATED(void llama_sample_classifier_free_guidance(struct llama_context * ctx,llama_token_data_array * candidates,struct llama_context * guidance_ctx,- float scale);+ float scale),+ "use llama_sample_apply_guidance() instead");/// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.LLAMA_API void llama_sample_softmax(@@ -742,6 +769,14 @@ extern "C" {float p,size_t min_keep);+ /// @details Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.+ LLAMA_API void llama_sample_entropy(+ struct llama_context * ctx,+ llama_token_data_array * candidates_p,+ float min_temp,+ float max_temp,+ float exponent_val);+LLAMA_API void llama_sample_temp(struct llama_context * ctx,llama_token_data_array * candidates,diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex 0e6e236a..3b8d53e4 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -14,6 +14,7 @@#include <map>#include <regex>#include <stdexcept>+#include <sstream>#include <vector>#include "clip.h"@@ -66,6 +67,7 @@ static std::string format(const char * fmt, ...) {#define KEY_PATCH_SIZE "clip.vision.patch_size"#define KEY_IMAGE_MEAN "clip.vision.image_mean"#define KEY_IMAGE_STD "clip.vision.image_std"+#define KEY_PROJ_TYPE "clip.projector_type"//// tensor name constants@@ -88,6 +90,21 @@ static std::string format(const char * fmt, ...) {#define TN_TEXT_PROJ "text_projection.weight"#define TN_VIS_PROJ "visual_projection.weight"#define TN_LLAVA_PROJ "mm.%d.%s"+#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"+#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"+++enum projector_type {+ PROJECTOR_TYPE_MLP,+ PROJECTOR_TYPE_LDP,+ PROJECTOR_TYPE_UNKNOWN,+};++static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {+ { PROJECTOR_TYPE_MLP, "mlp" },+ { PROJECTOR_TYPE_LDP, "ldp" },+};+//// utilities to get data from a gguf file@@ -125,24 +142,92 @@ static struct ggml_tensor * get_tensor(struct ggml_context * ctx, const std::str}static std::string get_ftype(int ftype) {- switch (ftype) {- case 0:- return "f32";- case 1:- return "f16";- case 2:- return "q4_0";- case 3:- return "q4_1";- case 6:- return "q5_0";- case 7:- return "q5_1";- case 8:- return "q8_0";- default:- throw std::runtime_error(format("%s: Unrecognized file type: %d\n", __func__, ftype));+ return ggml_type_name(static_cast<ggml_type>(ftype));+}++static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {+ switch (type) {+ case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);+ case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);+ case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);+ case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);+ case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);+ case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);+ case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);+ case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);+ case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);+ case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);+ case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";+ default: return format("unknown type %d", type);+ }+}+++static void replace_all(std::string & s, const std::string & search, const std::string & replace) {+ std::string result;+ for (size_t pos = 0; ; pos += search.length()) {+ auto new_pos = s.find(search, pos);+ if (new_pos == std::string::npos) {+ result += s.substr(pos, s.size() - pos);+ break;+ }+ result += s.substr(pos, new_pos - pos) + replace;+ pos = new_pos;+ }+ s = std::move(result);+}++static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {+ const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);++ switch (type) {+ case GGUF_TYPE_STRING:+ return gguf_get_val_str(ctx_gguf, i);+ case GGUF_TYPE_ARRAY:+ {+ const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);+ int arr_n = gguf_get_arr_n(ctx_gguf, i);+ const void * data = gguf_get_arr_data(ctx_gguf, i);+ std::stringstream ss;+ ss << "[";+ for (int j = 0; j < arr_n; j++) {+ if (arr_type == GGUF_TYPE_STRING) {+ std::string val = gguf_get_arr_str(ctx_gguf, i, j);+ // escape quotes+ replace_all(val, "\\", "\\\\");+ replace_all(val, "\"", "\\\"");+ ss << '"' << val << '"';+ } else if (arr_type == GGUF_TYPE_ARRAY) {+ ss << "???";+ } else {+ ss << gguf_data_to_str(arr_type, data, j);+ }+ if (j < arr_n - 1) {+ ss << ", ";+ }+ }+ ss << "]";+ return ss.str();+ }+ default:+ return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);+ }+}++static void print_tensor_info(const ggml_tensor* tensor, const char* prefix = "") {+ size_t tensor_size = ggml_nbytes(tensor);+ tinylogf("%s: n_dims = %d, name = %s, tensor_size=%zu, shape:[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "], type = %s\n",+ prefix, ggml_n_dims(tensor), tensor->name, tensor_size,+ tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], ggml_type_name(tensor->type));+}++static projector_type clip_projector_type_from_string(const std::string & name) {+ for (const auto & kv : PROJECTOR_TYPE_NAMES) { // NOLINT+ if (kv.second == name) {+ return kv.first;+ }}+ return PROJECTOR_TYPE_UNKNOWN;}//@@ -221,6 +306,32 @@ struct clip_vision_model {struct ggml_tensor * mm_0_b;struct ggml_tensor * mm_2_w;struct ggml_tensor * mm_2_b;++ // MobileVLM projection+ struct ggml_tensor * mm_model_mlp_1_w;+ struct ggml_tensor * mm_model_mlp_1_b;+ struct ggml_tensor * mm_model_mlp_3_w;+ struct ggml_tensor * mm_model_mlp_3_b;+ struct ggml_tensor * mm_model_block_1_block_0_0_w;+ struct ggml_tensor * mm_model_block_1_block_0_1_w;+ struct ggml_tensor * mm_model_block_1_block_0_1_b;+ struct ggml_tensor * mm_model_block_1_block_1_fc1_w;+ struct ggml_tensor * mm_model_block_1_block_1_fc1_b;+ struct ggml_tensor * mm_model_block_1_block_1_fc2_w;+ struct ggml_tensor * mm_model_block_1_block_1_fc2_b;+ struct ggml_tensor * mm_model_block_1_block_2_0_w;+ struct ggml_tensor * mm_model_block_1_block_2_1_w;+ struct ggml_tensor * mm_model_block_1_block_2_1_b;+ struct ggml_tensor * mm_model_block_2_block_0_0_w;+ struct ggml_tensor * mm_model_block_2_block_0_1_w;+ struct ggml_tensor * mm_model_block_2_block_0_1_b;+ struct ggml_tensor * mm_model_block_2_block_1_fc1_w;+ struct ggml_tensor * mm_model_block_2_block_1_fc1_b;+ struct ggml_tensor * mm_model_block_2_block_1_fc2_w;+ struct ggml_tensor * mm_model_block_2_block_1_fc2_b;+ struct ggml_tensor * mm_model_block_2_block_2_0_w;+ struct ggml_tensor * mm_model_block_2_block_2_1_w;+ struct ggml_tensor * mm_model_block_2_block_2_1_b;};struct clip_ctx {@@ -229,6 +340,7 @@ struct clip_ctx {bool has_llava_projector = false;struct clip_vision_model vision_model;+ projector_type proj_type = PROJECTOR_TYPE_MLP;float image_mean[3];float image_std[3];@@ -446,16 +558,135 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32free(patches_data);}+ // shape [1, 576, 1024]+ // ne is whcn, ne = [1024, 576, 1, 1]embeddings = ggml_get_rows(ctx0, embeddings, patches);- // mm projection 0- embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);- embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);+ // print_tensor_info(embeddings, "embeddings");++ // llava projector+ if (ctx->proj_type == PROJECTOR_TYPE_MLP) {+ embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);+ embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);- embeddings = ggml_gelu(ctx0, embeddings);+ embeddings = ggml_gelu(ctx0, embeddings);- embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);- embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);+ embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);+ embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);+ }+ else if (ctx->proj_type == PROJECTOR_TYPE_LDP) {+ // MobileVLM projector+ int n_patch = 24;+ struct ggml_tensor * mlp_1 = ggml_mul_mat(ctx0, model.mm_model_mlp_1_w, embeddings);+ mlp_1 = ggml_add(ctx0, mlp_1, model.mm_model_mlp_1_b);+ mlp_1 = ggml_gelu(ctx0, mlp_1);+ struct ggml_tensor * mlp_3 = ggml_mul_mat(ctx0, model.mm_model_mlp_3_w, mlp_1);+ mlp_3 = ggml_add(ctx0, mlp_3, model.mm_model_mlp_3_b);+ // mlp_3 shape = [1, 576, 2048], ne = [2048, 576, 1, 1]++ // block 1+ struct ggml_tensor * block_1 = nullptr;+ {+ // transpose from [1, 576, 2048] --> [1, 2048, 576] --> [1, 2048, 24, 24]+ mlp_3 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_3, 1, 0, 2, 3));+ mlp_3 = ggml_reshape_4d(ctx0, mlp_3, n_patch, n_patch, mlp_3->ne[1], mlp_3->ne[2]);+ // stride = 1, padding = 1, bias is nullptr+ block_1 = ggml_conv_depthwise_2d(ctx0, model.mm_model_block_1_block_0_0_w, mlp_3, 1, 1, 1, 1, 1, 1);++ // layer norm+ // // block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));+ // block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]+ block_1 = ggml_norm(ctx0, block_1, eps);+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_0_1_w), model.mm_model_block_1_block_0_1_b);+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));++ // block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]+ // hardswish+ struct ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);++ block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);+ // block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]+ // pointwise conv+ block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_1_fc1_w, block_1);+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc1_b);+ block_1 = ggml_relu(ctx0, block_1);+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_1_fc2_w, block_1);+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc2_b);+ block_1 = ggml_hardsigmoid(ctx0, block_1);+ // block_1_hw shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1], block_1 shape = [1, 2048], ne = [2048, 1, 1, 1]+ block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);+ block_1 = ggml_mul(ctx0, block_1_hw, block_1);++ int w = block_1->ne[0], h = block_1->ne[1];+ block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));++ // block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_1_block_2_0_w, block_1);+ block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);++ // block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]+ block_1 = ggml_norm(ctx0, block_1, eps);+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_2_1_w), model.mm_model_block_1_block_2_1_b);+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));+ // block1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]+ // residual+ block_1 = ggml_add(ctx0, mlp_3, block_1);+ }++ // block_2+ {+ // stride = 2+ block_1 = ggml_conv_depthwise_2d(ctx0, model.mm_model_block_2_block_0_0_w, block_1, 2, 2, 1, 1, 1, 1);++ // block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]+ // layer norm+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));+ // block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]+ block_1 = ggml_norm(ctx0, block_1, eps);+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_0_1_w), model.mm_model_block_2_block_0_1_b);+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));+ // block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]+ // hardswish+ struct ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);++ // not sure the parameters is right for globalAvgPooling+ block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);+ // block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]+ // pointwise conv+ block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_1_fc1_w, block_1);+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc1_b);+ block_1 = ggml_relu(ctx0, block_1);+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_1_fc2_w, block_1);+ block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc2_b);+ block_1 = ggml_hardsigmoid(ctx0, block_1);++ // block_1_hw shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1], block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]+ block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);+ block_1 = ggml_mul(ctx0, block_1_hw, block_1);++ int w = block_1->ne[0], h = block_1->ne[1];+ block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);+ block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));+ // block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]+ block_1 = ggml_mul_mat(ctx0, model.mm_model_block_2_block_2_0_w, block_1);+ block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);+++ // block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]+ block_1 = ggml_norm(ctx0, block_1, eps);+ block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_2_1_w), model.mm_model_block_2_block_2_1_b);+ block_1 = ggml_reshape_3d(ctx0, block_1, block_1->ne[0], block_1->ne[1] * block_1->ne[2], block_1->ne[3]);+ // block_1 shape = [1, 144, 2048], ne = [2048, 144, 1]+ }+ embeddings = block_1;+ }+ else {+ GGML_ASSERT(false);+ }}// build the graph@@ -506,16 +737,47 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {tinylogf("\n");}const int n_tensors = gguf_get_n_tensors(ctx);+// kv- if (verbosity >= 3) {- const int n_kv = gguf_get_n_kv(ctx);+ const int n_kv = gguf_get_n_kv(ctx);+ tinylogf("%s: loaded meta data with %d key-value pairs and %d tensors from %s\n",+ __func__, n_kv, n_tensors, fname);+ {+ std::map<enum ggml_type, uint32_t> n_type;- for (int i = 0; i < n_kv; ++i) {- const char * key = gguf_get_key(ctx, i);+ for (int i = 0; i < n_tensors; i++) {+ enum ggml_type type = gguf_get_tensor_type(ctx, i);- tinylogf("%s: kv[%d]: key = %s\n", __func__, i, key);+ n_type[type]++;+ }++ tinylogf("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);+ for (int i = 0; i < n_kv; i++) {+ const char * name = gguf_get_key(ctx, i);+ const enum gguf_type type = gguf_get_kv_type(ctx, i);+ const std::string type_name =+ type == GGUF_TYPE_ARRAY+ ? format("%s[%s,%d]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(ctx, i)), gguf_get_arr_n(ctx, i))+ : gguf_type_name(type);++ std::string value = gguf_kv_to_str(ctx, i);+ const size_t MAX_VALUE_LEN = 40;+ if (value.size() > MAX_VALUE_LEN) {+ value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());+ }+ replace_all(value, "\n", "\\n");++ tinylogf("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());+ }++ // print type counts+ for (auto & kv : n_type) {+ if (kv.second == 0) {+ continue;+ }++ tinylogf("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);}- tinylogf("\n");}// data@@ -524,12 +786,13 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {for (int i = 0; i < n_tensors; ++i) {const char * name = gguf_get_tensor_name(ctx, i);const size_t offset = gguf_get_tensor_offset(ctx, i);+ enum ggml_type type = gguf_get_tensor_type(ctx, i);struct ggml_tensor * cur = ggml_get_tensor(meta, name);size_t tensor_size = ggml_nbytes(cur);buffer_size += tensor_size;if (verbosity >= 3) {- tinylogf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu\n", __func__, i,- ggml_n_dims(cur), cur->name, tensor_size, offset);+ tinylogf("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",+ __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));}}}@@ -538,16 +801,23 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {clip_ctx * new_clip = new clip_ctx;- if ((new_clip->backend = ggml_backend_cuda_init(0))) {- tinylogf("%s: CLIP using CUDA backend\n", __func__);+ // update projector type+ {+ int idx = gguf_find_key(ctx, KEY_PROJ_TYPE);+ if (idx != -1) {+ const std::string proj_type = gguf_get_val_str(ctx, idx);+ new_clip->proj_type = clip_projector_type_from_string(proj_type);+ }+ else {+ new_clip->proj_type = PROJECTOR_TYPE_MLP;+ }}- if (!new_clip->backend &&- (new_clip->backend = ggml_backend_metal_init())) {+ if ((new_clip->backend = ggml_backend_metal_init())) {tinylogf("%s: CLIP using Metal backend\n", __func__);- }-- if (!new_clip->backend) {+ } else if ((new_clip->backend = ggml_backend_cuda_init(0))) {+ tinylogf("%s: CLIP using CUDA backend\n", __func__);+ } else {new_clip->backend = ggml_backend_cpu_init();tinylogf("%s: CLIP using CPU backend\n", __func__);}@@ -594,7 +864,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {new_clip->ctx_data = ggml_init(params);if (!new_clip->ctx_data) {- tinylogf("%s: ggml_init() failed\n", __func__);+ fprintf(stderr, "%s: ggml_init() failed\n", __func__);clip_free(new_clip);return nullptr;}@@ -680,10 +950,45 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));- vision_model.mm_0_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "weight"));- vision_model.mm_0_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "bias"));- vision_model.mm_2_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "weight"));- vision_model.mm_2_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "bias"));++ // LLaVA projection+ if (new_clip->proj_type == PROJECTOR_TYPE_MLP) {+ vision_model.mm_0_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "weight"));+ vision_model.mm_0_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "bias"));+ vision_model.mm_2_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "weight"));+ vision_model.mm_2_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "bias"));+ }+ else if (new_clip->proj_type == PROJECTOR_TYPE_LDP) {+ // MobileVLM projection+ vision_model.mm_model_mlp_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "weight"));+ vision_model.mm_model_mlp_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 1, "bias"));+ vision_model.mm_model_mlp_3_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "weight"));+ vision_model.mm_model_mlp_3_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_MLP, 3, "bias"));+ vision_model.mm_model_block_1_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "0.weight"));+ vision_model.mm_model_block_1_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.weight"));+ vision_model.mm_model_block_1_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 0, "1.bias"));+ vision_model.mm_model_block_1_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.weight"));+ vision_model.mm_model_block_1_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc1.bias"));+ vision_model.mm_model_block_1_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.weight"));+ vision_model.mm_model_block_1_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 1, "fc2.bias"));+ vision_model.mm_model_block_1_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "0.weight"));+ vision_model.mm_model_block_1_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.weight"));+ vision_model.mm_model_block_1_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 1, 2, "1.bias"));+ vision_model.mm_model_block_2_block_0_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "0.weight"));+ vision_model.mm_model_block_2_block_0_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.weight"));+ vision_model.mm_model_block_2_block_0_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 0, "1.bias"));+ vision_model.mm_model_block_2_block_1_fc1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.weight"));+ vision_model.mm_model_block_2_block_1_fc1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc1.bias"));+ vision_model.mm_model_block_2_block_1_fc2_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.weight"));+ vision_model.mm_model_block_2_block_1_fc2_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 1, "fc2.bias"));+ vision_model.mm_model_block_2_block_2_0_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "0.weight"));+ vision_model.mm_model_block_2_block_2_1_w = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.weight"));+ vision_model.mm_model_block_2_block_2_1_b = get_tensor(new_clip->ctx_data, format(TN_MVLM_PROJ_BLOCK, 2, 2, "1.bias"));+ }+ else {+ std::string proj_type = PROJECTOR_TYPE_NAMES[new_clip->proj_type];+ throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));+ }vision_model.layers.resize(hparams.n_layer);for (int il = 0; il < hparams.n_layer; ++il) {@@ -933,26 +1238,8 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const iggml_type type = GGML_TYPE_Q4_1;- switch (itype) {- case 2:- type = GGML_TYPE_Q4_0;- break;- case 3:- type = GGML_TYPE_Q4_1;- break;- case 6:- type = GGML_TYPE_Q5_0;- break;- case 7:- type = GGML_TYPE_Q5_1;- break;- case 8:- type = GGML_TYPE_Q8_0;- break;- default:- fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype);- return false;- };+ assert(itype < GGML_TYPE_COUNT);+ type = static_cast<ggml_type>(itype);auto * ctx_clip = clip_model_load(fname_inp, 2);@@ -1012,6 +1299,10 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const iif (quantize) {new_type = type;+ if (new_type >= GGML_TYPE_Q2_K && name.find("embd") != std::string::npos) {+ new_type = GGML_TYPE_Q8_0; // ggml_get_rows needs non K type+ // fprintf(stderr, "%s: quantizing %s to %s\n", __func__, name.c_str(), ggml_type_name(new_type));+ }const size_t n_elms = ggml_nelements(cur);float * f32_data;@@ -1056,6 +1347,21 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const icase GGML_TYPE_Q8_0: {new_size = ggml_quantize_q8_0(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());} break;+ case GGML_TYPE_Q2_K: {+ new_size = ggml_quantize_q2_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());+ } break;+ case GGML_TYPE_Q3_K: {+ new_size = ggml_quantize_q3_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());+ } break;+ case GGML_TYPE_Q4_K: {+ new_size = ggml_quantize_q4_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());+ } break;+ case GGML_TYPE_Q5_K: {+ new_size = ggml_quantize_q5_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());+ } break;+ case GGML_TYPE_Q6_K: {+ new_size = ggml_quantize_q6_K(f32_data, new_data, n_elms, cur->ne[0], hist_cur.data());+ } break;default: {fprintf(stderr, "%s: unsupported quantization type %d\n", __func__, new_type);return false;@@ -1082,7 +1388,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i}tinylogf("%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), ggml_n_dims(cur), quantize,- orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);+ orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);}// go back to beginning of file and write the updated metadata@@ -1116,13 +1422,25 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i}int clip_n_mmproj_embd(const struct clip_ctx * ctx) {- return ctx->vision_model.mm_2_b->ne[0];+ if (ctx->proj_type == PROJECTOR_TYPE_LDP) {+ return ctx->vision_model.mm_model_block_1_block_2_1_b->ne[0];+ }+ else if (ctx->proj_type == PROJECTOR_TYPE_MLP) {+ return ctx->vision_model.mm_2_b->ne[0];+ }+ else {+ std::string proj_type = PROJECTOR_TYPE_NAMES[ctx->proj_type];+ throw std::runtime_error(format("%s: don't support projector with: %s currently\n", __func__, proj_type.c_str()));+ }}int clip_n_patches(const struct clip_ctx * ctx) {auto & params = ctx->vision_model.hparams;-- return (params.image_size / params.patch_size) * (params.image_size / params.patch_size);+ int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size);+ if (ctx->proj_type == PROJECTOR_TYPE_LDP) {+ n_patches /= 4;+ }+ return n_patches;}size_t clip_embd_nbytes(const struct clip_ctx * ctx) {diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex 738ad963..bf32a01c 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -236,20 +236,14 @@ static void llava_free(struct llava_context * ctx_llava) {llama_backend_free();}-int llava_cli(int argc, char ** argv) {- gpt_params params;+int llava_cli(int argc, char ** argv, gpt_params * params) {- if (!gpt_params_parse(argc, argv, params)) {- show_additional_info(argc, argv);- return 1;- }-- if (params.mmproj.empty()) {+ if (params->mmproj.empty()) {fprintf(stderr, "%s: fatal error: --mmproj must also be passed when an --image is specified in cli mode\n", argv[0]);return 1;}- auto ctx_llava = llava_init(¶ms);+ auto ctx_llava = llava_init(params);if (ctx_llava == NULL) {fprintf(stderr, "%s: error: failed to init llava\n", __func__);return 1;@@ -262,13 +256,13 @@ int llava_cli(int argc, char ** argv) {sa.sa_flags = 0;sigaction(SIGINT, &sa, NULL);- auto image_embed = load_image(ctx_llava, ¶ms);+ auto image_embed = load_image(ctx_llava, params);if (!image_embed) {return 1;}// process the prompt- process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);+ process_prompt(ctx_llava, image_embed, params, params->prompt);llama_print_timings(ctx_llava->ctx_llama);diff --git a/llama.cpp/llava/llava.h b/llama.cpp/llava/llava.hindex 045a4036..b0948f04 100644--- a/llama.cpp/llava/llava.h+++ b/llama.cpp/llava/llava.h@@ -43,7 +43,7 @@ LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);/** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);-LLAVA_API int llava_cli(int argc, char ** argv);+LLAVA_API int llava_cli(int argc, char ** argv, gpt_params * params);#ifdef __cplusplus}diff --git a/llama.cpp/log.h b/llama.cpp/log.hindex 7916083e..3f8c1be4 100644--- a/llama.cpp/log.h+++ b/llama.cpp/log.h@@ -66,13 +66,13 @@// #define LOG_TARGET stderr// #include "log.h"//-// The log target can also be redirected to a diffrent function+// The log target can also be redirected to a different function// like so://-// #define LOG_TARGET log_handler_diffrent()+// #define LOG_TARGET log_handler_different()// #include "log.h"//-// FILE* log_handler_diffrent()+// FILE* log_handler_different()// {// return stderr;// }@@ -426,7 +426,7 @@ inline FILE *log_handler2_impl(bool change = false, LogTriState append = LogTriS// Disables logs entirely at runtime.// Makes LOG() and LOG_TEE() produce no output,-// untill enabled back.+// until enabled back.#define log_disable() log_disable_impl()// INTERNAL, DO NOT USEdiff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 6b05d6c7..67c497ff 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -84,10 +84,24 @@ Number of threads to use during generation..PpDefault: $(nproc)/2.It Fl tb Ar N , Fl Fl threads-batch Ar N-Number of threads to use during batch and prompt processing.+Set the number of threads to use during batch and prompt processing. In+some systems, it is beneficial to use a higher number of threads during+batch processing than during generation. If not specified, the number of+threads used for batch processing will be the same as the number of+threads used for generation.+.Pp+Default: Same as+.Fl Fl threads+.It Fl td Ar N , Fl Fl threads-draft Ar N+Number of threads to use during generation..PpDefault: Same as.Fl Fl threads+.It Fl tbd Ar N , Fl Fl threads-batch-draft Ar N+Number of threads to use during batch and prompt processing.+.Pp+Default: Same as+.Fl Fl threads-draft.It Fl Fl in-prefix-bosPrefix BOS to user inputs, preceding the.Fl Fl in-prefix@@ -452,8 +466,30 @@ Never use GPU and instead use CPU inference. This setting is implied byNumber of layers to store in VRAM..It Fl ngld Ar N , Fl Fl n-gpu-layers-draft Ar NNumber of layers to store in VRAM for the draft model.+.It Fl sm Ar SPLIT_MODE , Fl Fl split-mode Ar SPLIT_MODE+How to split the model across multiple GPUs, one of:+.Bl -dash -compact+.It+none: use one GPU only+.It+layer (default): split layers and KV across GPUs+.It+row: split rows across GPUs+.El.It Fl ts Ar SPLIT , Fl Fl tensor-split Ar SPLIT-How to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1++When using multiple GPUs this option controls how large tensors should+be split across all GPUs.+.Ar SPLIT+is a comma-separated list of non-negative values that assigns the+proportion of data that each GPU should get in order. For example,+\[dq]3,2\[dq] will assign 60% of the data to GPU 0 and 40% to GPU 1. By+default the data is split in proportion to VRAM but this may not be+optimal for performance. Requires cuBLAS.++How to split tensors across multiple GPUs, comma-separated list of+proportions, e.g. 3,1+.It Fl mg Ar i , Fl Fl main-gpu Ar iThe GPU to use for scratch and small tensors..It Fl nommq , Fl Fl no-mul-mat-q@@ -495,6 +531,37 @@ Default: 1Group-attention width..PpDefault: 512+.It Fl bf Ar FNAME , Fl Fl binary-file Ar FNAME+Binary file containing multiple choice tasks.+.It Fl Fl winogrande+Compute Winogrande score over random tasks from datafile supplied by the+.Fl f+flag.+.It Fl Fl winogrande-tasks Ar N+Number of tasks to use when computing the Winogrande score.+.Pp+Default: 0+.It Fl Fl multiple-choice+Compute multiple choice score over random tasks from datafile supplied+by the+.Fl f+flag.+.It Fl Fl multiple-choice-tasks Ar N+Number of tasks to use when computing the multiple choice score.+.Pp+Default: 0+.It Fl Fl kl-divergence+Computes KL-divergence to logits provided via the+.Fl Fl kl-divergence-base+flag.+.It Fl Fl save-all-logits Ar FNAME , Fl Fl kl-divergence-base Ar FNAME+Save logits to filename.+.It Fl ptc Ar N , Fl Fl print-token-count Ar N+Print token count every+.Ar N+tokens.+.Pp+Default: -1.El.Sh CLI OPTIONSThe following options may be specified when@@ -532,7 +599,7 @@ generating text. For example, the grammar:.Ppwill force the LLM to only output yes or no before exiting. This isuseful for shell scripts when the-.Fl Fl silent-prompt+.Fl Fl no-display-promptflag is also supplied..It Fl Fl grammar-file Ar FNAMEFile to read grammar from.@@ -587,7 +654,7 @@ flag to add a space or any other characters after the reverse prompt..It Fl Fl colorEnable colorized output to differentiate visually distinguishing betweenprompts, user input, and generated text.-.It Fl Fl silent-prompt+.It Fl Fl no-display-prompt , Fl Fl silent-promptDon't echo the prompt itself to standard output..It Fl Fl multiline-inputAllows you to write or paste multiple lines without ending each in '\[rs]'.@@ -715,7 +782,7 @@ Here's an example of how you can use llamafile to summarize HTML URLs:-c 0 \[rs]--temp 0 \[rs]-n 500 \[rs]- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/null.Ed.PpHere's how you can use llamafile to describe a jpg/png/gif/bmp image:@@ -725,7 +792,7 @@ llamafile --temp 0 \[rs]-m llava-v1.5-7b-Q4_K.gguf \[rs]--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]-e -p '### User: What do you see?\[rs]n### Assistant: ' \[rs]- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/null.Ed.PpIf you wanted to write a script to rename all your image files, you@@ -737,7 +804,7 @@ llamafile --temp 0 \[rs]--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]--grammar 'root ::= [a-z]+ (" " [a-z]+)+' \[rs]-e -p '### User: The image has...\[rs]n### Assistant: ' \[rs]- --silent-prompt 2>/dev/null |+ --no-display-prompt 2>/dev/null |sed -e's/ /_/g' -e's/$/.jpg/'three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg.Eddiff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascindex c933c374..8475833a 100644--- a/llama.cpp/main/main.1.asc+++ b/llama.cpp/main/main.1.asc@@ -59,10 +59,24 @@ OOPPTTIIOONNSSDefault: $(nproc)/2--ttbb _N, ----tthhrreeaaddss--bbaattcchh _N- Number of threads to use during batch and prompt processing.+ Set the number of threads to use during batch and prompt process‐+ ing. In some systems, it is beneficial to use a higher number of+ threads during batch processing than during generation. If not+ specified, the number of threads used for batch processing will+ be the same as the number of threads used for generation.Default: Same as ----tthhrreeaaddss+ --ttdd _N, ----tthhrreeaaddss--ddrraafftt _N+ Number of threads to use during generation.++ Default: Same as ----tthhrreeaaddss++ --ttbbdd _N, ----tthhrreeaaddss--bbaattcchh--ddrraafftt _N+ Number of threads to use during batch and prompt processing.++ Default: Same as ----tthhrreeaaddss--ddrraafftt+----iinn--pprreeffiixx--bboossPrefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.@@ -407,7 +421,22 @@ OOPPTTIIOONNSS--nngglldd _N, ----nn--ggppuu--llaayyeerrss--ddrraafftt _NNumber of layers to store in VRAM for the draft model.+ --ssmm _S_P_L_I_T___M_O_D_E, ----sspplliitt--mmooddee _S_P_L_I_T___M_O_D_E+ How to split the model across multiple GPUs, one of:+ -- none: use one GPU only+ -- layer (default): split layers and KV across GPUs+ -- row: split rows across GPUs+--ttss _S_P_L_I_T, ----tteennssoorr--sspplliitt _S_P_L_I_T++ When using multiple GPUs this option controls how large tensors+ should be split across all GPUs. _S_P_L_I_T is a comma-separated list+ of non-negative values that assigns the proportion of data that+ each GPU should get in order. For example, "3,2" will assign 60%+ of the data to GPU 0 and 40% to GPU 1. By default the data is+ split in proportion to VRAM but this may not be optimal for per‐+ formance. Requires cuBLAS.+How to split tensors across multiple GPUs, comma-separated listof proportions, e.g. 3,1@@ -471,6 +500,39 @@ OOPPTTIIOONNSSDefault: 512+ --bbff _F_N_A_M_E, ----bbiinnaarryy--ffiillee _F_N_A_M_E+ Binary file containing multiple choice tasks.++ ----wwiinnooggrraannddee+ Compute Winogrande score over random tasks from datafile supplied+ by the --ff flag.++ ----wwiinnooggrraannddee--ttaasskkss _N+ Number of tasks to use when computing the Winogrande score.++ Default: 0++ ----mmuullttiippllee--cchhooiiccee+ Compute multiple choice score over random tasks from datafile+ supplied by the --ff flag.++ ----mmuullttiippllee--cchhooiiccee--ttaasskkss _N+ Number of tasks to use when computing the multiple choice score.++ Default: 0++ ----kkll--ddiivveerrggeennccee+ Computes KL-divergence to logits provided via the+ ----kkll--ddiivveerrggeennccee--bbaassee flag.++ ----ssaavvee--aallll--llooggiittss _F_N_A_M_E, ----kkll--ddiivveerrggeennccee--bbaassee _F_N_A_M_E+ Save logits to filename.++ --ppttcc _N, ----pprriinntt--ttookkeenn--ccoouunntt _N+ Print token count every _N tokens.++ Default: -1+CCLLII OOPPTTIIOONNSSThe following options may be specified when llllaammaaffiillee is running in ----cclliimode.@@ -505,8 +567,8 @@ CCLLII OOPPTTIIOONNSSroot ::= "yes" | "no"will force the LLM to only output yes or no before exiting. This- is useful for shell scripts when the ----ssiilleenntt--pprroommpptt flag is also- supplied.+ is useful for shell scripts when the ----nnoo--ddiissppllaayy--pprroommpptt flag is+ also supplied.----ggrraammmmaarr--ffiillee _F_N_A_M_EFile to read grammar from.@@ -570,7 +632,7 @@ CCLLII OOPPTTIIOONNSSEnable colorized output to differentiate visually distinguishingbetween prompts, user input, and generated text.- ----ssiilleenntt--pprroommpptt+ ----nnoo--ddiissppllaayy--pprroommpptt, ----ssiilleenntt--pprroommppttDon't echo the prompt itself to standard output.----mmuullttiilliinnee--iinnppuutt@@ -706,7 +768,7 @@ EEXXAAMMPPLLEESS-c 0 \--temp 0 \-n 500 \- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/nullHere's how you can use llamafile to describe a jpg/png/gif/bmp image:@@ -715,7 +777,7 @@ EEXXAAMMPPLLEESS-m llava-v1.5-7b-Q4_K.gguf \--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \-e -p '### User: What do you see?\n### Assistant: ' \- --silent-prompt 2>/dev/null+ --no-display-prompt 2>/dev/nullIf you wanted to write a script to rename all your image files, you coulduse the following command to generate a safe filename:@@ -726,7 +788,7 @@ EEXXAAMMPPLLEESS--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \--grammar 'root ::= [a-z]+ (" " [a-z]+)+' \-e -p '### User: The image has...\n### Assistant: ' \- --silent-prompt 2>/dev/null |+ --no-display-prompt 2>/dev/null |sed -e's/ /_/g' -e's/$/.jpg/'three_baby_lemurs_on_the_back_of_an_adult_lemur.jpgdiff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex f9fec061..560b510f 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -126,10 +126,6 @@ int main(int argc, char ** argv) {return server_cli(argc, argv);}- if (llamafile_has(argv, "--image")) {- return llava_cli(argc, argv);- }-gpt_params params;g_params = ¶ms;@@ -145,6 +141,11 @@ int main(int argc, char ** argv) {llama_log_set(llama_log_callback_logTee, nullptr);#endif // LOG_DISABLE_LOGS+ if (!params.image.empty() ||+ params.prompt.find("<img src=\"") != std::string::npos) {+ return llava_cli(argc, argv, ¶ms);+ }+// TODO: Dump params ?//LOG("Params perplexity: %s\n", LOG_TOSTR(params.perplexity));@@ -153,7 +154,7 @@ int main(int argc, char ** argv) {console::init(params.simple_io, params.use_color);atexit([]() { console::cleanup(); });- if (!params.unsecure && !llamafile_gpu_supported()) {+ if (!params.unsecure && !llamafile_has_gpu()) {// Enable pledge() security on Linux and OpenBSD.// - We do this *after* opening the log file for writing.// - We do this *before* loading any weights or graphdefs.@@ -514,7 +515,8 @@ int main(int argc, char ** argv) {}bool is_antiprompt = false;- bool input_echo = !params.silent_prompt;+ bool input_echo = true;+ bool display = true;bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < embd_inp.size();int n_past = 0;@@ -529,6 +531,7 @@ int main(int argc, char ** argv) {// the first thing we will do is to output the prompt, so set color accordinglyconsole::set_display(console::prompt);+ display = params.display_prompt;std::vector<llama_token> embd;std::vector<llama_token> embd_guidance;@@ -538,7 +541,7 @@ int main(int argc, char ** argv) {while ((n_remain != 0 && !is_antiprompt) || params.interactive) {// predictif (!embd.empty()) {- // Note: n_ctx - 4 here is to match the logic for commandline prompt handling via+ // Note: (n_ctx - 4) here is to match the logic for commandline prompt handling via// --prompt or --file which uses the same value.int max_embd_size = n_ctx - 4;@@ -688,6 +691,10 @@ int main(int argc, char ** argv) {n_past += n_eval;LOG("n_past = %d\n", n_past);+ // Display total tokens alongside total time+ if (params.n_print > 0 && n_past % params.n_print == 0) {+ LOG_TEE("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx);+ }}if (!embd.empty() && !path_session.empty()) {@@ -741,7 +748,7 @@ int main(int argc, char ** argv) {}// display text- if (input_echo) {+ if (input_echo && display) {for (auto id : embd) {const std::string token_str = llama_token_to_piece(ctx, id);printf("%s", token_str.c_str());@@ -758,6 +765,7 @@ int main(int argc, char ** argv) {// reset color to default if there is no pending user inputif (input_echo && (int) embd_inp.size() == n_consumed) {console::set_display(console::reset);+ display = true;}// if not currently processing queued inputs;@@ -807,11 +815,6 @@ int main(int argc, char ** argv) {printf("\n");} else if (params.instruct || params.chatml) {is_interacting = true;- } else if (params.silent_prompt) {- // --silent-prompt usually goes with `./main 2>/dev/null` and- // doing that usually causes the llm terminal output to not- // include a trailing newline. that's an unpleasant ux.- printf("\n");}}@@ -835,6 +838,7 @@ int main(int argc, char ** argv) {// color user input onlyconsole::set_display(console::user_input);+ display = params.display_prompt;std::string line;bool another_line = true;@@ -845,6 +849,7 @@ int main(int argc, char ** argv) {// done taking input, reset colorconsole::set_display(console::reset);+ display = true;// Add tokens to embd only if the input buffer is non-empty// Entering a empty line lets the user pass control back@@ -932,6 +937,9 @@ int main(int argc, char ** argv) {}}+ // ensure trailing newline+ printf("\n");+if (!path_session.empty() && params.prompt_cache_all && !params.prompt_cache_ro) {LOG_TEE("\n%s: saving final output to session file '%s'\n", __func__, path_session.c_str());llama_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());diff --git a/llama.cpp/perplexity/README.md b/llama.cpp/perplexity/README.mdnew file mode 100644index 00000000..50e1af01--- /dev/null+++ b/llama.cpp/perplexity/README.md@@ -0,0 +1,21 @@+# perplexity++TODO++## Llama 2 70B Scorechart+Quantization | Model size (GiB) | Perplexity | Delta to fp16+-- | -- | -- | --+Q4_0 | 36.20 | 3.5550 | 3.61%+Q4_1 | 40.20 | 3.5125 | 2.37%+Q5_0 | 44.20 | 3.4744 | 1.26%+Q2_K | 27.27 | 3.7339 | 8.82%+Q3_K_S | 27.86 | 3.7019 | 7.89%+Q3_K_M | 30.83 | 3.5932 | 4.72%+Q3_K_L | 33.67 | 3.5617 | 3.80%+Q4_K_S | 36.39 | 3.4852 | 1.57%+Q4_K_M | 38.54 | 3.4725 | 1.20%+Q5_K_S | 44.20 | 3.4483 | 0.50%+Q5_K_M | 45.41 | 3.4451 | 0.40%+Q6_K | 52.70 | 3.4367 | 0.16%+fp16 | 128.5 | 3.4313 | -+diff --git a/llama.cpp/perplexity/perplexity.cpp b/llama.cpp/perplexity/perplexity.cppindex b9f9d479..b9041ec6 100644--- a/llama.cpp/perplexity/perplexity.cpp+++ b/llama.cpp/perplexity/perplexity.cpp@@ -10,11 +10,11 @@#include <sstream>#include <thread>#include <mutex>+#include <atomic>#include <vector>--#if defined(_MSC_VER)-#pragma warning(disable: 4244 4267) // possible loss of data-#endif+#include <array>+#include <fstream>+#include <sstream>struct results_perplexity {std::vector<llama_token> tokens;@@ -110,6 +110,43 @@ static results_log_softmax log_softmax(int n_vocab, const float * logits, int toreturn {logits[tok] - max_logit - log(sum_exp), logits[tok], expf(logits[tok] - max_logit) / (float) sum_exp};}+static inline int nearest_int(float fval) {+ //assert(fval <= 4194303.f);+ float val = fval + 12582912.f;+ int i; memcpy(&i, &val, sizeof(int));+ return (i & 0x007fffff) - 0x00400000;+}++static double log_softmax(int n_vocab, const float * logits, uint16_t * log_prob, int tok) {+ float max_logit = logits[0];+ float min_logit = logits[0];+ for (int i = 1; i < n_vocab; ++i) {+ max_logit = std::max(max_logit, logits[i]);+ min_logit = std::min(min_logit, logits[i]);+ }+ min_logit = std::max(min_logit, max_logit - 16);+ double sum_exp = 0.0;+ for (int i = 0; i < n_vocab; ++i) {+ sum_exp += expf(logits[i] - max_logit);+ }+ const float log_sum_exp = log(sum_exp);+ const float min_log_prob = min_logit - max_logit - log_sum_exp;+ const float scale = (max_logit - min_logit)/65535.f;+ float * d = (float *)log_prob;+ d[0] = scale;+ d[1] = min_log_prob;+ log_prob += 4;+ if (scale) {+ const float inv_scale = 1/scale;+ for (int i = 0; i < n_vocab; ++i) {+ log_prob[i] = logits[i] > min_logit ? nearest_int(inv_scale*(logits[i] - min_logit)) : 0;+ }+ } else {+ std::memset(log_prob, 0, n_vocab*sizeof(uint16_t));+ }+ return max_logit + log_sum_exp - logits[tok];+}+static void process_logits(int n_vocab, const float * logits, const int * tokens, int n_token, std::vector<std::thread> & workers,double & nll, double & nll2, float * logit_history, float * prob_history@@ -145,6 +182,130 @@ static void process_logits(}}+static void process_logits(std::ostream& out, int n_vocab, const float * logits, const int * tokens, int n_token,+ std::vector<std::thread> & workers, std::vector<uint16_t> & log_probs, double & nll, double & nll2) {+ std::mutex mutex;+ const int nv = 2*((n_vocab + 1)/2) + 4;+ int counter = 0;+ auto compute = [&mutex, &counter, &log_probs, &nll, &nll2, n_vocab, logits, tokens, n_token, nv] () {+ double local_nll = 0;+ double local_nll2 = 0;+ while (true) {+ std::unique_lock<std::mutex> lock(mutex);+ int i = counter++;+ if (i >= n_token) {+ nll += local_nll; nll2 += local_nll2;+ break;+ }+ lock.unlock();+ const double v = log_softmax(n_vocab, logits + i*n_vocab, log_probs.data() + i*nv, tokens[i+1]);+ local_nll += v;+ local_nll2 += v*v;+ }+ };+ for (auto & w : workers) {+ w = std::thread(compute);+ }+ compute();+ for (auto & w : workers) {+ w.join();+ }+ out.write((const char *)log_probs.data(), n_token*nv*sizeof(uint16_t));+}++struct kl_divergence_result {+ double sum_nll = 0;+ double sum_nll2 = 0;+ double sum_kld = 0;+ double sum_kld2 = 0;+ double sum_nll_diff = 0;+ double sum_nll_diff2 = 0;+ size_t n_same_top = 0;+ size_t count = 0;+};++static double log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {+ float max_logit = logits[0];+ int imax = 0;+ for (int i = 1; i < n_vocab; ++i) {+ if (logits[i] > max_logit) {+ max_logit = logits[i];+ imax = i;+ }+ }+ double sum_exp = 0.0;+ for (int i = 0; i < n_vocab; ++i) {+ sum_exp += expf(logits[i] - max_logit);+ }+ const float log_sum_exp = log(sum_exp);+ const float * d = (const float *)base_log_prob;+ const float scale = d[0];+ const float min_log_prob = d[1];+ base_log_prob += 4;+ float nll = max_logit + log_sum_exp - logits[tok];+ kld.sum_nll += nll;+ kld.sum_nll2 += nll*nll;+ nll += (scale*base_log_prob[tok] + min_log_prob);+ kld.sum_nll_diff += nll;+ kld.sum_nll_diff2 += nll*nll;+ max_logit += log_sum_exp;+ double sum = 0;+ int imax_base = -1;+ float p_log_base_max = 0;+ for (int i = 0; i < n_vocab; ++i) {+ const float p_log_base = scale*base_log_prob[i] + min_log_prob;+ if (i == 0 || p_log_base > p_log_base_max) {+ p_log_base_max = p_log_base;+ imax_base = i;+ }+ if (p_log_base > -16.f) {+ const float p_base = expf(p_log_base);+ sum += p_base * (p_log_base - logits[i] + max_logit);+ }+ }+ kld.sum_kld += sum;+ kld.sum_kld2 += sum*sum;+ ++kld.count;+ if (imax == imax_base) ++kld.n_same_top;+ return sum;+}++static void process_logits(int n_vocab, const float * logits, const int * tokens, int n_token,+ std::vector<std::thread> & workers, const std::vector<uint16_t> & base_log_probs, kl_divergence_result & kld,+ float * kld_values) {+ std::mutex mutex;+ const int nv = 2*((n_vocab + 1)/2) + 4;+ int counter = 0;+ auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values] () {+ kl_divergence_result local_kld;+ while (true) {+ std::unique_lock<std::mutex> lock(mutex);+ int i = counter++;+ if (i >= n_token) {+ kld.sum_nll += local_kld.sum_nll;+ kld.sum_nll2 += local_kld.sum_nll2;+ kld.sum_kld += local_kld.sum_kld;+ kld.sum_kld2 += local_kld.sum_kld2;+ kld.sum_nll_diff += local_kld.sum_nll_diff;+ kld.sum_nll_diff2 += local_kld.sum_nll_diff2;+ kld.n_same_top += local_kld.n_same_top;+ kld.count += local_kld.count;+ break;+ }+ lock.unlock();+ double v = log_softmax(n_vocab, logits + i*n_vocab, base_log_probs.data() + i*nv, tokens[i+1], local_kld);+ kld_values[i] = (float)v;+ }+ };+ for (auto & w : workers) {+ w = std::thread(compute);+ }+ compute();+ for (auto & w : workers) {+ w.join();+ }+}+static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & params) {// Download: https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip?ref=salesforce-research// Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw`@@ -292,6 +453,18 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parconst bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));const int n_ctx = llama_n_ctx(ctx);+ std::ofstream logits_stream;+ if (!params.logits_file.empty()) {+ logits_stream.open(params.logits_file.c_str());+ if (!logits_stream.is_open()) {+ fprintf(stderr, "%s: failed to open %s for writing\n", __func__, params.logits_file.c_str());+ return {};+ }+ fprintf(stderr, "%s: saving all logits to %s\n", __func__, params.logits_file.c_str());+ logits_stream.write("_logits_", 8);+ logits_stream.write((const char *)&n_ctx, sizeof(n_ctx));+ }+auto tim1 = std::chrono::high_resolution_clock::now();fprintf(stderr, "%s: tokenizing the input ..\n", __func__);@@ -323,18 +496,30 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & pardouble nll = 0.0;double nll2 = 0.0;+ const int num_batches = (n_ctx + n_batch - 1) / n_batch;++ std::vector<float> logits;+ if (num_batches > 1) {+ logits.reserve((size_t)n_ctx * n_vocab);+ }+fprintf(stderr, "%s: calculating perplexity over %d chunks, batch_size=%d\n", __func__, n_chunk, n_batch);std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);+ std::vector<uint16_t> log_probs;+ if (!params.logits_file.empty()) {+ logits_stream.write((const char *)&n_vocab, sizeof(n_vocab));+ logits_stream.write((const char *)&n_chunk, sizeof(n_chunk));+ logits_stream.write((const char *)tokens.data(), n_chunk*n_ctx*sizeof(tokens[0]));+ const int nv = 2*((n_vocab + 1)/2) + 4;+ log_probs.resize(n_ctx * nv);+ }+for (int i = 0; i < n_chunk; ++i) {const int start = i * n_ctx;const int end = start + n_ctx;- const int num_batches = (n_ctx + n_batch - 1) / n_batch;-- std::vector<float> logits;-const auto t_start = std::chrono::high_resolution_clock::now();// clear the KV cache@@ -360,8 +545,10 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par// restore the original token in case it was set to BOStokens[batch_start] = token_org;- const auto * batch_logits = llama_get_logits(ctx);- logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);+ if (num_batches > 1) {+ const auto * batch_logits = llama_get_logits(ctx);+ logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);+ }}const auto t_end = std::chrono::high_resolution_clock::now();@@ -390,8 +577,14 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par// last 256 tokens. Then, we split the input up into context window size chunks to// process the entire prompt.const int first = n_ctx/2;- process_logits(n_vocab, logits.data() + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,- workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first);+ const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);+ if (!params.logits_file.empty()) {+ process_logits(logits_stream, n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,+ workers, log_probs, nll, nll2);+ } else {+ process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,+ workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first);+ }count += n_ctx - first - 1;// perplexity is e^(average negative log-likelihood)@@ -404,6 +597,8 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parprintf("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2);}fflush(stdout);++ logits.clear();}printf("\n");@@ -421,26 +616,73 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & parreturn {tokens, ppl, logit_history, prob_history};}-static std::vector<float> hellaswag_evaluate_tokens(- llama_context * ctx, std::vector<int> & tokens, int n_past, int n_batch, int n_vocab-) {- std::vector<float> result;- result.reserve(tokens.size() * n_vocab);- size_t n_chunk = (tokens.size() + n_batch - 1)/n_batch;- for (size_t i_chunk = 0; i_chunk < n_chunk; ++i_chunk) {- size_t n_tokens = tokens.size() - i_chunk * n_batch;- n_tokens = std::min(n_tokens, size_t(n_batch));- if (llama_decode(ctx, llama_batch_get_one(tokens.data() + i_chunk * n_batch, n_tokens, n_past, 0))) {- fprintf(stderr, "%s : failed to eval\n", __func__);- return {};+static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<float> & batch_logits, int32_t n_batch, int32_t n_vocab) {+ for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {+ const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));++ llama_batch batch_view = {+ n_tokens,+ batch.token + i,+ nullptr,+ batch.pos + i,+ batch.n_seq_id + i,+ batch.seq_id + i,+ batch.logits + i,+ 0, 0, 0, // unused+ };++ const int ret = llama_decode(ctx, batch_view);+ if (ret != 0) {+ LOG_TEE("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret);+ return false;}- const auto logits = llama_get_logits(ctx);- result.insert(result.end(), logits, logits + n_tokens * n_vocab);+ memcpy(batch_logits.data() + i*n_vocab, llama_get_logits(ctx), n_tokens*n_vocab*sizeof(float));+ }++ return true;+}++#define K_TOKEN_CHUNK 4- n_past += n_tokens;+static void compute_logprobs(const float * batch_logits, int n_vocab, std::vector<std::thread>& workers,+ const std::vector<std::pair<size_t, llama_token>>& eval_pairs, std::vector<float>& eval_results) {+ if (eval_results.size() != eval_pairs.size()) {+ eval_results.resize(eval_pairs.size());+ }+ if (eval_pairs.empty()) return;++ size_t max_threads = std::min((eval_pairs.size() + K_TOKEN_CHUNK - 1)/K_TOKEN_CHUNK, workers.size());++ std::atomic<int> counter(0);+ auto compute = [&counter, &eval_pairs, &eval_results, batch_logits, n_vocab] () {+ float local_logprobs[K_TOKEN_CHUNK];+ while (true) {+ size_t first = counter.fetch_add(K_TOKEN_CHUNK, std::memory_order_relaxed);+ if (first >= eval_results.size()) break;+ size_t last = std::min(first + K_TOKEN_CHUNK, eval_results.size());+ for (size_t i = first; i < last; ++i) {+ auto logits = batch_logits + eval_pairs[i].first * n_vocab;+ float max_logit = logits[0];+ for (int j = 1; j < n_vocab; ++j) {+ max_logit = std::max(max_logit, logits[j]);+ }+ float sum_p = 0.f;+ for (int j = 0; j < n_vocab; ++j) {+ sum_p += expf(logits[j] - max_logit);+ }+ local_logprobs[i - first] = logits[eval_pairs[i].second] - max_logit - std::log(sum_p);+ }+ std::memcpy(eval_results.data() + first, local_logprobs, (last - first)*sizeof(float));+ }+ };++ for (size_t it = 0; it < max_threads; ++it) {+ workers[it] = std::thread(compute);+ }+ for (size_t it = 0; it < max_threads; ++it) {+ workers[it].join();}- return result;}static void hellaswag_score(llama_context * ctx, const gpt_params & params) {@@ -469,7 +711,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {prompt_lines.push_back(line);}- if( prompt_lines.size() % 6 != 0) {+ if (prompt_lines.size() % 6 != 0) {fprintf(stderr, "%s : number of lines in prompt not a multiple of 6.\n", __func__);return;}@@ -483,14 +725,14 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {// This is needed as usual for LLaMA modelsconst bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));+ // The tasks should be randomized so the score stabilizes quickly.+ bool randomize_tasks = true;+// Number of tasks to use when computing the score- if ( params.hellaswag_tasks < hs_task_count ) {+ if (params.hellaswag_tasks < hs_task_count) {hs_task_count = params.hellaswag_tasks;}- // The tasks should be randomized so the score stabilizes quickly.- bool randomize_tasks = true;-// The random seed should not impact the final result if the computation is done over enough tasks, so kept hardcoded for nowstd::mt19937 rng(1);@@ -501,27 +743,53 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {std::string ending[4];size_t ending_logprob_count[4];double ending_logprob[4];++ size_t i_batch; // starting index in the llama_batch+ size_t common_prefix; // max number of initial tokens that are the same in all sentences+ size_t required_tokens; // needed number of tokens to evaluate all 4 endings+ std::vector<llama_token> seq_tokens[4];};fprintf(stderr, "%s : selecting %zu %s tasks.\n", __func__, hs_task_count, (randomize_tasks?"randomized":"the first") );// Select and read data from prompt lines- hs_data_t *hs_data = new hs_data_t[hs_task_count];- for (size_t i=0; i < hs_task_count; i++) {+ std::vector<hs_data_t> hs_data(hs_task_count);+ for (size_t i = 0; i < hs_task_count; i++) {size_t idx = i;+ auto & hs_cur = hs_data[i];+// Select a random example of those left in the promptif (randomize_tasks) {std::uniform_int_distribution<size_t> dist(0, prompt_lines.size()/6-1 ) ;idx = dist(rng);}- hs_data[i].context = prompt_lines[idx*6];- hs_data[i].gold_ending_idx = std::stoi( prompt_lines[idx*6+1] );- for (size_t j=0; j < 4; j++) {- hs_data[i].ending[j] = prompt_lines[idx*6+2+j];+ hs_cur.context = prompt_lines[idx*6];+ hs_cur.gold_ending_idx = std::stoi( prompt_lines[idx*6+1] );+ for (size_t j = 0; j < 4; j++) {+ hs_cur.ending[j] = prompt_lines[idx*6+2+j];+ hs_cur.seq_tokens[j] = ::llama_tokenize(ctx, hs_cur.context + " " + hs_cur.ending[j], add_bos);}+ // determine the common prefix of the endings+ hs_cur.common_prefix = 0;+ for (size_t k = 0; k < hs_cur.seq_tokens[0].size(); k++) {+ if (hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[1][k] ||+ hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[2][k] ||+ hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[3][k]) {+ break;+ }+ hs_cur.common_prefix++;+ }+ hs_cur.required_tokens = hs_cur.common_prefix ++ hs_cur.seq_tokens[0].size() - hs_cur.common_prefix ++ hs_cur.seq_tokens[1].size() - hs_cur.common_prefix ++ hs_cur.seq_tokens[2].size() - hs_cur.common_prefix ++ hs_cur.seq_tokens[3].size() - hs_cur.common_prefix;++ //GGML_ASSERT(hs_cur.common_prefix >= ::llama_tokenize(ctx, hs_cur.context, add_bos).size());+// Delete the selected random example from the promptif (randomize_tasks) {prompt_lines.erase( std::next(prompt_lines.begin(),idx*6) , std::next(prompt_lines.begin(),idx*6+6) );@@ -529,152 +797,984 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {}fprintf(stderr, "%s : calculating hellaswag score over selected tasks.\n", __func__);+printf("\ntask\tacc_norm\n");double acc = 0.0f;+const int n_vocab = llama_n_vocab(llama_get_model(ctx));- const int n_ctx = llama_n_ctx(ctx);+ const int n_ctx = llama_n_ctx(ctx);+ const int n_batch = params.n_batch;++ const int max_tasks_per_batch = 32;+ const int max_seq = 4*max_tasks_per_batch;- std::vector<std::vector<int>> ending_tokens(4);+ llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);std::vector<float> tok_logits(n_vocab);+ std::vector<float> batch_logits(n_vocab*n_ctx);- for (size_t task_idx = 0; task_idx < hs_task_count; task_idx++) {- // Tokenize the context to count tokens- std::vector<int> context_embd = ::llama_tokenize(ctx, hs_data[task_idx].context, add_bos);- size_t context_size = context_embd.size();+ std::vector<std::pair<size_t, llama_token>> eval_pairs;+ std::vector<float> eval_results;+ std::vector<std::thread> workers(std::thread::hardware_concurrency());- for (int i = 0; i < 4; ++i) {- ending_tokens[i] = ::llama_tokenize(ctx, hs_data[task_idx].context + " " + hs_data[task_idx].ending[i], add_bos);- for (int k = 0; k < int(context_size); ++k) {- if (ending_tokens[i][k] != context_embd[k]) {- fprintf(stderr, "Oops: ending %d of task %d differs from context at position %d\n",i,int(task_idx),k);- break;+ for (size_t i0 = 0; i0 < hs_task_count; i0++) {+ int n_cur = 0;++ size_t i1 = i0;+ size_t i_batch = 0; // this tells us where in `llama_batch` we are currently++ llama_batch_clear(batch);++ // batch as much tasks as possible into the available context+ // each task has 4 unique seuqnce ids - one for each ending+ // the common prefix is shared among the 4 sequences to save tokens+ // we extract logits only from the last common token and from all ending tokens of each sequence+ while (n_cur + (int) hs_data[i1].required_tokens <= n_ctx) {+ auto & hs_cur = hs_data[i1];++ const int s0 = 4*(i1 - i0);+ if (s0 + 4 > max_seq) {+ break;+ }++ for (size_t i = 0; i < hs_cur.common_prefix; ++i) {+ llama_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false);+ }+ batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix++ for (int s = 0; s < 4; ++s) {+ for (size_t i = hs_cur.common_prefix; i < hs_cur.seq_tokens[s].size(); ++i) {+ llama_batch_add(batch, hs_cur.seq_tokens[s][i], i, { s0 + s }, true);}}++ hs_cur.i_batch = i_batch;+ i_batch += hs_cur.required_tokens;++ n_cur += hs_data[i1].required_tokens;+ if (++i1 == hs_task_count) {+ break;+ }+ }++ if (i0 == i1) {+ fprintf(stderr, "%s : task %zu does not fit in the context window\n", __func__, i0);+ return;}- // Do the 1st ending- // In this case we include the context when evaluating- //auto query_embd = ::llama_tokenize(ctx, hs_data[task_idx].context + hs_data[task_idx].ending[0], add_bos);- auto query_embd = ending_tokens[0];- auto query_size = query_embd.size();+ llama_kv_cache_clear(ctx);- // Stop if query wont fit the ctx window- if (query_size > (size_t)n_ctx) {- fprintf(stderr, "%s : number of tokens in query %zu > n_ctxl\n", __func__, query_size);+ // decode all tasks [i0, i1)+ if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) {+ fprintf(stderr, "%s: llama_decode() failed\n", __func__);return;}- // Speedup small evaluations by evaluating atleast 32 tokens- if (query_size < 32) {- query_embd.resize(32);+ // Compute log-probs in parallel+ // First we collect all tasks+ eval_pairs.clear();+ for (size_t i = i0; i < i1; ++i) {+ auto & hs_cur = hs_data[i];+ size_t li = hs_cur.common_prefix;+ for (int s = 0; s < 4; ++s) {+ for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) {+ eval_pairs.push_back(std::make_pair(hs_cur.i_batch + li++, hs_cur.seq_tokens[s][j + 1]));+ }+ ++li;+ }+ }+ // Then we do the actual calculation+ compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results);++ size_t ir = 0;++ // compute the logprobs for each ending of the decoded tasks+ for (size_t i = i0; i < i1; ++i) {+ auto & hs_cur = hs_data[i];++ std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*(hs_cur.i_batch + hs_cur.common_prefix - 1), n_vocab*sizeof(float));++ const auto first_probs = softmax(tok_logits);++ for (int s = 0; s < 4; ++s) {+ hs_cur.ending_logprob_count[s] = 1;+ hs_cur.ending_logprob[s] = std::log(first_probs[hs_cur.seq_tokens[s][hs_cur.common_prefix]]);+ for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) {+ hs_cur.ending_logprob[s] += eval_results[ir++];+ hs_cur.ending_logprob_count[s]++;+ }+ hs_cur.ending_logprob[s] /= hs_cur.ending_logprob_count[s];+ }++ // Find the ending with maximum logprob+ size_t ending_logprob_max_idx = 0;+ double ending_logprob_max_val = hs_cur.ending_logprob[0];+ for (size_t s = 1; s < 4; s++) {+ if (hs_cur.ending_logprob[s] > ending_logprob_max_val) {+ ending_logprob_max_idx = s;+ ending_logprob_max_val = hs_cur.ending_logprob[s];+ }+ }++ //printf("max logprob ending idx %lu, gold ending idx %lu\n", ending_logprob_max_idx, hs_cur.gold_ending_idx);++ // If the gold ending got the maximum logprobe add one accuracy point+ if (ending_logprob_max_idx == hs_cur.gold_ending_idx) {+ acc += 1.0;+ }++ // Print the accumulated accuracy mean x 100+ printf("%zu\t%.8lf\n", i + 1, acc/double(i + 1)*100.0);+ fflush(stdout);+ }++ i0 = i1 - 1;+ }++ llama_batch_free(batch);++ printf("\n");+}++struct winogrande_entry {+ std::string first;+ std::string second;+ std::array<std::string, 2> choices;+ int answer;++ size_t i_batch;+ size_t common_prefix;+ size_t required_tokens;+ size_t n_base1; // number of tokens for context + choice 1+ size_t n_base2; // number of tokens for context + choice 2+ std::vector<llama_token> seq_tokens[2];+};++static std::vector<winogrande_entry> load_winogrande_from_csv(const std::string& prompt) {+ std::vector<winogrande_entry> result;+ std::istringstream in(prompt);+ std::string line;+ std::array<int, 4> comma_pos;+ while (true) {+ std::getline(in, line);+ if (in.fail() || in.eof()) break;+ int ipos = 0;+ bool quote_open = false;+ for (int i = 0; i < int(line.size()); ++i) {+ if (!quote_open) {+ if (line[i] == ',') {+ comma_pos[ipos++] = i;+ if (ipos == 4) break;+ }+ else if (line[i] == '"') {+ quote_open = true;+ }+ }+ else {+ if (line[i] == '"') {+ quote_open = false;+ }+ }+ }+ if (ipos != 4) {+ printf("%s: failed to find comma separators in <%s>\n", __func__, line.c_str());+ continue;+ }+ auto sentence = line[comma_pos[0]+1] == '"' ? line.substr(comma_pos[0]+2, comma_pos[1] - comma_pos[0] - 3)+ : line.substr(comma_pos[0]+1, comma_pos[1] - comma_pos[0] - 1);+ auto choice1 = line.substr(comma_pos[1]+1, comma_pos[2] - comma_pos[1] - 1);+ auto choice2 = line.substr(comma_pos[2]+1, comma_pos[3] - comma_pos[2] - 1);+ auto answer = line.substr(comma_pos[3]+1, line.size() - comma_pos[3] - 1);+ auto index = line.substr(0, comma_pos[0]);+ int where = 0;+ for ( ; where < int(sentence.size()); ++where) {+ if (sentence[where] == '_') break;+ }+ if (where == int(sentence.size())) {+ printf("%s: no _ in <%s>\n", __func__, sentence.c_str());+ continue;+ }+ std::istringstream stream(answer.c_str());+ int i_answer; stream >> i_answer;+ if (stream.fail() || i_answer < 1 || i_answer > 2) {+ printf("%s: failed to parse answer <%s>\n", __func__, answer.c_str());+ continue;+ }+ result.emplace_back();+ auto& wg = result.back();+ wg.first = sentence.substr(0, where);+ wg.second = sentence.substr(where + 1, sentence.size() - where - 1);+ wg.choices[0] = std::move(choice1);+ wg.choices[1] = std::move(choice2);+ wg.answer = i_answer;+ }+ return result;+}++/*+ * Evaluates the Winogrande score.+ * Uses a CSV containing task index, dentence, choice 1, choice 2, answer (1 or 2)+ * You can get one such dataset from e.g. https://huggingface.co/datasets/ikawrakow/winogrande-eval-for-llama.cpp+ * As an example, the 1st row in the above dataset is+ *+ * 0,Sarah was a much better surgeon than Maria so _ always got the easier cases.,Sarah,Maria,2+ *+ */+static void winogrande_score(llama_context * ctx, const gpt_params & params) {++ constexpr int k_min_trailing_ctx = 3;++ auto data = load_winogrande_from_csv(params.prompt);+ if (data.empty()) {+ fprintf(stderr, "%s: no tasks\n", __func__);+ return;+ }++ fprintf(stderr, "%s : loaded %zu tasks from prompt.\n", __func__, data.size());++ if (params.winogrande_tasks > 0 && params.winogrande_tasks < data.size()) {+ fprintf(stderr, "%s : selecting %zu random tasks\n", __func__, params.winogrande_tasks);+ std::mt19937 rng(1);+ std::vector<int> aux(data.size());+ for (int i = 0; i < int(data.size()); ++i) {+ aux[i] = i;+ }+ float scale = 1/(1.f + (float)rng.max());+ std::vector<winogrande_entry> selected;+ selected.resize(params.winogrande_tasks);+ for (int i = 0; i < int(params.winogrande_tasks); ++i) {+ int j = int(scale*rng()*aux.size());+ selected[i] = std::move(data[aux[j]]);+ aux[j] = aux.back();+ aux.pop_back();+ }+ data = std::move(selected);+ }++ fprintf(stderr, "%s : tokenizing selected tasks\n", __func__);++ // This is needed as usual for LLaMA models+ const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));++ for (auto & task : data) {+ task.seq_tokens[0] = ::llama_tokenize(ctx, task.first + task.choices[0] + task.second, add_bos);+ task.seq_tokens[1] = ::llama_tokenize(ctx, task.first + task.choices[1] + task.second, add_bos);++ task.common_prefix = 0;+ for (size_t k = 0; k < task.seq_tokens[0].size(); k++) {+ if (task.seq_tokens[0][k] != task.seq_tokens[1][k]) {+ break;+ }+ task.common_prefix++;+ }++ task.required_tokens = task.common_prefix ++ task.seq_tokens[0].size() - task.common_prefix ++ task.seq_tokens[1].size() - task.common_prefix;++ task.n_base1 = ::llama_tokenize(ctx, task.first + task.choices[0], add_bos).size();+ task.n_base2 = ::llama_tokenize(ctx, task.first + task.choices[1], add_bos).size();+ }++ fprintf(stderr, "%s : calculating winogrande score over selected tasks.\n", __func__);++ const int n_vocab = llama_n_vocab(llama_get_model(ctx));+ const int n_ctx = llama_n_ctx(ctx);+ const int n_batch = params.n_batch;++ const int max_tasks_per_batch = 128;+ const int max_seq = 2*max_tasks_per_batch;++ llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);++ std::vector<float> tok_logits(n_vocab);+ std::vector<float> batch_logits(n_vocab*n_ctx);++ std::vector<std::pair<size_t, llama_token>> eval_pairs;+ std::vector<float> eval_results;+ std::vector<std::thread> workers(std::thread::hardware_concurrency());++ int n_correct = 0;+ int n_done = 0;++ for (size_t i0 = 0; i0 < data.size(); i0++) {+ int n_cur = 0;++ size_t i1 = i0;+ size_t i_batch = 0;++ llama_batch_clear(batch);++ while (n_cur + (int) data[i1].required_tokens <= n_ctx) {+ const int s0 = 2*(i1 - i0);+ if (s0 + 2 > max_seq) {+ break;+ }++ for (size_t i = 0; i < data[i1].common_prefix; ++i) {+ llama_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1}, false);+ }+ batch.logits[batch.n_tokens - 1] = true;++ for (int s = 0; s < 2; ++s) {+ for (size_t i = data[i1].common_prefix; i < data[i1].seq_tokens[s].size(); ++i) {+ llama_batch_add(batch, data[i1].seq_tokens[s][i], i, { s0 + s }, true);+ }+ }++ data[i1].i_batch = i_batch;+ i_batch += data[i1].required_tokens;++ n_cur += data[i1].required_tokens;+ if (++i1 == data.size()) {+ break;+ }+ }++ if (i0 == i1) {+ fprintf(stderr, "%s : task %zu does not fit in the context window\n", __func__, i0);+ return;}- // clear the KV cachellama_kv_cache_clear(ctx);- auto logits = hellaswag_evaluate_tokens(ctx, query_embd, 0, params.n_batch, n_vocab);- if (logits.empty()) {- fprintf(stderr, "%s : failed to eval\n", __func__);+ // decode all tasks [i0, i1)+ if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) {+ fprintf(stderr, "%s: llama_decode() failed\n", __func__);return;}- std::memcpy(tok_logits.data(), logits.data() + (context_size-1)*n_vocab, n_vocab*sizeof(float));- const auto first_probs = softmax(tok_logits);+ eval_pairs.clear();+ for (size_t i = i0; i < i1; ++i) {+ auto & task = data[i];- hs_data[task_idx].ending_logprob_count[0] = 1;- hs_data[task_idx].ending_logprob[0] = std::log(first_probs[query_embd[context_size]]);+ const bool skip_choice =+ task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx &&+ task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx;- // Calculate the logprobs over the ending- for (size_t j = context_size; j < query_size - 1; j++) {+ const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix;+ const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0;+ size_t li = n_base1 - 1;+ for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) {+ eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[0][j+1]));+ }+ const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix;+ const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0;+ li = task.seq_tokens[0].size() - task.common_prefix + n_base2 - 1;+ for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) {+ eval_pairs.push_back(std::make_pair(task.i_batch + li++, task.seq_tokens[1][j+1]));+ }+ }+ compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results);++ size_t ir = 0;+ for (size_t i = i0; i < i1; ++i) {+ auto & task = data[i];++ const bool skip_choice =+ task.seq_tokens[0].size() - task.common_prefix > k_min_trailing_ctx &&+ task.seq_tokens[1].size() - task.common_prefix > k_min_trailing_ctx;++ float score_1st = 0;+ const auto& n_base1 = skip_choice ? task.n_base1 : task.common_prefix;+ const int last_1st = task.seq_tokens[0].size() - n_base1 > 1 ? 1 : 0;+ for (size_t j = n_base1-1; j < task.seq_tokens[0].size()-1-last_1st; ++j) {+ score_1st += eval_results[ir++];+ }+ score_1st /= (task.seq_tokens[0].size() - n_base1 - last_1st);++ float score_2nd = 0;+ const auto& n_base2 = skip_choice ? task.n_base2 : task.common_prefix;+ const int last_2nd = task.seq_tokens[1].size() - n_base2 > 1 ? 1 : 0;+ for (size_t j = n_base2-1; j < task.seq_tokens[1].size()-1-last_2nd; ++j) {+ score_2nd += eval_results[ir++];+ }+ score_2nd /= (task.seq_tokens[1].size() - n_base2 - last_2nd);- std::memcpy(tok_logits.data(), logits.data() + j*n_vocab, n_vocab*sizeof(float));+ int result = score_1st > score_2nd ? 1 : 2;- const float prob = softmax(tok_logits)[query_embd[j + 1]];+ if (result == task.answer) {+ ++n_correct;+ }+ ++n_done;- hs_data[task_idx].ending_logprob[0] += std::log(prob);- hs_data[task_idx].ending_logprob_count[0]++;+ // print the accumulated accuracy mean x 100+ printf("%zu\t%.4lf\t%10.6f %10.6f %d %d\n", i+1, 100.0 * n_correct/n_done, score_1st, score_2nd, result, task.answer);+ fflush(stdout);}- // Calculate the mean token logprob for acc_norm- hs_data[task_idx].ending_logprob[0] /= hs_data[task_idx].ending_logprob_count[0];+ i0 = i1 - 1;+ }- // Do the remaining endings- // For these, we use the bare ending with n_past = context_size- //- for (size_t ending_idx = 1; ending_idx < 4; ending_idx++) {+ printf("\n");++ if (n_done < 100) return;- // Tokenize the query- query_embd.resize(ending_tokens[ending_idx].size() - context_size);- std::memcpy(query_embd.data(), ending_tokens[ending_idx].data() + context_size, query_embd.size()*sizeof(int));- query_size = query_embd.size();+ const float p = 1.f*n_correct/n_done;+ const float sigma = 100.f*sqrt(p*(1-p)/(n_done-1));+ printf("Final Winogrande score(%d tasks): %.4lf +/- %.4lf\n", n_done, 100*p, sigma);+}- // Stop if query wont fit the ctx window- if (context_size + query_size > (size_t)n_ctx) {- fprintf(stderr, "%s : number of tokens in query %zu > n_ctxl\n", __func__, query_size);+static bool deserialize_string(std::istream & in, std::string & str) {+ uint32_t size;+ if (!in.read((char *)&size, sizeof(size)).fail()) {+ str.resize(size);+ if (!in.read((char *)&str[0], size).fail()) return true;+ }+ return false;+}++struct multiple_choice_answers {+ std::vector<std::string> answers;+ std::vector<int> labels;+ bool deserialize(std::istream& in) {+ uint32_t n;+ in.read((char *)&n, sizeof(n));+ if (in.fail() || n > 100) return false; // 100 as max. number of answers should be good enough for any practical purpose+ answers.resize(n);+ labels.resize(n);+ for (auto& a : answers) {+ if (!deserialize_string(in, a)) return false;+ }+ in.read((char *)labels.data(), n*sizeof(int));+ return !in.fail();+ }+};++struct multiple_choice_task {+ std::string question; // the question (or context that needs to be continued)+ multiple_choice_answers mc1; // possible answers (continuations) with a single correct answer+ multiple_choice_answers mc2; // possible answers (continuations) with multiple correct answers - not handled yet+ bool deserialize(std::istream& in) {+ if (!deserialize_string(in, question)) return false;+ return mc1.deserialize(in) && mc2.deserialize(in);+ }++ // For evaluation+ size_t i_batch; // starting index in the llama_batch+ size_t common_prefix; // max number of initial tokens that are the same in all sentences+ size_t required_tokens; // needed number of tokens to evaluate all answers+ std::vector<std::vector<llama_token>> seq_tokens;+ std::vector<float> log_probs;+};++static bool multiple_choice_prepare_one_task(llama_context * ctx, bool add_bos, multiple_choice_task& task, bool log_error) {+ if (task.question.empty() || task.mc1.answers.empty()) {+ if (log_error) {+ printf("%s: found bad task with empty question and/or answers\n", __func__);+ }+ return false;+ }+ task.seq_tokens.reserve(task.mc1.answers.size());+ for (auto& answer : task.mc1.answers) {+ if (answer.empty()) {+ if (log_error) {+ printf("%s: found empty answer\n", __func__);+ }+ return false;+ }+ task.seq_tokens.emplace_back(::llama_tokenize(ctx, task.question + " " + answer, add_bos));+ }+ auto min_len = task.seq_tokens.front().size();+ for (auto& seq : task.seq_tokens) {+ min_len = std::min(min_len, seq.size());+ }+ task.common_prefix = 0;+ for (size_t k = 0; k < min_len; ++k) {+ auto token = task.seq_tokens[0][k];+ bool all_same = true;+ for (size_t i = 1; i < task.seq_tokens.size(); ++i) {+ if (task.seq_tokens[i][k] != token) {+ all_same = false;+ break;+ }+ }+ if (!all_same) {+ break;+ }+ ++task.common_prefix;+ }+ task.required_tokens = task.common_prefix;+ for (auto& seq : task.seq_tokens) {+ task.required_tokens += seq.size() - task.common_prefix;+ }+ return true;+}++//+// Calculates score for multiple choice tasks with single correct answer from prompt.+// Commonly used LLM evaluation metrics of this type are+// * ARC+// * HellaSwag+// * MMLU+// * TruthfulQA+//+// Validation datasets for these 4 tests can be found at+// https://huggingface.co/datasets/ikawrakow/validation-datasets-for-llama.cpp+// The data for these datasets was extracted from+// [email protected]:datasets/allenai/ai2_arc+// https://github.com/rowanz/hellaswag/blob/master/data/hellaswag_val.jsonl+// [email protected]:datasets/Stevross/mmlu+// https://huggingface.co/datasets/truthful_qa+//+static void multiple_choice_score(llama_context * ctx, const gpt_params & params) {++ std::istringstream strstream(params.prompt);+ uint32_t n_task;+ strstream.read((char *)&n_task, sizeof(n_task));+ if (strstream.fail() || n_task == 0) {+ printf("%s: no tasks\n", __func__);+ return;+ }+ printf("%s: there are %u tasks in prompt\n", __func__, n_task);+ std::vector<uint32_t> task_pos(n_task);+ strstream.read((char *)task_pos.data(), task_pos.size()*sizeof(uint32_t));+ if (strstream.fail()) {+ printf("%s: failed to raad task positions from prompt\n", __func__);+ return;+ }++ std::vector<multiple_choice_task> tasks;+ if (params.multiple_choice_tasks == 0 || params.multiple_choice_tasks >= (size_t)n_task) {+ // Use all tasks+ tasks.resize(n_task);+ printf("%s: reading tasks", __func__);+ int n_dot = n_task/100;+ int i = 0;+ for (auto& task : tasks) {+ ++i;+ if (!task.deserialize(strstream)) {+ printf("%s: failed to read task %d of %u\n", __func__, i, n_task);+ return;+ }+ if (i%n_dot == 0) printf(".");+ }+ printf("done\n");+ }+ else {+ printf("%s: selecting %zu random tasks from %u tasks available\n", __func__, params.multiple_choice_tasks, n_task);+ std::mt19937 rng(1);+ std::vector<int> aux(n_task);+ for (uint32_t i = 0; i < n_task; ++i) aux[i] = i;+ float scale = 1.f/(1.f + (float)std::mt19937::max());+ tasks.resize(params.multiple_choice_tasks);+ for (auto& task : tasks) {+ int j = (int)(scale * rng() * aux.size());+ int idx = aux[j];+ aux[j] = aux.back();+ aux.pop_back();+ strstream.seekg(task_pos[idx], std::ios::beg);+ if (!task.deserialize(strstream)) {+ printf("%s: failed to read task %d at position %u\n", __func__, idx, task_pos[idx]);return;}+ }+ n_task = params.multiple_choice_tasks;+ }- // Speedup small evaluations by evaluating atleast 32 tokens- // No, resizing to 32 is actually slightly slower (at least on CUDA)- //if (query_size < 32) {- // query_embd.resize(32);- //}+ // This is needed as usual for LLaMA models+ const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));- // Evaluate the query- logits = hellaswag_evaluate_tokens(ctx, query_embd, context_size, params.n_batch, n_vocab);- if (logits.empty()) {- fprintf(stderr, "%s : failed to eval\n", __func__);+ printf("%s: preparing task data", __func__);+ fflush(stdout);+ if (n_task > 500) {+ printf("...");+ fflush(stdout);+ std::atomic<int> counter(0);+ std::atomic<int> n_bad(0);+ auto prepare = [&counter, &n_bad, &tasks, ctx, add_bos] () {+ int num_tasks = tasks.size();+ int n_bad_local = 0;+ while (true) {+ int first = counter.fetch_add(K_TOKEN_CHUNK);+ if (first >= num_tasks) {+ if (n_bad_local > 0) n_bad += n_bad_local;+ break;+ }+ int last = std::min(first + K_TOKEN_CHUNK, num_tasks);+ for (int i = first; i < last; ++i) {+ if (!multiple_choice_prepare_one_task(ctx, add_bos, tasks[i], false)) ++n_bad_local;+ }+ }+ };+ size_t max_thread = std::thread::hardware_concurrency();+ max_thread = std::min(max_thread, (tasks.size() + K_TOKEN_CHUNK - 1)/K_TOKEN_CHUNK);+ std::vector<std::thread> workers(max_thread-1);+ for (auto& w : workers) w = std::thread(prepare);+ prepare();+ for (auto& w : workers) w.join();+ printf("done\n");+ fflush(stdout);+ int nbad = n_bad;+ if (nbad > 0) {+ printf("%s: found %d malformed tasks\n", __func__, nbad);+ return;+ }+ } else {+ int n_dot = n_task/100;+ int i_task = 0;+ for (auto& task : tasks) {+ ++i_task;+ if (!multiple_choice_prepare_one_task(ctx, add_bos, task, true)) {return;}+ if (i_task%n_dot == 0) {+ printf(".");+ fflush(stdout);+ }+ }+ printf("done\n");+ }++ printf("%s : calculating TruthfulQA score over %zu tasks.\n", __func__, tasks.size());++ printf("\ntask\tacc_norm\n");++ const int n_vocab = llama_n_vocab(llama_get_model(ctx));+ const int n_ctx = llama_n_ctx(ctx);+ const int n_batch = params.n_batch;++ const int max_tasks_per_batch = 32;+ const int max_seq = 4*max_tasks_per_batch;++ llama_batch batch = llama_batch_init(n_ctx, 0, max_seq);++ std::vector<float> tok_logits(n_vocab);+ std::vector<float> batch_logits(n_vocab*n_ctx);++ std::vector<std::pair<size_t, llama_token>> eval_pairs;+ std::vector<float> eval_results;+ std::vector<std::thread> workers(std::thread::hardware_concurrency());+ std::vector<int> batch_indeces;++ int n_done = 0;+ int n_correct = 0;+ int n_tot_answers = 0;++ for (size_t i0 = 0; i0 < tasks.size(); i0++) {+ int n_cur = 0;++ size_t i1 = i0;+ size_t i_batch = 0; // this tells us where in `llama_batch` we are currently++ llama_batch_clear(batch);++ // batch as much tasks as possible into the available context+ // each task has 4 unique seuqnce ids - one for each ending+ // the common prefix is shared among the 4 sequences to save tokens+ // we extract logits only from the last common token and from all ending tokens of each sequence+ int s0 = 0;+ while (n_cur + (int) tasks[i1].required_tokens <= n_ctx) {+ auto& cur_task = tasks[i1];- hs_data[task_idx].ending_logprob_count[ending_idx] = 1;- hs_data[task_idx].ending_logprob[ending_idx] = std::log(first_probs[query_embd[0]]);+ int num_answers = cur_task.seq_tokens.size();+ if (s0 + num_answers > max_seq) {+ break;+ }- // Calculate the logprobs over the ending- for (size_t j = 0; j < query_size - 1; j++) {- std::memcpy(tok_logits.data(), logits.data() + j*n_vocab, n_vocab*sizeof(float));+ if (int(batch_indeces.size()) != num_answers) {+ batch_indeces.resize(num_answers);+ }+ for (int s = 0; s < num_answers; ++s) batch_indeces[s] = s0 + s;- const float prob = softmax(tok_logits)[query_embd[j + 1]];+ for (size_t i = 0; i < cur_task.common_prefix; ++i) {+ //llama_batch_add(batch, cur_task.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false);+ llama_batch_add(batch, cur_task.seq_tokens[0][i], i, batch_indeces, false);+ }+ batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix- hs_data[task_idx].ending_logprob[ending_idx] += std::log(prob);- hs_data[task_idx].ending_logprob_count[ending_idx]++;+ for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {+ for (size_t i = cur_task.common_prefix; i < cur_task.seq_tokens[s].size(); ++i) {+ llama_batch_add(batch, cur_task.seq_tokens[s][i], i, { s0 + s }, true);+ }}- // Calculate the mean token logprob for acc_norm- hs_data[task_idx].ending_logprob[ending_idx] /= hs_data[task_idx].ending_logprob_count[ending_idx];+ s0 += num_answers;++ cur_task.i_batch = i_batch;+ i_batch += cur_task.required_tokens;+ n_cur += cur_task.required_tokens;+ if (++i1 == tasks.size()) {+ break;+ }+ }-// printf("task %lu, ending %lu, whole_len %lu, context_len %lu, ending_logprob_count %lu, ending_logprob %.4f\n",-// task_idx,ending_idx,whole_size,context_size, hs_data[task_idx].ending_logprob_count[ending_idx], hs_data[task_idx].ending_logprob[ending_idx] );+ if (i0 == i1) {+ fprintf(stderr, "%s : task %zu does not fit in the context window\n", __func__, i0);+ return;}- // Find the ending with maximum logprob- size_t ending_logprob_max_idx = 0;- double ending_logprob_max_val = hs_data[task_idx].ending_logprob[0];- for (size_t j = 1; j < 4; j++) {- if (hs_data[task_idx].ending_logprob[j] > ending_logprob_max_val) {- ending_logprob_max_idx = j;- ending_logprob_max_val = hs_data[task_idx].ending_logprob[j];+ llama_kv_cache_clear(ctx);++ // decode all tasks [i0, i1)+ if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) {+ fprintf(stderr, "%s: llama_decode() failed\n", __func__);+ return;+ }++ // Compute log-probs in parallel+ // First we collect all tasks+ eval_pairs.clear();+ for (size_t i = i0; i < i1; ++i) {+ auto& cur_task = tasks[i];+ size_t li = cur_task.common_prefix;+ for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {+ for (size_t j = cur_task.common_prefix; j < cur_task.seq_tokens[s].size() - 1; j++) {+ eval_pairs.push_back(std::make_pair(cur_task.i_batch + li++, cur_task.seq_tokens[s][j + 1]));+ }+ ++li;}}+ // Then we do the actual calculation+ compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results);++ size_t ir = 0;++ // compute the logprobs for each ending of the decoded tasks+ for (size_t i = i0; i < i1; ++i) {+ auto & cur_task = tasks[i];+ //printf("==== Evaluating <%s> with correct answer ", cur_task.question.c_str());+ //for (int j = 0; j < int(cur_task.mc1.labels.size()); ++j) {+ // if (cur_task.mc1.labels[j] == 1) {+ // printf("%d", j+1);+ // }+ //}+ //printf("\n common_prefix: %zu\n", cur_task.common_prefix);++ std::memcpy(tok_logits.data(), batch_logits.data() + n_vocab*(cur_task.i_batch + cur_task.common_prefix - 1), n_vocab*sizeof(float));++ const auto first_probs = softmax(tok_logits);++ cur_task.log_probs.resize(cur_task.seq_tokens.size());+ for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {+ size_t count = 1;+ float log_prob = std::log(first_probs[cur_task.seq_tokens[s][cur_task.common_prefix]]);+ for (size_t j = cur_task.common_prefix; j < cur_task.seq_tokens[s].size() - 1; j++) {+ //printf(" %zu %g\n", ir, eval_results[ir]);+ ++count;+ log_prob += eval_results[ir++];+ }+ cur_task.log_probs[s] = log_prob / count;+ //printf(" Final: %g\n", log_prob / count);+ //printf(" <%s> : %g\n", cur_task.mc1.answers[s].c_str(), log_prob/count);+ }-// printf("max logprob ending idx %lu, gold ending idx %lu\n", ending_logprob_max_idx, hs_data[task_idx].gold_ending_idx);+ // Find the ending with maximum logprob+ size_t logprob_max_idx = 0;+ float logprob_max_val = cur_task.log_probs[0];+ for (size_t s = 1; s < cur_task.log_probs.size(); s++) {+ if (cur_task.log_probs[s] > logprob_max_val) {+ logprob_max_val = cur_task.log_probs[s];+ logprob_max_idx = s;+ }+ }- // If the gold ending got the maximum logprobe add one accuracy point- if (ending_logprob_max_idx == hs_data[task_idx].gold_ending_idx) {- acc += 1.0;+ n_tot_answers += cur_task.log_probs.size();+ if (cur_task.mc1.labels[logprob_max_idx] == 1) {+ ++n_correct;+ }+ ++n_done;++ // Print the accumulated accuracy mean x 100+ printf("%d\t%.8lf\n", n_done, 100.*n_correct/n_done);+ fflush(stdout);}- // Print the accumulated accuracy mean x 100- printf("%zu\t%.8lf\n",task_idx+1, acc/double(task_idx+1)*100.0);- fflush(stdout);+ i0 = i1 - 1;+ }++ llama_batch_free(batch);++ if (n_done < 100) return;++ float p = 1.f*n_correct/n_done;+ float sigma = sqrt(p*(1-p)/(n_done-1));+ printf("\n Final result: %.4f +/- %.4f\n", 100.f*p, 100.f*sigma);+ p = 1.f*n_done/n_tot_answers;+ sigma = sqrt(p*(1-p)/(n_done-1));+ printf("Random chance: %.4f +/- %.4f\n", 100.f*p, 100.f*sigma);++ printf("\n");+}++static void kl_divergence(llama_context * ctx, const gpt_params & params) {+ if (params.logits_file.empty()) {+ fprintf(stderr, "%s: you must provide a name of a file containing the log probabilities of the base model\n", __func__);+ return;+ }+ std::ifstream in(params.logits_file.c_str(), std::ios::binary);+ if (!in) {+ fprintf(stderr, "%s: failed to open %s\n", __func__, params.logits_file.c_str());+ return;+ }+ {+ char check[9]; check[8] = 0;+ in.read(check, 8);+ if (in.fail() || strncmp("_logits_", check, 8) != 0) {+ fprintf(stderr, "%s: %s does not look like a file containing log-probabilities\n", __func__, params.logits_file.c_str());+ return;+ }}- delete [] hs_data;+ uint32_t n_ctx;+ in.read((char *)&n_ctx, sizeof(n_ctx));+ if (n_ctx > llama_n_ctx(ctx)) {+ fprintf(stderr, "%s: %s has been computed with %d, while the current context is %d. Increase it with -c and retry\n",+ __func__, params.logits_file.c_str(), n_ctx, params.n_ctx);+ }++ int n_vocab, n_chunk;+ in.read((char *)&n_vocab, sizeof(n_vocab));+ in.read((char *)&n_chunk, sizeof(n_chunk));+ if (in.fail()) {+ fprintf(stderr, "%s: failed reading n_vocab, n_chunk from %s\n", __func__, params.logits_file.c_str());+ return;+ }+ if (n_vocab != llama_n_vocab(llama_get_model(ctx))) {+ fprintf(stderr, "%s: inconsistent vocabulary (%d vs %d)\n", __func__, n_vocab, llama_n_vocab(llama_get_model(ctx)));+ }++ std::vector<llama_token> tokens(n_ctx * n_chunk);+ if (in.read((char *)tokens.data(), tokens.size()*sizeof(tokens[0])).fail()) {+ fprintf(stderr, "%s: failed reading evaluation tokens from %s\n", __func__, params.logits_file.c_str());+ return;+ }++ const int n_batch = params.n_batch;+ const int num_batches = (n_ctx + n_batch - 1)/n_batch;+ const int nv = 2*((n_vocab + 1)/2) + 4;+ const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));++ std::vector<uint16_t> log_probs_uint16(size_t(n_ctx - 1 - n_ctx/2) * nv);+ std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);+ std::vector<float> logits;+ if (num_batches > 1) {+ logits.reserve(n_ctx * n_vocab);+ }++ std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);++ auto mean_and_uncertainty = [] (double sum, double sum2, size_t count) {+ if (count < 1) {+ return std::make_pair(0., 0.);+ }+ double f = sum/count;+ double df = sum2/count - f*f;+ df = df > 0 && count > 10 ? sqrt(df/(count-1)) : 0.;+ return std::make_pair(f, df);+ };++ kl_divergence_result kld;+ auto kld_ptr = kld_values.data();++ for (int i = 0; i < n_chunk; ++i) {+ const int start = i * n_ctx;+ const int end = start + n_ctx;++ const auto t_start = std::chrono::high_resolution_clock::now();++ if (in.read((char *)log_probs_uint16.data(), log_probs_uint16.size()*sizeof(uint16_t)).fail()) {+ fprintf(stderr, "%s: failed reading log-probs for chunk %d\n", __func__, i);+ return;+ }++ // clear the KV cache+ llama_kv_cache_clear(ctx);++ for (int j = 0; j < num_batches; ++j) {+ const int batch_start = start + j * n_batch;+ const int batch_size = std::min(end - batch_start, n_batch);++ // save original token and restore it after eval+ const auto token_org = tokens[batch_start];++ // add BOS token for the first batch of each chunk+ if (add_bos && j == 0) {+ tokens[batch_start] = llama_token_bos(llama_get_model(ctx));+ }++ if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {+ fprintf(stderr, "%s : failed to eval\n", __func__);+ return;+ }++ // restore the original token in case it was set to BOS+ tokens[batch_start] = token_org;++ if (num_batches > 1) {+ const auto * batch_logits = llama_get_logits(ctx);+ logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);+ }+ }++ const auto t_end = std::chrono::high_resolution_clock::now();++ if (i == 0) {+ const float t_total = std::chrono::duration<float>(t_end - t_start).count();+ fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total);+ int total_seconds = (int)(t_total * n_chunk);+ if (total_seconds >= 60*60) {+ fprintf(stderr, "%d hours ", total_seconds / (60*60));+ total_seconds = total_seconds % (60*60);+ }+ fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);++ printf("\nchunk PPL ln(PPL(Q)/PPL(base)) KL-Divergence Same top\n");+ }++ const int first = n_ctx/2;+ const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);+ process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,+ workers, log_probs_uint16, kld, kld_ptr);+ kld_ptr += n_ctx - 1 - first;++ auto ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);+ auto log_ppl_ratio = mean_and_uncertainty(kld.sum_nll_diff, kld.sum_nll_diff2, kld.count);+ auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);+ auto p_top = 1.*kld.n_same_top/kld.count;+ auto d_p_top = sqrt(p_top*(1 - p_top)/(kld.count - 1));++ printf("%4d %10.4lf %10.5lf ± %10.5f %10.5f ± %10.5lf %.5f ± %.5f\n", i+1, exp(ppl.first),+ log_ppl_ratio.first, log_ppl_ratio.second, kl_div.first, kl_div.second,+ p_top, d_p_top);++ fflush(stdout);+ logits.clear();+ }printf("\n");++ if (kld.count < 100) return; // we do not wish to do statistics on so few values++ std::sort(kld_values.begin(), kld_values.end());++ printf("===== KL-divergence statistics\n");+ auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);+ printf("Average: %10.6f ±%10.6lf\n", kl_div.first, kl_div.second);+ auto kld_median = kld_values.size()%2 == 0 ? 0.5f*(kld_values[kld_values.size()/2] + kld_values[kld_values.size()/2-1])+ : kld_values[kld_values.size()/2];+ printf("Median : %10.6f\n", kld_median);++ auto percentile = [&kld_values] (float fraction) {+ if (fraction <= 0) return kld_values.front();+ if (fraction >= 1) return kld_values.back();+ float p = fraction*(kld_values.size() - 1);+ size_t ip = size_t(p); p -= ip;+ return (1 - p)*kld_values[ip] + p*kld_values[std::min(ip+1, kld_values.size()-1)];+ };++ printf("Maximum: %10.6f\n", kld_values.back());+ printf("KLD_99 : %10.6f\n", percentile(0.99f));+ printf("KLD_95 : %10.6f\n", percentile(0.95f));+ printf("KLD_90 : %10.6f\n", percentile(0.90f));++ printf("Minimum: %10.6f\n", kld_values.front());+ printf("KLD_01 : %10.6f\n", percentile(0.01f));+ printf("KLD_05 : %10.6f\n", percentile(0.05f));+ printf("KLD_10 : %10.6f\n", percentile(0.10f));+}int main(int argc, char ** argv) {@@ -695,6 +1795,7 @@ int main(int argc, char ** argv) {llamafile_check_cpu();gpt_params params;+params.n_batch = 512;if (!gpt_params_parse(argc, argv, params)) {return 1;@@ -749,6 +1850,12 @@ int main(int argc, char ** argv) {struct results_perplexity results;if (params.hellaswag) {hellaswag_score(ctx, params);+ } else if (params.winogrande) {+ winogrande_score(ctx, params);+ } else if (params.multiple_choice) {+ multiple_choice_score(ctx, params);+ } else if (params.kl_divergence) {+ kl_divergence(ctx, params);} else {results = perplexity(ctx, params);}diff --git a/llama.cpp/quantize/BUILD.mk b/llama.cpp/quantize/BUILD.mkindex fb58d809..be814afe 100644--- a/llama.cpp/quantize/BUILD.mk+++ b/llama.cpp/quantize/BUILD.mk@@ -8,11 +8,11 @@ LLAMA_CPP_QUANTIZE_HDRS = $(filter %.h,$(LLAMA_CPP_QUANTIZE_FILES))LLAMA_CPP_QUANTIZE_SRCS = $(filter %.cpp,$(LLAMA_CPP_QUANTIZE_FILES))LLAMA_CPP_QUANTIZE_OBJS = $(LLAMA_CPP_QUANTIZE_SRCS:%.cpp=o/$(MODE)/%.o)-.PHONY: o/$(MODE)/llama.cpp/quantize-o/$(MODE)/llama.cpp/quantize: \- o/$(MODE)/llama.cpp/quantize/quantize-o/$(MODE)/llama.cpp/quantize/quantize: \o/$(MODE)/llama.cpp/quantize/quantize.o \o/$(MODE)/llama.cpp/quantize/quantize.1.asc.zip.o \o/$(MODE)/llama.cpp/llama.cpp.a++.PHONY: o/$(MODE)/llama.cpp/quantize+o/$(MODE)/llama.cpp/quantize: \+ o/$(MODE)/llama.cpp/quantize/quantizediff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1index 1afe57e4..7efd9ea9 100644--- a/llama.cpp/quantize/quantize.1+++ b/llama.cpp/quantize/quantize.1@@ -84,6 +84,7 @@ COPY Only copy tensors, no quantizing..El.Sh SEE ALSO.Xr llamafile 1 ,+.Xr llamafile-imatrix 1 ,.Xr llamafile-perplexity 1 ,.Xr llava-quantize 1 ,.Xr zipalign 1 ,diff --git a/llama.cpp/quantize/quantize.1.asc b/llama.cpp/quantize/quantize.1.ascindex cb975dbd..0ae3ed39 100644--- a/llama.cpp/quantize/quantize.1.asc+++ b/llama.cpp/quantize/quantize.1.asc@@ -1,7 +1,7 @@-LLAMAFILE-QUANTIZE(1) General Commands Manual LLAMAFILE-QUANTIZE(1)+LLAMAFILE-QUANTIZE(1) BSD General Commands Manual LLAMAFILE-QUANTIZE(1)NNAAMMEE- llllaammaaffiillee--qquuaannttiizzee - large language model quantizer+ llllaammaaffiillee--qquuaannttiizzee — large language model quantizerSSYYNNOOPPSSIISSllllaammaaffiillee--qquuaannttiizzee [flags...] _m_o_d_e_l_-_f_3_2_._g_g_u_f [_m_o_d_e_l_-_q_u_a_n_t_._g_g_u_f] _t_y_p_e@@ -39,8 +39,8 @@ AARRGGUUMMEENNTTSS[inp path]/ggml-model-[ftype].gguf._t_y_p_e Is the desired quantization format, which may be the integer id- of a supported quantization type, or its name. See the- quantization types section below for acceptable formats.+ of a supported quantization type, or its name. See the quantiza‐+ tion types section below for acceptable formats._n_t_h_r_e_a_d_sNumber of threads to use during computation (default: nproc/2)@@ -48,29 +48,29 @@ AARRGGUUMMEENNTTSSQQUUAANNTTIIZZAATTIIOONN TTYYPPEESSThe following quantization types are available:- -- 2 Q4_0 3.56G +0.2166 ppl @ LLaMA-v1-7B- -- 3 Q4_1 3.90G +0.1585 ppl @ LLaMA-v1-7B- -- 8 Q5_0 4.33G +0.0683 ppl @ LLaMA-v1-7B- -- 9 Q5_1 4.70G +0.0349 ppl @ LLaMA-v1-7B- -- 10 Q2_K 2.63G +0.6717 ppl @ LLaMA-v1-7B- -- 12 Q3_K alias for Q3_K_M- -- 11 Q3_K_S 2.75G +0.5551 ppl @ LLaMA-v1-7B- -- 12 Q3_K_M 3.07G +0.2496 ppl @ LLaMA-v1-7B- -- 13 Q3_K_L 3.35G +0.1764 ppl @ LLaMA-v1-7B- -- 15 Q4_K alias for Q4_K_M- -- 14 Q4_K_S 3.59G +0.0992 ppl @ LLaMA-v1-7B- -- 15 Q4_K_M 3.80G +0.0532 ppl @ LLaMA-v1-7B- -- 17 Q5_K alias for Q5_K_M- -- 16 Q5_K_S 4.33G +0.0400 ppl @ LLaMA-v1-7B- -- 17 Q5_K_M 4.45G +0.0122 ppl @ LLaMA-v1-7B- -- 18 Q6_K 5.15G -0.0008 ppl @ LLaMA-v1-7B- -- 7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B- -- 1 F16 13.00G @ 7B- -- 0 F32 26.00G @ 7B+ -- 2 Q4_0 3.56G +0.2166 ppl @ LLaMA-v1-7B+ -- 3 Q4_1 3.90G +0.1585 ppl @ LLaMA-v1-7B+ -- 8 Q5_0 4.33G +0.0683 ppl @ LLaMA-v1-7B+ -- 9 Q5_1 4.70G +0.0349 ppl @ LLaMA-v1-7B+ -- 10 Q2_K 2.63G +0.6717 ppl @ LLaMA-v1-7B+ -- 12 Q3_K alias for Q3_K_M+ -- 11 Q3_K_S 2.75G +0.5551 ppl @ LLaMA-v1-7B+ -- 12 Q3_K_M 3.07G +0.2496 ppl @ LLaMA-v1-7B+ -- 13 Q3_K_L 3.35G +0.1764 ppl @ LLaMA-v1-7B+ -- 15 Q4_K alias for Q4_K_M+ -- 14 Q4_K_S 3.59G +0.0992 ppl @ LLaMA-v1-7B+ -- 15 Q4_K_M 3.80G +0.0532 ppl @ LLaMA-v1-7B+ -- 17 Q5_K alias for Q5_K_M+ -- 16 Q5_K_S 4.33G +0.0400 ppl @ LLaMA-v1-7B+ -- 17 Q5_K_M 4.45G +0.0122 ppl @ LLaMA-v1-7B+ -- 18 Q6_K 5.15G -0.0008 ppl @ LLaMA-v1-7B+ -- 7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B+ -- 1 F16 13.00G @ 7B+ -- 0 F32 26.00G @ 7B-- COPY Only copy tensors, no quantizing.SSEEEE AALLSSOO- llamafile(1), llamafile-perplexity(1), llava-quantize(1), zipalign(1),- unzip(1)+ llamafile(1), llamafile-imatrix(1), llamafile-perplexity(1),+ llava-quantize(1), zipalign(1), unzip(1)Llamafile Manual December 5, 2023 Llamafile Manualdiff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex bb751ebf..570566d4 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -5,10 +5,15 @@#include "llama.cpp/llama.h"#include "llamafile/version.h"+#include <cctype>#include <cstdio>#include <cstring>#include <vector>#include <string>+#include <unordered_map>+#include <fstream>+#include <cmath>+#include <algorithm>struct quant_option {std::string name;@@ -21,8 +26,12 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 3.90G, +0.1585 ppl @ LLaMA-v1-7B", },{ "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 4.33G, +0.0683 ppl @ LLaMA-v1-7B", },{ "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 4.70G, +0.0349 ppl @ LLaMA-v1-7B", },+ { "IQ2_XXS",LLAMA_FTYPE_MOSTLY_IQ2_XXS," 2.06 bpw quantization", },+ { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", },{ "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.63G, +0.6717 ppl @ LLaMA-v1-7B", },+ { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.16G, +9.0634 ppl @ LLaMA-v1-7B", },{ "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" },+ { "Q3_K_XS",LLAMA_FTYPE_MOSTLY_Q3_K_XS,"3-bit extra small quantization" , },{ "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 2.75G, +0.5551 ppl @ LLaMA-v1-7B", },{ "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.07G, +0.2496 ppl @ LLaMA-v1-7B", },{ "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 3.35G, +0.1764 ppl @ LLaMA-v1-7B", },@@ -40,6 +49,12 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },};+static bool is_integer_str(const char *s) {+ if (*s == '-') ++s;+ if (!*s) return false;+ while (isdigit(*s)) ++s;+ return !*s;+}static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {std::string ftype_str;@@ -54,7 +69,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftypreturn true;}}- try {+ if (is_integer_str(ftype_str.c_str())) {int ftype_int = std::stoi(ftype_str);for (auto & it : QUANT_OPTIONS) {if (it.ftype == ftype_int) {@@ -64,9 +79,6 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp}}}- catch (...) {- // stoi failed- }return false;}@@ -75,10 +87,14 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp//[[noreturn]]static void usage(const char * executable) {- printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);+ printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights] [--exclude-weights] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n");printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n");printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n");+ printf(" --imatrix file_name: use data in file_name as importance matrix for quant optimizations\n");+ printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n");+ printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n");+ printf("Note: --include-weights and --exclude-weights cannot be used together\n");printf("\nAllowed quantization types:\n");for (auto & it : QUANT_OPTIONS) {if (it.name != "COPY") {@@ -86,11 +102,93 @@ static void usage(const char * executable) {} else {printf(" ");}- printf("%-6s : %s\n", it.name.c_str(), it.desc.c_str());+ printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str());}exit(1);}+static void load_imatrix(const std::string& imatrix_file, std::unordered_map<std::string, std::vector<float>>& imatrix_data) {+ std::ifstream in(imatrix_file.c_str(), std::ios::binary);+ if (!in) {+ printf("%s: failed to open %s\n",__func__,imatrix_file.c_str());+ return;+ }+ int n_entries;+ in.read((char*)&n_entries, sizeof(n_entries));+ if (in.fail() || n_entries < 1) {+ printf("%s: no data in file %s\n", __func__, imatrix_file.c_str());+ return;+ }+ for (int i = 0; i < n_entries; ++i) {+ int len; in.read((char *)&len, sizeof(len));+ std::vector<char> name_as_vec(len+1);+ in.read((char *)name_as_vec.data(), len);+ if (in.fail()) {+ printf("%s: failed reading name for entry %d from %s\n",__func__,i+1,imatrix_file.c_str());+ return;+ }+ name_as_vec[len] = 0;+ std::string name{name_as_vec.data()};+ auto& e = imatrix_data[std::move(name)];+ int ncall;+ in.read((char*)&ncall, sizeof(ncall));+ int nval;+ in.read((char *)&nval, sizeof(nval));+ if (in.fail() || nval < 1) {+ printf("%s: failed reading number of values for entry %d\n",__func__,i);+ imatrix_data = {};+ return;+ }+ e.resize(nval);+ in.read((char*)e.data(), nval*sizeof(float));+ if (in.fail()) {+ printf("%s: failed reading data for entry %d\n",__func__,i);+ imatrix_data = {};+ return;+ }+ if (ncall > 0) {+ for (auto& v : e) v /= ncall;+ }+ }+ printf("%s: loaded %d importance matrix entries from %s\n",__func__,int(imatrix_data.size()),imatrix_file.c_str());+}++static void prepare_imatrix(const std::string& imatrix_file,+ const std::vector<std::string>& included_weights,+ const std::vector<std::string>& excluded_weights,+ std::unordered_map<std::string, std::vector<float>>& imatrix_data) {+ if (!imatrix_file.empty()) {+ load_imatrix(imatrix_file, imatrix_data);+ }+ if (imatrix_data.empty()) {+ return;+ }+ if (!excluded_weights.empty()) {+ for (auto& name : excluded_weights) {+ for (auto it = imatrix_data.begin(); it != imatrix_data.end(); ) {+ auto pos = it->first.find(name);+ if (pos != std::string::npos) it = imatrix_data.erase(it);+ else ++it;+ }+ }+ }+ if (!included_weights.empty()) {+ std::unordered_map<std::string, std::vector<float>> tmp;+ for (auto& name : included_weights) {+ for (auto& e : imatrix_data) {+ auto pos = e.first.find(name);+ if (pos != std::string::npos) {+ tmp.emplace(std::move(e));+ }+ }+ }+ imatrix_data = std::move(tmp);+ }+ if (!imatrix_data.empty()) {+ printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));+ }+}+int main(int argc, char ** argv) {if (llamafile_has(argv, "--version")) {@@ -115,6 +213,8 @@ int main(int argc, char ** argv) {llama_model_quantize_params params = llama_model_quantize_default_params();int arg_idx = 1;+ std::string imatrix_file;+ std::vector<std::string> included_weights, excluded_weights;for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {@@ -123,15 +223,43 @@ int main(int argc, char ** argv) {params.allow_requantize = true;} else if (strcmp(argv[arg_idx], "--pure") == 0) {params.pure = true;+ } else if (strcmp(argv[arg_idx], "--imatrix") == 0) {+ if (arg_idx < argc-1) {+ imatrix_file = argv[++arg_idx];+ } else {+ usage(argv[0]);+ }+ } else if (strcmp(argv[arg_idx], "--include-weights") == 0) {+ if (arg_idx < argc-1) {+ included_weights.push_back(argv[++arg_idx]);+ } else {+ usage(argv[0]);+ }+ } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {+ if (arg_idx < argc-1) {+ excluded_weights.push_back(argv[++arg_idx]);+ } else {+ usage(argv[0]);+ }} else {usage(argv[0]);}}if (argc - arg_idx < 2) {+ printf("%s: bad arguments\n", argv[0]);+ usage(argv[0]);+ }+ if (!included_weights.empty() && !excluded_weights.empty()) {usage(argv[0]);}+ std::unordered_map<std::string, std::vector<float>> imatrix_data;+ prepare_imatrix(imatrix_file, included_weights, excluded_weights, imatrix_data);+ if (!imatrix_data.empty()) {+ params.imatrix = &imatrix_data;+ }+llama_backend_init(false);// parse command line arguments@@ -182,6 +310,13 @@ int main(int argc, char ** argv) {}}+ if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) && imatrix_data.empty()) {+ fprintf(stderr, "\n===============================================================================================\n");+ fprintf(stderr, "Please do not use IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n");+ fprintf(stderr, "===============================================================================================\n\n\n");+ return 1;+ }+print_build_info();fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex 006333dc..e6a332f9 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -132,6 +132,8 @@ static void sampler_queue(const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));const float temp = params.temp;+ const float dynatemp_range = params.dynatemp_range;+ const float dynatemp_exponent = params.dynatemp_exponent;const int32_t top_k = params.top_k <= 0 ? n_vocab : params.top_k;const float top_p = params.top_p;const float min_p = params.min_p;@@ -146,7 +148,15 @@ static void sampler_queue(case 'y': llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;case 'p': llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;case 'm': llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;- case 't': llama_sample_temp (ctx_main, &cur_p, temp); break;+ case 't':+ if (dynatemp_range > 0) {+ float dynatemp_min = std::max(0.0f, temp - dynatemp_range);+ float dynatemp_max = std::max(0.0f, temp + dynatemp_range);+ llama_sample_entropy(ctx_main, &cur_p, dynatemp_min, dynatemp_max, dynatemp_exponent);+ } else {+ llama_sample_temp(ctx_main, &cur_p, temp);+ }+ break;default : break;}}@@ -193,6 +203,11 @@ static llama_token llama_sampling_sample_impl(logits[it->first] += it->second;}+ if (ctx_cfg) {+ float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx);+ llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale);+ }+cur.clear();for (llama_token token_id = 0; token_id < n_vocab; token_id++) {@@ -201,10 +216,6 @@ static llama_token llama_sampling_sample_impl(llama_token_data_array cur_p = { cur.data(), cur.size(), false };- if (ctx_cfg) {- llama_sample_classifier_free_guidance(ctx_main, &cur_p, ctx_cfg, params.cfg_scale);- }-// apply penaltiesconst auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev;const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n);diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex f16ef97e..88899c09 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -17,7 +17,9 @@ typedef struct llama_sampling_params {float min_p = 0.05f; // 0.0 = disabledfloat tfs_z = 1.00f; // 1.0 = disabledfloat typical_p = 1.00f; // 1.0 = disabled- float temp = 0.80f; // 1.0 = disabled+ float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities+ float dynatemp_range = 0.00f; // 0.0 = disabled+ float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature samplerint32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)float penalty_repeat = 1.10f; // 1.0 = disabledfloat penalty_freq = 0.00f; // 0.0 = disableddiff --git a/llama.cpp/server/httplib.h b/llama.cpp/server/httplib.hindex 98c59d5d..28746000 100644--- a/llama.cpp/server/httplib.h+++ b/llama.cpp/server/httplib.h@@ -2227,18 +2227,12 @@ inline std::string decode_url(const std::string &s,}inline void read_file(const std::string &path, std::string &out) {- out.resize(0);- int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);- if (fd != -1) {- ssize_t size = lseek(fd, 0, SEEK_END);- if (size != -1) {- out.resize(static_cast<size_t>(size));- ssize_t got = pread(fd, &out[0], size, 0);- if (got == -1) got = 0;- out.resize(static_cast<size_t>(got));- }- close(fd);- }+ std::ifstream fs(path, std::ios_base::binary);+ fs.seekg(0, std::ios_base::end);+ auto size = fs.tellg();+ fs.seekg(0);+ out.resize(static_cast<size_t>(size));+ fs.read(&out[0], static_cast<std::streamsize>(size));}inline std::string file_extension(const std::string &path) {diff --git a/llama.cpp/server/oai.h b/llama.cpp/server/oai.hnew file mode 100644index 00000000..691706c9--- /dev/null+++ b/llama.cpp/server/oai.h@@ -0,0 +1,211 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi++#pragma once++#include <string>+#include <vector>+#include <set>+#include <mutex>+#include <condition_variable>+#include <unordered_map>++#include "json.h"+#include "utils.h"++#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"++using json = nlohmann::json;++inline static json oaicompat_completion_params_parse(+ const json &body /* openai api json semantics */)+{+ json llama_params;++ llama_params["__oaicompat"] = true;++ // Map OpenAI parameters to llama.cpp parameters+ //+ // For parameters that are defined by the OpenAI documentation (e.g.+ // temperature), we explicitly specify OpenAI's intended default; we+ // need to do that because sometimes OpenAI disagrees with llama.cpp+ //+ // https://platform.openai.com/docs/api-reference/chat/create+ llama_sampling_params default_sparams;+ llama_params["model"] = json_value(body, "model", std::string("unknown"));+ llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt'+ llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);+ llama_params["temperature"] = json_value(body, "temperature", 0.0);+ llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);+ llama_params["top_p"] = json_value(body, "top_p", 1.0);+ llama_params["n_predict"] = json_value(body, "max_tokens", -1);+ llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());+ llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);+ llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);+ llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);+ llama_params["stream"] = json_value(body, "stream", false);+ llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);+ llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);+ llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);+ llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);+ llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);+ llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);+ llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);+ llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);++ if (body.count("grammar") != 0) {+ llama_params["grammar"] = json_value(body, "grammar", json::object());+ }++ // Handle 'stop' field+ if (body.contains("stop") && body["stop"].is_string()) {+ llama_params["stop"] = json::array({body["stop"].get<std::string>()});+ } else {+ llama_params["stop"] = json_value(body, "stop", json::array());+ }++ // Ensure there is ChatML-specific end sequence among stop words+ llama_params["stop"].push_back("<|im_end|>");++ return llama_params;+}++inline static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false)+{+ json result = response.result_json;++ bool stopped_word = result.count("stopped_word") != 0;+ bool stopped_eos = json_value(result, "stopped_eos", false);+ int num_tokens_predicted = json_value(result, "tokens_predicted", 0);+ int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason = "length";+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }++ json choices =+ streaming ? json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}})+ : json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"message", json{{"content", content},+ {"role", "assistant"}}}}});++ std::time_t t = std::time(0);++ json res =+ json{{"choices", choices},+ {"created", t},+ {"model",+ json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},+ {"object", streaming ? "chat.completion.chunk" : "chat.completion"},+ {"usage",+ json{{"completion_tokens", num_tokens_predicted},+ {"prompt_tokens", num_prompt_tokens},+ {"total_tokens", num_tokens_predicted + num_prompt_tokens}}},+ {"id", gen_chatcmplid()}};++ if (server_verbose) {+ res["__verbose"] = result;+ }++ if (result.contains("completion_probabilities")) {+ res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());+ }++ return res;+}++// return value is vector as there is one case where we might need to generate two responses+inline static std::vector<json> format_partial_response_oaicompat(const task_result &response) {+ json result = response.result_json;++ if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {+ return std::vector<json>({response.result_json});+ }++ bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;+ std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));++ bool stopped_word = json_value(result, "stopped_word", false);+ bool stopped_eos = json_value(result, "stopped_eos", false);+ bool stopped_limit = json_value(result, "stopped_limit", false);+ std::string content = json_value(result, "content", std::string(""));++ std::string finish_reason;+ if (stopped_word || stopped_eos) {+ finish_reason = "stop";+ }+ if (stopped_limit) {+ finish_reason = "length";+ }++ std::time_t t = std::time(0);++ json choices;++ if (!finish_reason.empty()) {+ choices = json::array({json{{"finish_reason", finish_reason},+ {"index", 0},+ {"delta", json::object()}}});+ } else {+ if (first) {+ if (content.empty()) {+ choices = json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{{"role", "assistant"}}}}});+ } else {+ // We have to send this as two updates to conform to openai behavior+ json initial_ret = json{{"choices", json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"role", "assistant"}+ }}}})},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ json second_ret = json{+ {"choices", json::array({json{{"finish_reason", nullptr},+ {"index", 0},+ {"delta", json{+ {"content", content}}}+ }})},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ return std::vector<json>({initial_ret, second_ret});+ }+ } else {+ // Some idiosyncrasy in task processing logic makes several trailing calls+ // with empty content, we ignore these at the calee site.+ if (content.empty()) {+ return std::vector<json>({json::object()});+ }++ choices = json::array({json{+ {"finish_reason", nullptr},+ {"index", 0},+ {"delta",+ json{+ {"content", content},+ }},+ }});+ }+ }++ json ret = json{{"choices", choices},+ {"created", t},+ {"id", gen_chatcmplid()},+ {"model", modelname},+ {"object", "chat.completion.chunk"}};++ return std::vector<json>({ret});+}diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex b4134fae..d60304f7 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -2,46 +2,43 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#include "server.h"-#include "tool/args/args.h"++#include <atomic>+#include <chrono>+#include <condition_variable>+#include <cstddef>+#include <thread>+#include "llama.cpp/common.h"-#include "llama.cpp/llama.h"+#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-metal.h"#include "llama.cpp/grammar-parser.h"+#include "llama.cpp/llama.h"#include "llama.cpp/llava/clip.h"#include "llama.cpp/stb_image.h"-#include "llama.cpp/ggml-metal.h"-#include "llama.cpp/ggml-cuda.h"#include "llamafile/llamafile.h"-#include "llamafile/version.h"#include "llamafile/log.h"+#include "llamafile/version.h"+#include "libc/calls/pledge.h"+#include "tool/args/args.h"+#include "libc/dce.h"+#include "oai.h"+// increase max payload length to allow use of larger context size#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 1048576-#include "httplib.h"#include "json.h"-#include <cosmo.h>-#include <unistd.h>-#include <cstdio>-#include <cstddef>-#include <thread>-#include <mutex>-#include <chrono>-#include <condition_variable>-#include <sys/types.h>-#include <ifaddrs.h>-#ifndef SERVER_VERBOSE#define SERVER_VERBOSE 1#endif-#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo-0613"-using json = nlohmann::json;struct server_params{std::string hostname = "127.0.0.1";- std::string api_key;+ std::vector<std::string> api_keys;std::string public_path = "/zip/llama.cpp/server/public";int32_t port = 8080;int32_t read_timeout = 600;@@ -50,191 +47,7 @@ struct server_paramsbool unsecure = false;};-static bool server_verbose = false;--#if SERVER_VERBOSE != 1-#define LOG_VERBOSE(MSG, ...)-#else-#define LOG_VERBOSE(MSG, ...) \- do \- { \- if (server_verbose) \- { \- server_log("VERBOSE", __func__, __LINE__, MSG, __VA_ARGS__); \- } \- } while (0)-#endif--#define LOG_ERROR( MSG, ...) server_log("ERROR", __func__, __LINE__, MSG, __VA_ARGS__)-#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__)-#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)--json oaicompat_completion_params_parse(const json &body);-std::string format_chatml(std::vector<json> messages);---//-// base64 utils (TODO: move to common in the future)-//--static const std::string base64_chars =- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"- "abcdefghijklmnopqrstuvwxyz"- "0123456789+/";--static inline bool is_base64(uint8_t c)-{- return (isalnum(c) || (c == '+') || (c == '/'));-}--static std::vector<uint8_t> base64_decode(const std::string & encoded_string)-{- int i = 0;- int j = 0;- int in_ = 0;-- int in_len = encoded_string.size();-- uint8_t char_array_4[4];- uint8_t char_array_3[3];-- std::vector<uint8_t> ret;-- while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))- {- char_array_4[i++] = encoded_string[in_]; in_++;- if (i == 4)- {- for (i = 0; i <4; i++)- {- char_array_4[i] = base64_chars.find(char_array_4[i]);- }-- char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);- char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);- char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];-- for (i = 0; (i < 3); i++)- {- ret.push_back(char_array_3[i]);- }- i = 0;- }- }-- if (i)- {- for (j = i; j <4; j++)- {- char_array_4[j] = 0;- }-- for (j = 0; j <4; j++)- {- char_array_4[j] = base64_chars.find(char_array_4[j]);- }-- char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);- char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);- char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];-- for (j = 0; (j < i - 1); j++)- {- ret.push_back(char_array_3[j]);- }- }-- return ret;-}--//-// parallel-//--enum task_type {- COMPLETION_TASK,- CANCEL_TASK-};--struct task_server {- int id;- int target_id;- task_type type;- json data;- bool infill_mode = false;- bool embedding_mode = false;- int multitask_id = -1;-};--struct task_result {- int id;- int multitask_id = -1;- bool stop;- bool error;- json result_json;-};--struct task_multi {- int id;- std::set<int> subtasks_remaining{};- std::vector<task_result> results{};-};--// TODO: can become bool if we can't find use of more states-enum slot_state-{- IDLE,- PROCESSING,-};--enum slot_command-{- NONE,- LOAD_PROMPT,- RELEASE,-};--struct slot_params-{- bool stream = true;- bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt-- uint32_t seed = -1; // RNG seed- int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_predict = -1; // new tokens to predict-- std::vector<std::string> antiprompt;-- json input_prefix;- json input_suffix;-};--struct slot_image-{- int32_t id;-- bool request_encode_image = false;- float * image_embedding = nullptr;- int32_t image_tokens = 0;-- clip_image_u8 * img_data;-- std::string prefix_prompt; // before of this image-};--// completion token output with probabilities-struct completion_token_output-{- struct token_prob- {- llama_token tok;- float prob;- };-- std::vector<token_prob> probs;- llama_token tok;- std::string text_to_send;-};+bool server_verbose = false;static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b){@@ -290,28 +103,6 @@ static std::string tokens_to_str(llama_context *ctx, Iter begin, Iter end)return ret;}-static void server_log(const char *level, const char *function, int line,- const char *message, const nlohmann::ordered_json &extra)-{- nlohmann::ordered_json log- {- {"timestamp", time(nullptr)},- {"level", level},- {"function", function},- {"line", line},- {"message", message},- };-- if (!extra.empty())- {- log.merge_patch(extra);- }-- const std::string str = log.dump(-1, ' ', false, json::error_handler_t::replace);- printf("%.*s\n", (int)str.size(), str.data());- fflush(stdout);-}-// format incomplete utf-8 multibyte character for outputstatic std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token){@@ -353,15 +144,6 @@ static json probs_vector_to_json(const llama_context *ctx, const std::vector<comreturn out;}-template <typename T>-static T json_value(const json &body, const std::string &key, const T &default_value)-{- // Fallback null to default value- return body.contains(key) && !body.at(key).is_null()- ? body.value(key, default_value)- : default_value;-}-struct llama_client_slot{int id;@@ -452,8 +234,14 @@ struct llama_client_slot}bool has_budget(gpt_params &global_params) {+ if (params.n_predict == -1 && global_params.n_predict == -1)+ {+ return true; // limitless+ }+n_remaining = -1;- if(params.n_predict != -1)++ if (params.n_predict != -1){n_remaining = params.n_predict - n_decoded;}@@ -461,7 +249,8 @@ struct llama_client_slot{n_remaining = global_params.n_predict - n_decoded;}- return n_remaining > 0 || n_remaining == -1; // no budget || limitless++ return n_remaining > 0; // no budget}bool available() const {@@ -482,7 +271,7 @@ struct llama_client_slot}void release() {- if (state == IDLE || state == PROCESSING)+ if (state == PROCESSING){t_token_generation = (ggml_time_us() - t_start_genereration) / 1e3;command = RELEASE;@@ -530,7 +319,6 @@ struct llama_server_contextbool all_slots_are_idle = false;bool add_bos_token = true;- int32_t id_gen;int32_t n_ctx; // total context for all clients / slots// system prompt@@ -545,13 +333,8 @@ struct llama_server_context// slots / clientsstd::vector<llama_client_slot> slots;- std::vector<task_server> queue_tasks;- std::vector<task_result> queue_results;- std::vector<task_multi> queue_multitasks;- std::mutex mutex_tasks; // also guards id_gen, and queue_multitasks- std::condition_variable condition_tasks;- std::mutex mutex_results;- std::condition_variable condition_results;+ llama_server_queue queue_tasks;+ llama_server_response queue_results;~llama_server_context(){@@ -610,8 +393,6 @@ struct llama_server_context}void initialize() {- id_gen = 0;-// create slotsall_slots_are_idle = true;@@ -1107,7 +888,7 @@ struct llama_server_context}// check the limits- if (slot.n_decoded > 2 && slot.has_next_token && !slot.has_budget(params))+ if (slot.n_decoded > 0 && slot.has_next_token && !slot.has_budget(params)){slot.stopped_limit = true;slot.has_next_token = false;@@ -1171,41 +952,16 @@ struct llama_server_contextreturn slot.images.size() > 0;}- void send_error(task_server& task, std::string error)+ void send_error(task_server& task, const std::string &error){- std::unique_lock<std::mutex> lock(mutex_results);+ LOG_TEE("task %i - error: %s\n", task.id, error.c_str());task_result res;res.id = task.id;res.multitask_id = task.multitask_id;res.stop = false;res.error = true;res.result_json = { { "content", error } };- queue_results.push_back(res);- condition_results.notify_all();- }-- void add_multi_task(int id, std::vector<int>& sub_ids)- {- std::lock_guard<std::mutex> lock(mutex_tasks);- task_multi multi;- multi.id = id;- std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));- queue_multitasks.push_back(multi);- condition_tasks.notify_one();- }-- void update_multi_task(int multitask_id, int subtask_id, task_result& result)- {- std::lock_guard<std::mutex> lock(mutex_tasks);- for (auto& multitask : queue_multitasks)- {- if (multitask.id == multitask_id)- {- multitask.subtasks_remaining.erase(subtask_id);- multitask.results.push_back(result);- condition_tasks.notify_one();- }- }+ queue_results.send(res);}json get_model_props()@@ -1251,7 +1007,6 @@ struct llama_server_contextvoid send_partial_response(llama_client_slot &slot, completion_token_output tkn){- std::unique_lock<std::mutex> lock(mutex_results);task_result res;res.id = slot.task_id;res.multitask_id = slot.multitask_id;@@ -1286,13 +1041,11 @@ struct llama_server_contextres.result_json["model"] = slot.oaicompat_model;}- queue_results.push_back(res);- condition_results.notify_all();+ queue_results.send(res);}void send_final_response(llama_client_slot &slot){- std::unique_lock<std::mutex> lock(mutex_results);task_result res;res.id = slot.task_id;res.multitask_id = slot.multitask_id;@@ -1341,19 +1094,11 @@ struct llama_server_contextres.result_json["model"] = slot.oaicompat_model;}- // parent multitask, if any, needs to be updated- if (slot.multitask_id != -1)- {- update_multi_task(slot.multitask_id, slot.task_id, res);- }-- queue_results.push_back(res);- condition_results.notify_all();+ queue_results.send(res);}void send_embedding(llama_client_slot &slot){- std::unique_lock<std::mutex> lock(mutex_results);task_result res;res.id = slot.task_id;res.multitask_id = slot.multitask_id;@@ -1380,66 +1125,28 @@ struct llama_server_context{"embedding", embedding },};}- queue_results.push_back(res);- condition_results.notify_all();+ queue_results.send(res);}- int request_completion(json data, bool infill, bool embedding, int multitask_id)+ void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id){- std::unique_lock<std::mutex> lock(mutex_tasks);task_server task;- task.id = id_gen++;+ task.id = task_id;task.target_id = 0;task.data = std::move(data);task.infill_mode = infill;task.embedding_mode = embedding;- task.type = COMPLETION_TASK;+ task.type = TASK_TYPE_COMPLETION;task.multitask_id = multitask_id;// when a completion task's prompt array is not a singleton, we split it into multiple requests- if (task.data.at("prompt").size() > 1)+ if (task.data.count("prompt") && task.data.at("prompt").size() > 1){- lock.unlock(); // entering new func scope- return split_multiprompt_task(task);+ split_multiprompt_task(task_id, task);}// otherwise, it's a single-prompt task, we actually queue it- queue_tasks.push_back(task);- condition_tasks.notify_one();- return task.id;- }-- task_result next_result(int task_id)- {- while (true)- {- std::unique_lock<std::mutex> lock(mutex_results);- condition_results.wait(lock, [&]{- return !queue_results.empty();- });-- for (int i = 0; i < (int) queue_results.size(); i++)- {- // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result- if (queue_results[i].multitask_id == task_id)- {- update_multi_task(task_id, queue_results[i].id, queue_results[i]);- queue_results.erase(queue_results.begin() + i);- continue;- }-- if (queue_results[i].id == task_id)- {- assert(queue_results[i].multitask_id == -1);- task_result res = queue_results[i];- queue_results.erase(queue_results.begin() + i);- return res;- }- }- }-- // never reached- //return task_result{-1, false, false, {}};+ queue_tasks.post(task);}// for multiple images processing@@ -1512,127 +1219,118 @@ struct llama_server_contextvoid request_cancel(int task_id){- std::unique_lock<std::mutex> lock(mutex_tasks);task_server task;- task.id = id_gen++;- task.type = CANCEL_TASK;+ task.type = TASK_TYPE_CANCEL;task.target_id = task_id;- queue_tasks.push_back(task);- condition_tasks.notify_one();+ queue_tasks.post(task);}- int split_multiprompt_task(task_server& multiprompt_task)+ void split_multiprompt_task(int multitask_id, task_server& multiprompt_task){int prompt_count = multiprompt_task.data.at("prompt").size();assert(prompt_count > 1);- int multitask_id = id_gen++;+ // generate all the ID for subtaskstd::vector<int> subtask_ids(prompt_count);for (int i = 0; i < prompt_count; i++)+ {+ subtask_ids[i] = queue_tasks.get_new_id();+ }++ // queue up the multitask so we can track its subtask progression+ queue_tasks.add_multitask(multitask_id, subtask_ids);++ // add subtasks+ for (int i = 0; i < prompt_count; i++){json subtask_data = multiprompt_task.data;subtask_data["prompt"] = subtask_data["prompt"][i];// subtasks inherit everything else (infill mode, embedding mode, etc.)- subtask_ids[i] = request_completion(subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);+ request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);}-- // queue up the multitask so we can track its subtask progression- add_multi_task(multitask_id, subtask_ids);- return multitask_id;}- void process_tasks()+ void process_single_task(task_server& task){- std::unique_lock<std::mutex> lock(mutex_tasks);- while (!queue_tasks.empty())+ switch (task.type){- task_server task = queue_tasks.front();- queue_tasks.erase(queue_tasks.begin());- switch (task.type)- {- case COMPLETION_TASK: {- llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1));- if (slot == nullptr)- {- LOG_TEE("slot unavailable\n");- // send error result- send_error(task, "slot unavailable");- return;+ case TASK_TYPE_COMPLETION: {+ llama_client_slot *slot = get_slot(json_value(task.data, "slot_id", -1));+ if (slot == nullptr)+ {+ // if no slot is available, we defer this task for processing later+ LOG_VERBOSE("no slot is available", {});+ queue_tasks.defer(task);+ break;+ }++ if (task.data.contains("system_prompt"))+ {+ if (!all_slots_are_idle) {+ send_error(task, "system prompt can only be updated when all slots are idle");+ break;}+ process_system_prompt_data(task.data["system_prompt"]);- if (task.data.contains("system_prompt"))+ // reset cache_tokens for all slots+ for (llama_client_slot &slot : slots){- process_system_prompt_data(task.data["system_prompt"]);+ slot.cache_tokens.clear();}+ }- slot->reset();+ slot->reset();- slot->infill = task.infill_mode;- slot->embedding = task.embedding_mode;- slot->task_id = task.id;- slot->multitask_id = task.multitask_id;+ slot->infill = task.infill_mode;+ slot->embedding = task.embedding_mode;+ slot->task_id = task.id;+ slot->multitask_id = task.multitask_id;- if (!launch_slot_with_data(slot, task.data))+ if (!launch_slot_with_data(slot, task.data))+ {+ // send error result+ send_error(task, "internal_error");+ break;+ }+ } break;+ case TASK_TYPE_CANCEL: { // release slot linked with the task id+ for (auto & slot : slots)+ {+ if (slot.task_id == task.target_id){- // send error result- send_error(task, "internal_error");+ slot.release();break;}- } break;- case CANCEL_TASK: { // release slot linked with the task id- for (auto & slot : slots)- {- if (slot.task_id == task.target_id)- {- slot.release();- break;- }- }- } break;- }- }-- // remove finished multitasks from the queue of multitasks, and add the corresponding result to the result queue- auto queue_iterator = queue_multitasks.begin();- while (queue_iterator != queue_multitasks.end())- {- if (queue_iterator->subtasks_remaining.empty())- {- // all subtasks done == multitask is done- task_result aggregate_result;- aggregate_result.id = queue_iterator->id;- aggregate_result.stop = true;- aggregate_result.error = false;-- // collect json results into one json result- std::vector<json> result_jsons;- for (auto& subres : queue_iterator->results)- {- result_jsons.push_back(subres.result_json);- aggregate_result.error = aggregate_result.error && subres.error;}- aggregate_result.result_json = json{ "results", result_jsons };+ } break;+ case TASK_TYPE_NEXT_RESPONSE: {+ // do nothing+ } break;+ }+ }- std::lock_guard<std::mutex> lock(mutex_results);- queue_results.push_back(aggregate_result);- condition_results.notify_all();+ void on_finish_multitask(task_multi& multitask)+ {+ // all subtasks done == multitask is done+ task_result result;+ result.id = multitask.id;+ result.stop = true;+ result.error = false;- queue_iterator = queue_multitasks.erase(queue_iterator);- }- else- {- ++queue_iterator;- }+ // collect json results into one json result+ std::vector<json> result_jsons;+ for (auto& subres : multitask.results)+ {+ result_jsons.push_back(subres.result_json);+ result.error = result.error && subres.error;}+ result.result_json = json{ { "results", result_jsons } };+ queue_results.send(result);}bool update_slots() {- // attend tasks- process_tasks();-- // update the system prompt wait until all slots are idle state- if (system_need_update && all_slots_are_idle)+ if (system_need_update){LOG_TEE("updating system prompt\n");update_system_prompt();@@ -1647,10 +1345,12 @@ struct llama_server_contextLOG_TEE("all slots are idle and system prompt is empty, clear the KV cache\n");kv_cache_clear();}- std::unique_lock<std::mutex> lock(mutex_tasks);- condition_tasks.wait(lock, [&]{- return !queue_tasks.empty();- });+ return true;+ } else {+ task_server task;+ task.type = TASK_TYPE_NEXT_RESPONSE;+ task.target_id = -1;+ queue_tasks.post(task);}for (llama_client_slot &slot : slots)@@ -1695,6 +1395,7 @@ struct llama_server_contextslot.t_last_used = ggml_time_us();LOG_TEE("slot %d released (%d tokens in cache)\n", slot.id, (int) slot.cache_tokens.size());+ queue_tasks.notify_slot_changed();continue;}@@ -1708,7 +1409,6 @@ struct llama_server_contextllama_batch_add(batch, slot.sampled, system_tokens.size() + slot.n_past, { slot.id }, true);- slot.n_decoded += 1;slot.n_past += 1;}@@ -1723,7 +1423,8 @@ struct llama_server_contextconst bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();// empty prompt passed -> release the slot and send empty response- if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt)+ // note: infill mode allows empty prompt+ if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill){slot.release();slot.print_timings();@@ -1826,7 +1527,7 @@ struct llama_server_contextslot.cache_tokens = prompt_tokens;- if (slot.n_past == slot.num_prompt_tokens)+ if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0){// we have to evaluate at least 1 token to generate logits.LOG_TEE("slot %d : we have to evaluate at least 1 token to generate logits\n", slot.id);@@ -1926,6 +1627,7 @@ struct llama_server_contextllama_sampling_accept(slot.ctx_sampling, ctx, id, true);+ slot.n_decoded += 1;if (slot.n_decoded == 1){slot.t_start_genereration = ggml_time_us();@@ -1959,6 +1661,10 @@ struct llama_server_context}return true;}++ void run_on_all_tasks_finished() {+ update_slots();+ }};static void server_print_usage(const char *argv0, const gpt_params ¶ms,@@ -1992,14 +1698,19 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");}printf(" --numa attempt optimizations that help on some NUMA systems\n");+#ifdef LLAMA_SUPPORTS_GPU_OFFLOADprintf(" -ngl N, --n-gpu-layers N\n");printf(" number of layers to store in VRAM\n");+ printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");+ printf(" how to split the model across multiple GPUs, one of:\n");+ printf(" - none: use one GPU only\n");+ printf(" - layer (default): split layers and KV across GPUs\n");+ printf(" - row: split rows across GPUs\n");printf(" -ts SPLIT --tensor-split SPLIT\n");- printf(" how to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1\n");- printf(" -mg i, --main-gpu i the GPU to use for scratch and small tensors\n");- printf(" -nommq, --no-mul-mat-q\n");- printf(" use cuBLAS instead of custom mul_mat_q CUDA kernels.\n");- printf(" Not recommended since this is both slower and uses more VRAM.\n");+ printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");+ printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");+ printf(" or for intermediate results and KV (with split-mode = row)\n");+#endifprintf(" -m FNAME, --model FNAME\n");printf(" model path (default: %s)\n", params.model.c_str());printf(" -a ALIAS, --alias ALIAS\n");@@ -2010,6 +1721,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");+ printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);@@ -2033,9 +1745,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,std::string arg;bool invalid_param = false;- assert(FLAG_gpu == LLAMAFILE_GPU_ERROR);- FLAG_gpu = LLAMAFILE_GPU_AUTO;-for (int i = 1; i < argc; i++){arg = argv[i];@@ -2073,7 +1782,28 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- sparams.api_key = argv[i];+ sparams.api_keys.push_back(argv[i]);+ }+ else if (arg == "--api-key-file")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ std::ifstream key_file(argv[i]);+ if (!key_file) {+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);+ invalid_param = true;+ break;+ }+ std::string key;+ while (std::getline(key_file, key)) {+ if (key.size() > 0) {+ sparams.api_keys.push_back(key);+ }+ }+ key_file.close();}else if (arg == "--timeout" || arg == "-to"){@@ -2215,7 +1945,37 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}+#ifdef LLAMA_SUPPORTS_GPU_OFFLOADparams.n_gpu_layers = std::stoi(argv[i]);+#else+ LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "+ "See main README.md for information on enabling GPU BLAS support",+ {{"n_gpu_layers", params.n_gpu_layers}});+#endif+ }+ else if (arg == "--split-mode" || arg == "-sm")+ {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ std::string arg_next = argv[i];+ if (arg_next == "none")+ {+ params.split_mode = LLAMA_SPLIT_NONE;+ }+ else if (arg_next == "layer")+ {+ params.split_mode = LLAMA_SPLIT_LAYER;+ }+ else if (arg_next == "row")+ {+ params.split_mode = LLAMA_SPLIT_ROW;+ }+ else {+ invalid_param = true;+ break;+ }}else if (arg == "--tensor-split" || arg == "-ts"){@@ -2470,270 +2230,37 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}}+/* llama.cpp completion api semantics */+static json format_partial_response(+ llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs+) {+ json res = json+ {+ {"content", content },+ {"stop", false},+ {"slot_id", slot->id },+ {"multimodal", llama.multimodal }+ };-static std::string random_string()-{- static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");-- std::random_device rd;- std::mt19937 generator(rd());-- std::string result(32, ' ');-- for (int i = 0; i < 32; ++i) {- result[i] = str[generator() % str.size()];+ if (slot->sparams.n_probs > 0)+ {+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);}- return result;+ return res;}-static std::string gen_chatcmplid()+static json format_tokenizer_response(const std::vector<llama_token> &tokens){- std::stringstream chatcmplid;- chatcmplid << "chatcmpl-" << random_string();- return chatcmplid.str();+ return json{+ {"tokens", tokens}};}-std::string format_chatml(std::vector<json> messages)+static json format_detokenized_response(std::string content){- std::ostringstream chatml_msgs;-- for (auto it = messages.begin(); it != messages.end(); ++it) {- chatml_msgs << "<|im_start|>"- << json_value(*it, "role", std::string("user")) << '\n';- chatml_msgs << json_value(*it, "content", std::string(""))- << "<|im_end|>\n";- }-- chatml_msgs << "<|im_start|>assistant" << '\n';-- return chatml_msgs.str();-}--/* llama.cpp completion api semantics */-json oaicompat_completion_params_parse(- const json &body /* openai api json semantics */)-{- json llama_params;-- llama_params["__oaicompat"] = true;-- // Map OpenAI parameters to llama.cpp parameters- //- // For parameters that are defined by the OpenAI documentation (e.g.- // temperature), we explicitly specify OpenAI's intended default; we- // need to do that because sometimes OpenAI disagrees with llama.cpp- //- // https://platform.openai.com/docs/api-reference/chat/create- llama_sampling_params default_sparams;- llama_params["model"] = json_value(body, "model", std::string("unknown"));- llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt'- llama_params["cache_prompt"] = json_value(body, "cache_prompt", false);- llama_params["temperature"] = json_value(body, "temperature", 0.0);- llama_params["top_k"] = json_value(body, "top_k", default_sparams.top_k);- llama_params["top_p"] = json_value(body, "top_p", 1.0);- llama_params["n_predict"] = json_value(body, "max_tokens", -1);- llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());- llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);- llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);- llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED);- llama_params["stream"] = json_value(body, "stream", false);- llama_params["mirostat"] = json_value(body, "mirostat", default_sparams.mirostat);- llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", default_sparams.mirostat_tau);- llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", default_sparams.mirostat_eta);- llama_params["penalize_nl"] = json_value(body, "penalize_nl", default_sparams.penalize_nl);- llama_params["typical_p"] = json_value(body, "typical_p", default_sparams.typical_p);- llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);- llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);- llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);-- if (body.count("grammar") != 0) {- llama_params["grammar"] = json_value(body, "grammar", json::object());- }-- // Handle 'stop' field- if (body.contains("stop") && body["stop"].is_string()) {- llama_params["stop"] = json::array({body["stop"].get<std::string>()});- } else {- llama_params["stop"] = json_value(body, "stop", json::array());- }-- // Ensure there is ChatML-specific end sequence among stop words- llama_params["stop"].push_back("<|im_end|>");-- return llama_params;-}--static json format_final_response_oaicompat(const json &request, const task_result &response, bool streaming = false)-{- json result = response.result_json;-- bool stopped_word = result.count("stopped_word") != 0;- bool stopped_eos = json_value(result, "stopped_eos", false);- int num_tokens_predicted = json_value(result, "tokens_predicted", 0);- int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason = "length";- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }-- json choices =- streaming ? json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}})- : json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"message", json{{"content", content},- {"role", "assistant"}}}}});-- std::time_t t = std::time(0);-- json res =- json{{"choices", choices},- {"created", t},- {"model",- json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},- {"object", streaming ? "chat.completion.chunk" : "chat.completion"},- {"usage",- json{{"completion_tokens", num_tokens_predicted},- {"prompt_tokens", num_prompt_tokens},- {"total_tokens", num_tokens_predicted + num_prompt_tokens}}},- {"id", gen_chatcmplid()}};-- if (server_verbose) {- res["__verbose"] = result;- }-- if (result.contains("completion_probabilities")) {- res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());- }-- return res;-}--// return value is vector as there is one case where we might need to generate two responses-static std::vector<json> format_partial_response_oaicompat(const task_result &response) {- json result = response.result_json;-- if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) {- return std::vector<json>({response.result_json});- }-- bool first = json_value(result, "oaicompat_token_ctr", 0) == 0;- std::string modelname = json_value(result, "model", std::string(DEFAULT_OAICOMPAT_MODEL));-- bool stopped_word = json_value(result, "stopped_word", false);- bool stopped_eos = json_value(result, "stopped_eos", false);- bool stopped_limit = json_value(result, "stopped_limit", false);- std::string content = json_value(result, "content", std::string(""));-- std::string finish_reason;- if (stopped_word || stopped_eos) {- finish_reason = "stop";- }- if (stopped_limit) {- finish_reason = "length";- }-- std::time_t t = std::time(0);-- json choices;-- if (!finish_reason.empty()) {- choices = json::array({json{{"finish_reason", finish_reason},- {"index", 0},- {"delta", json::object()}}});- } else {- if (first) {- if (content.empty()) {- choices = json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{{"role", "assistant"}}}}});- } else {- // We have to send this as two updates to conform to openai behavior- json initial_ret = json{{"choices", json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"role", "assistant"}- }}}})},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- json second_ret = json{- {"choices", json::array({json{{"finish_reason", nullptr},- {"index", 0},- {"delta", json{- {"content", content}}}- }})},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- return std::vector<json>({initial_ret, second_ret});- }- } else {- // Some idiosyncrasy in task processing logic makes several trailing calls- // with empty content, we ignore these at the calee site.- if (content.empty()) {- return std::vector<json>({json::object()});- }-- choices = json::array({json{- {"finish_reason", nullptr},- {"index", 0},- {"delta",- json{- {"content", content},- }},- }});- }- }-- json ret = json{{"choices", choices},- {"created", t},- {"id", gen_chatcmplid()},- {"model", modelname},- {"object", "chat.completion.chunk"}};-- return std::vector<json>({ret});-}--static json format_partial_response(- llama_server_context &llama, llama_client_slot *slot, const std::string &content, const std::vector<completion_token_output> &probs-) {- json res = json- {- {"content", content },- {"stop", false},- {"slot_id", slot->id },- {"multimodal", llama.multimodal }- };-- if (slot->sparams.n_probs > 0)- {- res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);- }-- return res;-}--static json format_tokenizer_response(const std::vector<llama_token> &tokens)-{- return json{- {"tokens", tokens}};-}--static json format_detokenized_response(std::string content)-{- return json{- {"content", content}};-}+ return json{+ {"content", content}};+}static void log_server_request(const httplib::Request &req, const httplib::Response &res)@@ -2819,20 +2346,196 @@ int server_cli(int argc, char **argv){"system_info", llama_print_system_info()},});+ httplib::Server svr;++ std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};++ svr.set_default_headers({{"Server", "llama.cpp"}});++ // CORS preflight+ svr.Options(R"(.*)", [](const httplib::Request &req, httplib::Response &res) {+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));+ res.set_header("Access-Control-Allow-Credentials", "true");+ res.set_header("Access-Control-Allow-Methods", "POST");+ res.set_header("Access-Control-Allow-Headers", "*");+ });++ svr.Get("/health", [&](const httplib::Request&, httplib::Response& res) {+ server_state current_state = state.load();+ switch(current_state) {+ case SERVER_STATE_READY:+ res.set_content(R"({"status": "ok"})", "application/json");+ res.status = 200; // HTTP OK+ break;+ case SERVER_STATE_LOADING_MODEL:+ res.set_content(R"({"status": "loading model"})", "application/json");+ res.status = 503; // HTTP Service Unavailable+ break;+ case SERVER_STATE_ERROR:+ res.set_content(R"({"status": "error", "error": "Model failed to load"})", "application/json");+ res.status = 500; // HTTP Internal Server Error+ break;+ }+ });++ svr.set_logger(log_server_request);++ svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)+ {+ const char fmt[] = "500 Internal Server Error\n%s";+ char buf[BUFSIZ];+ try+ {+ std::rethrow_exception(std::move(ep));+ }+ catch (std::exception &e)+ {+ snprintf(buf, sizeof(buf), fmt, e.what());+ }+ catch (...)+ {+ snprintf(buf, sizeof(buf), fmt, "Unknown Exception");+ }+ res.set_content(buf, "text/plain; charset=utf-8");+ res.status = 500;+ });++ svr.set_error_handler([](const httplib::Request &, httplib::Response &res)+ {+ if (res.status == 401)+ {+ res.set_content("Unauthorized", "text/plain; charset=utf-8");+ }+ if (res.status == 400)+ {+ res.set_content("Invalid request", "text/plain; charset=utf-8");+ }+ else if (res.status == 404)+ {+ res.set_content("File Not Found", "text/plain; charset=utf-8");+ res.status = 404;+ }+ });++ // set timeouts and change hostname and port+ svr.set_read_timeout (sparams.read_timeout);+ svr.set_write_timeout(sparams.write_timeout);++ for (int i = 0;; ++i) {+ if (svr.bind_to_port(sparams.hostname, sparams.port)) {+ break;+ } else if (i < 10) {+ ++sparams.port;+ } else {+ fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);+ return 1;+ }+ }++ // Set the base directory for serving static files+ svr.set_base_dir(sparams.public_path);++ // to make it ctrl+clickable:+ const char *connect_host;+ if (sparams.hostname != "0.0.0.0") {+ connect_host = sparams.hostname.c_str();+ LOG_TEE("\nllama server listening at http://%s:%d\n\n",+ sparams.hostname.c_str(), sparams.port);+ } else {+ struct ifaddrs *ifaddrs;+ connect_host = "127.0.0.1";+ if (!getifaddrs(&ifaddrs)) {+ LOG_TEE("\n");+ for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {+ char buf[128];+ if (!(ifa->ifa_flags & IFF_UP)) continue;+ LOG_TEE("llama server listening at http://%s%s%s:%d\n",+ ifa->ifa_addr->sa_family == AF_INET6 ? "[" : "",+ sockaddr2str(ifa->ifa_addr, buf, sizeof(buf)),+ ifa->ifa_addr->sa_family == AF_INET6 ? "]" : "",+ sparams.port);+ }+ LOG_TEE("\n");+ } else {+ perror("getifaddrs");+ LOG_TEE("\nllama server listening at http://%s:%d\n\n",+ sparams.hostname.c_str(), sparams.port);+ }+ }++ std::unordered_map<std::string, std::string> log_data;+ log_data["hostname"] = sparams.hostname;+ log_data["port"] = std::to_string(sparams.port);++ if (sparams.api_keys.size() == 1) {+ log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);+ } else if (sparams.api_keys.size() > 1) {+ log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";+ }++ LOG_INFO("HTTP server listening", log_data);+ // run the HTTP server in a thread - see comment below+ std::thread t([&]()+ {+ if (!svr.listen_after_bind())+ {+ state.store(SERVER_STATE_ERROR);+ return 1;+ }++ return 0;+ });+// load the modelif (!llama.load_model(params)){+ state.store(SERVER_STATE_ERROR);return 1;+ } else {+ llama.initialize();+ state.store(SERVER_STATE_READY);+ LOG_INFO("model loaded", {});}- llama.initialize();+ // launch browser tab+ if (!sparams.nobrowser) {+ char url[128];+ snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);+ llamafile_launch_browser(url);+ }- httplib::Server svr;+ if (!params.unsecure && !llamafile_has_gpu()) {+ // Enables pledge() security on Linux and OpenBSD.+ // - We do this *after* binding the server socket.+ // - We do this *after* opening the log file for writing.+ // - We do this *after* opening a tab in the user browser.+ // - We do this *before* loading any weights or graphdefs.+ // In effect, what this does is:+ // - Filesystem access is disabled entirely (except ZipOS).+ // - On Linux, network access is restricted to accept() only.+ // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.+ char promises[32];+ if (IsOpenbsd()) {+ strlcpy(promises, "stdio inet", sizeof(promises));+ } else {+ strlcpy(promises, "stdio anet", sizeof(promises));+ }+ if (!startswith(sparams.public_path.c_str(), "/zip/")) {+ strlcat(promises, " rpath", sizeof(promises));+ }+ __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;+ if (pledge(0, 0)) {+ LOG_TEE("warning: this OS doesn't support pledge() security\n");+ } else if (pledge(promises, 0)) {+ perror("pledge");+ exit(1);+ }+ }// Middleware for API key validationauto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {// If API key is not set, skip validation- if (sparams.api_key.empty()) {+ if (sparams.api_keys.empty()) {return true;}@@ -2841,7 +2544,7 @@ int server_cli(int argc, char **argv)std::string prefix = "Bearer ";if (auth_header.substr(0, prefix.size()) == prefix) {std::string received_api_key = auth_header.substr(prefix.size());- if (received_api_key == sparams.api_key) {+ if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) {return true; // API key is valid}}@@ -2855,13 +2558,9 @@ int server_cli(int argc, char **argv)return false;};- svr.set_default_headers({{"Server", "llama.cpp"},- {"Access-Control-Allow-Origin", "*"},- {"Access-Control-Allow-Headers", "content-type"}});-- svr.Get("/props", [&llama](const httplib::Request & /*req*/, httplib::Response &res)+ svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res){- res.set_header("Access-Control-Allow-Origin", "*");+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));json data = {{ "user_name", llama.name_user.c_str() },{ "assistant_name", llama.name_assistant.c_str() }@@ -2871,14 +2570,17 @@ int server_cli(int argc, char **argv)svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {return;}json data = json::parse(req.body);- const int task_id = llama.request_completion(data, false, false, -1);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, false, false, -1);if (!json_value(data, "stream", false)) {std::string completion_text;- task_result result = llama.next_result(task_id);+ task_result result = llama.queue_results.recv(task_id);if (!result.error && result.stop) {res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");}@@ -2886,14 +2588,14 @@ int server_cli(int argc, char **argv){res.status = 404;res.set_content(result.result_json["content"], "text/plain; charset=utf-8");- return;}+ llama.queue_results.remove_waiting_task_id(task_id);} else {const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink){while (true){- task_result result = llama.next_result(task_id);+ task_result result = llama.queue_results.recv(task_id);if (!result.error) {const std::string str ="data: " +@@ -2904,6 +2606,7 @@ int server_cli(int argc, char **argv)});if (!sink.write(str.c_str(), str.size())){+ llama.queue_results.remove_waiting_task_id(task_id);return false;}if (result.stop) {@@ -2919,11 +2622,14 @@ int server_cli(int argc, char **argv)});if (!sink.write(str.c_str(), str.size())){+ llama.queue_results.remove_waiting_task_id(task_id);return false;}break;}}++ llama.queue_results.remove_waiting_task_id(task_id);sink.done();return true;};@@ -2932,16 +2638,16 @@ int server_cli(int argc, char **argv){// cancelllama.request_cancel(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);};res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);}});--- svr.Get("/v1/models", [¶ms](const httplib::Request&, httplib::Response& res)+ svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));std::time_t t = std::time(0);json models = {@@ -2959,19 +2665,23 @@ int server_cli(int argc, char **argv)res.set_content(models.dump(), "application/json; charset=utf-8");});+// TODO: add mount point without "/v1" prefix -- how?svr.Post("/v1/chat/completions", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {return;}json data = oaicompat_completion_params_parse(json::parse(req.body));- const int task_id = llama.request_completion(data, false, false, -1);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, false, false, -1);if (!json_value(data, "stream", false)) {std::string completion_text;- task_result result = llama.next_result(task_id);+ task_result result = llama.queue_results.recv(task_id);if (!result.error && result.stop) {json oaicompat_result = format_final_response_oaicompat(data, result);@@ -2982,12 +2692,12 @@ int server_cli(int argc, char **argv)} else {res.status = 500;res.set_content(result.result_json["content"], "text/plain; charset=utf-8");- return;}+ llama.queue_results.remove_waiting_task_id(task_id);} else {const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) {while (true) {- task_result llama_result = llama.next_result(task_id);+ task_result llama_result = llama.queue_results.recv(task_id);if (!llama_result.error) {std::vector<json> result_array = format_partial_response_oaicompat( llama_result);@@ -3000,6 +2710,7 @@ int server_cli(int argc, char **argv)"\n\n";LOG_VERBOSE("data stream", {{"to_send", str}});if (!sink.write(str.c_str(), str.size())) {+ llama.queue_results.remove_waiting_task_id(task_id);return false;}}@@ -3015,18 +2726,21 @@ int server_cli(int argc, char **argv)"\n\n";LOG_VERBOSE("data stream", {{"to_send", str}});if (!sink.write(str.c_str(), str.size())) {+ llama.queue_results.remove_waiting_task_id(task_id);return false;}break;}}sink.done();+ llama.queue_results.remove_waiting_task_id(task_id);return true;};auto on_complete = [task_id, &llama](bool) {// cancel requestllama.request_cancel(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);};res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);@@ -3035,14 +2749,17 @@ int server_cli(int argc, char **argv)svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {return;}json data = json::parse(req.body);- const int task_id = llama.request_completion(data, true, false, -1);+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, data, true, false, -1);if (!json_value(data, "stream", false)) {std::string completion_text;- task_result result = llama.next_result(task_id);+ task_result result = llama.queue_results.recv(task_id);if (!result.error && result.stop){res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");@@ -3051,13 +2768,13 @@ int server_cli(int argc, char **argv){res.status = 404;res.set_content(result.result_json["content"], "text/plain; charset=utf-8");- return;}+ llama.queue_results.remove_waiting_task_id(task_id);} else {const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink & sink) {while (true){- task_result result = llama.next_result(task_id);+ task_result result = llama.queue_results.recv(task_id);if (!result.error) {const std::string str ="data: " +@@ -3068,6 +2785,7 @@ int server_cli(int argc, char **argv)});if (!sink.write(str.c_str(), str.size())){+ llama.queue_results.remove_waiting_task_id(task_id);return false;}if (result.stop)@@ -3081,8 +2799,8 @@ int server_cli(int argc, char **argv)}}+ llama.queue_results.remove_waiting_task_id(task_id);sink.done();-return true;};@@ -3107,6 +2825,7 @@ int server_cli(int argc, char **argv)svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);std::vector<llama_token> tokens;if (body.count("content") != 0)@@ -3119,6 +2838,7 @@ int server_cli(int argc, char **argv)svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);std::string content;if (body.count("tokens") != 0)@@ -3133,6 +2853,7 @@ int server_cli(int argc, char **argv)svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res){+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);json prompt;if (body.count("content") != 0)@@ -3153,164 +2874,46 @@ int server_cli(int argc, char **argv)image_data = "";}- const int task_id = llama.request_completion({ {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);- task_result result = llama.next_result(task_id);- return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");- });-- svr.set_logger(log_server_request);-- svr.set_exception_handler([](const httplib::Request &, httplib::Response &res, std::exception_ptr ep)- {- const char fmt[] = "500 Internal Server Error\n%s";- char buf[BUFSIZ];- try- {- std::rethrow_exception(std::move(ep));- }- catch (std::exception &e)- {- snprintf(buf, sizeof(buf), fmt, e.what());- }- catch (...)- {- snprintf(buf, sizeof(buf), fmt, "Unknown Exception");- }- res.set_content(buf, "text/plain; charset=utf-8");- res.status = 500;- });-- svr.set_error_handler([](const httplib::Request &, httplib::Response &res)- {- if (res.status == 401)- {- res.set_content("Unauthorized", "text/plain; charset=utf-8");- }- if (res.status == 400)- {- res.set_content("Invalid request", "text/plain; charset=utf-8");- }- else if (res.status == 404)- {- res.set_content("File Not Found", "text/plain; charset=utf-8");- res.status = 404;- }- });-- // set timeouts and change hostname and port- svr.set_read_timeout (sparams.read_timeout);- svr.set_write_timeout(sparams.write_timeout);-- for (int i = 0;; ++i) {- if (svr.bind_to_port(sparams.hostname, sparams.port)) {- break;- } else if (i < 10) {- ++sparams.port;- } else {- fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);- return 1;- }- }-- // Set the base directory for serving static files- svr.set_base_dir(sparams.public_path);-- // to make it ctrl+clickable:- const char *connect_host;- if (sparams.hostname != "0.0.0.0") {- connect_host = sparams.hostname.c_str();- LOG_TEE("\nllama server listening at http://%s:%d\n\n",- sparams.hostname.c_str(), sparams.port);- } else {- struct ifaddrs *ifaddrs;- connect_host = "127.0.0.1";- if (!getifaddrs(&ifaddrs)) {- LOG_TEE("\n");- for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {- char buf[128];- if (!(ifa->ifa_flags & IFF_UP)) continue;- LOG_TEE("llama server listening at http://%s%s%s:%d\n",- ifa->ifa_addr->sa_family == AF_INET6 ? "[" : "",- sockaddr2str(ifa->ifa_addr, buf, sizeof(buf)),- ifa->ifa_addr->sa_family == AF_INET6 ? "]" : "",- sparams.port);- }- LOG_TEE("\n");- } else {- perror("getifaddrs");- LOG_TEE("\nllama server listening at http://%s:%d\n\n",- sparams.hostname.c_str(), sparams.port);- }- }-- // launch browser tab- if (!sparams.nobrowser) {- char url[128];- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);- llamafile_launch_browser(url);- }-- if (!params.unsecure && !llamafile_gpu_supported()) {- // Enables pledge() security on Linux and OpenBSD.- // - We do this *after* binding the server socket.- // - We do this *after* opening the log file for writing.- // - We do this *after* opening a tab in the user browser.- // - We do this *before* loading any weights or graphdefs.- // In effect, what this does is:- // - Filesystem access is disabled entirely (except ZipOS).- // - On Linux, network access is restricted to accept() only.- // Cosmopolitan Libc implements pledge() on Linux using SECCOMP.- char promises[32];- if (IsOpenbsd()) {- strlcpy(promises, "stdio inet", sizeof(promises));- } else {- strlcpy(promises, "stdio anet", sizeof(promises));- }- if (!startswith(sparams.public_path.c_str(), "/zip/")) {- strlcat(promises, " rpath", sizeof(promises));- }- __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;- if (pledge(0, 0)) {- LOG_TEE("warning: this OS doesn't support pledge() security\n");- } else if (pledge(promises, 0)) {- perror("pledge");- exit(1);- }- }-- std::unordered_map<std::string, std::string> log_data;- log_data["hostname"] = sparams.hostname;- log_data["port"] = std::to_string(sparams.port);-- if (!sparams.api_key.empty()) {- log_data["api_key"] = "api_key: ****" + sparams.api_key.substr(sparams.api_key.length() - 4);- }+ // create and queue the task+ const int task_id = llama.queue_tasks.get_new_id();+ llama.queue_results.add_waiting_task_id(task_id);+ llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);- LOG_INFO("HTTP server listening", log_data);+ // get the result+ task_result result = llama.queue_results.recv(task_id);+ llama.queue_results.remove_waiting_task_id(task_id);- // run the HTTP server in a thread - see comment below- std::thread t([&]()- {- if (!svr.listen_after_bind())- {- return 1;- }-- return 0;+ // send the result+ return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");});// GG: if I put the main loop inside a thread, it crashes on the first request when build in Debug!?// "Bus error: 10" - this is on macOS, it does not crash on Linux//std::thread t2([&]()- {+ /*{bool running = true;while (running){running = llama.update_slots();}- }+ }*///);+ llama.queue_tasks.on_new_task(std::bind(+ &llama_server_context::process_single_task, &llama, std::placeholders::_1));+ llama.queue_tasks.on_finish_multitask(std::bind(+ &llama_server_context::on_finish_multitask, &llama, std::placeholders::_1));+ llama.queue_tasks.on_all_tasks_finished(std::bind(+ &llama_server_context::run_on_all_tasks_finished, &llama));+ llama.queue_results.on_multitask_update(std::bind(+ &llama_server_queue::update_multitask,+ &llama.queue_tasks,+ std::placeholders::_1,+ std::placeholders::_2,+ std::placeholders::_3+ ));+ llama.queue_tasks.start_loop();+t.join();llama_backend_free();diff --git a/llama.cpp/server/utils.h b/llama.cpp/server/utils.hnew file mode 100644index 00000000..a0a6235c--- /dev/null+++ b/llama.cpp/server/utils.h@@ -0,0 +1,510 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi++#pragma once++#include <string>+#include <vector>+#include <set>+#include <mutex>+#include <condition_variable>+#include <unordered_map>++#include "json.h"++#include "llama.cpp/llava/clip.h"++using json = nlohmann::json;++extern bool server_verbose;++#ifndef SERVER_VERBOSE+#define SERVER_VERBOSE 1+#endif++#if SERVER_VERBOSE != 1+#define LOG_VERBOSE(MSG, ...)+#else+#define LOG_VERBOSE(MSG, ...) \+ do \+ { \+ if (server_verbose) \+ { \+ server_log("VERBOSE", __func__, __LINE__, MSG, __VA_ARGS__); \+ } \+ } while (0)+#endif++#define LOG_ERROR( MSG, ...) server_log("ERROR", __func__, __LINE__, MSG, __VA_ARGS__)+#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__)+#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)++//+// parallel+//++enum server_state {+ SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet+ SERVER_STATE_READY, // Server is ready and model is loaded+ SERVER_STATE_ERROR // An error occurred, load_model failed+};++enum task_type {+ TASK_TYPE_COMPLETION,+ TASK_TYPE_CANCEL,+ TASK_TYPE_NEXT_RESPONSE+};++struct task_server {+ int id = -1; // to be filled by llama_server_queue+ int target_id;+ task_type type;+ json data;+ bool infill_mode = false;+ bool embedding_mode = false;+ int multitask_id = -1;+};++struct task_result {+ int id;+ int multitask_id = -1;+ bool stop;+ bool error;+ json result_json;+};++struct task_multi {+ int id;+ std::set<int> subtasks_remaining{};+ std::vector<task_result> results{};+};++// TODO: can become bool if we can't find use of more states+enum slot_state+{+ IDLE,+ PROCESSING,+};++enum slot_command+{+ NONE,+ LOAD_PROMPT,+ RELEASE,+};++struct slot_params+{+ bool stream = true;+ bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt++ uint32_t seed = -1; // RNG seed+ int32_t n_keep = 0; // number of tokens to keep from initial prompt+ int32_t n_predict = -1; // new tokens to predict++ std::vector<std::string> antiprompt;++ json input_prefix;+ json input_suffix;+};++struct slot_image+{+ int32_t id;++ bool request_encode_image = false;+ float * image_embedding = nullptr;+ int32_t image_tokens = 0;++ clip_image_u8 * img_data;++ std::string prefix_prompt; // before of this image+};++// completion token output with probabilities+struct completion_token_output+{+ struct token_prob+ {+ llama_token tok;+ float prob;+ };++ std::vector<token_prob> probs;+ llama_token tok;+ std::string text_to_send;+};++static inline void server_log(const char *level, const char *function, int line,+ const char *message, const nlohmann::ordered_json &extra)+{+ nlohmann::ordered_json log+ {+ {"timestamp", time(nullptr)},+ {"level", level},+ {"function", function},+ {"line", line},+ {"message", message},+ };++ if (!extra.empty())+ {+ log.merge_patch(extra);+ }++ const std::string str = log.dump(-1, ' ', false, json::error_handler_t::replace);+ printf("%.*s\n", (int)str.size(), str.data());+ fflush(stdout);+}++//+// server utils+//++template <typename T>+static T json_value(const json &body, const std::string &key, const T &default_value)+{+ // Fallback null to default value+ return body.contains(key) && !body.at(key).is_null()+ ? body.value(key, default_value)+ : default_value;+}++inline std::string format_chatml(std::vector<json> messages)+{+ std::ostringstream chatml_msgs;++ for (auto it = messages.begin(); it != messages.end(); ++it) {+ chatml_msgs << "<|im_start|>"+ << json_value(*it, "role", std::string("user")) << '\n';+ chatml_msgs << json_value(*it, "content", std::string(""))+ << "<|im_end|>\n";+ }++ chatml_msgs << "<|im_start|>assistant" << '\n';++ return chatml_msgs.str();+}++//+// work queue utils+//++struct llama_server_queue {+ int id = 0;+ std::mutex mutex_tasks;+ // queues+ std::vector<task_server> queue_tasks;+ std::vector<task_server> queue_tasks_deferred;+ std::vector<task_multi> queue_multitasks;+ std::condition_variable condition_tasks;+ // callback functions+ std::function<void(task_server&)> callback_new_task;+ std::function<void(task_multi&)> callback_finish_multitask;+ std::function<void(void)> callback_all_task_finished;++ // Add a new task to the end of the queue+ int post(task_server task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (task.id == -1) {+ task.id = id++;+ }+ queue_tasks.push_back(std::move(task));+ condition_tasks.notify_one();+ return task.id;+ }++ // Add a new task, but defer until one slot is available+ void (defer)(task_server task) {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ queue_tasks_deferred.push_back(std::move(task));+ }++ // Get the next id for creating anew task+ int get_new_id() {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ return id++;+ }++ // Register function to process a new task+ void on_new_task(std::function<void(task_server&)> callback) {+ callback_new_task = callback;+ }++ // Register function to process a multitask+ void on_finish_multitask(std::function<void(task_multi&)> callback) {+ callback_finish_multitask = callback;+ }++ // Register the function to be called when the batch of tasks is finished+ void on_all_tasks_finished(std::function<void(void)> callback) {+ callback_all_task_finished = callback;+ }++ // Call when the state of one slot is changed+ void notify_slot_changed() {+ // move deferred tasks back to main loop+ std::unique_lock<std::mutex> lock(mutex_tasks);+ for (auto & task : queue_tasks_deferred) {+ queue_tasks.push_back(std::move(task));+ }+ queue_tasks_deferred.clear();+ }++ // Start the main loop. This call is blocking+ void start_loop() {+ while (true) {+ // new task arrived+ LOG_VERBOSE("have new task", {});+ {+ while (true)+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ lock.unlock();+ break;+ }+ task_server task = queue_tasks.front();+ queue_tasks.erase(queue_tasks.begin());+ lock.unlock();+ LOG_VERBOSE("callback_new_task", {});+ callback_new_task(task);+ }+ LOG_VERBOSE("callback_all_task_finished", {});+ // process and update all the multitasks+ auto queue_iterator = queue_multitasks.begin();+ while (queue_iterator != queue_multitasks.end())+ {+ if (queue_iterator->subtasks_remaining.empty())+ {+ // all subtasks done == multitask is done+ task_multi current_multitask = *queue_iterator;+ callback_finish_multitask(current_multitask);+ // remove this multitask+ queue_iterator = queue_multitasks.erase(queue_iterator);+ }+ else+ {+ ++queue_iterator;+ }+ }+ // all tasks in the current loop is finished+ callback_all_task_finished();+ }+ LOG_VERBOSE("wait for new task", {});+ // wait for new task+ {+ std::unique_lock<std::mutex> lock(mutex_tasks);+ if (queue_tasks.empty()) {+ condition_tasks.wait(lock, [&]{+ return !queue_tasks.empty();+ });+ }+ }+ }+ }++ //+ // functions to manage multitasks+ //++ // add a multitask by specifying the id of all subtask (subtask is a task_server)+ void add_multitask(int multitask_id, std::vector<int>& sub_ids)+ {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ task_multi multi;+ multi.id = multitask_id;+ std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));+ queue_multitasks.push_back(multi);+ }++ // updatethe remaining subtasks, while appending results to multitask+ void update_multitask(int multitask_id, int subtask_id, task_result& result)+ {+ std::lock_guard<std::mutex> lock(mutex_tasks);+ for (auto& multitask : queue_multitasks)+ {+ if (multitask.id == multitask_id)+ {+ multitask.subtasks_remaining.erase(subtask_id);+ multitask.results.push_back(result);+ }+ }+ }+};++struct llama_server_response {+ typedef std::function<void(int, int, task_result&)> callback_multitask_t;+ callback_multitask_t callback_update_multitask;+ // for keeping track of all tasks waiting for the result+ std::set<int> waiting_task_ids;+ // the main result queue+ std::vector<task_result> queue_results;+ std::mutex mutex_results;+ std::condition_variable condition_results;++ void add_waiting_task_id(int task_id) {+ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.insert(task_id);+ }++ void remove_waiting_task_id(int task_id) {+ std::unique_lock<std::mutex> lock(mutex_results);+ waiting_task_ids.erase(task_id);+ }++ // This function blocks the thread until there is a response for this task_id+ task_result recv(int task_id) {+ while (true)+ {+ std::unique_lock<std::mutex> lock(mutex_results);+ condition_results.wait(lock, [&]{+ return !queue_results.empty();+ });+ LOG_VERBOSE("condition_results unblock", {});++ for (int i = 0; i < (int) queue_results.size(); i++)+ {+ if (queue_results[i].id == task_id)+ {+ assert(queue_results[i].multitask_id == -1);+ task_result res = queue_results[i];+ queue_results.erase(queue_results.begin() + i);+ return res;+ }+ }+ }++ // should never reach here+ }++ // Register the function to update multitask+ void on_multitask_update(callback_multitask_t callback) {+ callback_update_multitask = callback;+ }++ // Send a new result to a waiting task_id+ void send(task_result result) {+ std::unique_lock<std::mutex> lock(mutex_results);+ LOG_VERBOSE("send new result", {});+ for (auto& task_id : waiting_task_ids) {+ // LOG_TEE("waiting task id %i \n", task_id);+ // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result+ if (result.multitask_id == task_id)+ {+ LOG_VERBOSE("callback_update_multitask", {});+ callback_update_multitask(task_id, result.id, result);+ continue;+ }++ if (result.id == task_id)+ {+ LOG_VERBOSE("queue_results.push_back", {});+ queue_results.push_back(result);+ condition_results.notify_one();+ return;+ }+ }+ }+};++//+// base64 utils (TODO: move to common in the future)+//++static const std::string base64_chars =+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ "abcdefghijklmnopqrstuvwxyz"+ "0123456789+/";++static inline bool is_base64(uint8_t c)+{+ return (isalnum(c) || (c == '+') || (c == '/'));+}++static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)+{+ int i = 0;+ int j = 0;+ int in_ = 0;++ int in_len = encoded_string.size();++ uint8_t char_array_4[4];+ uint8_t char_array_3[3];++ std::vector<uint8_t> ret;++ while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))+ {+ char_array_4[i++] = encoded_string[in_]; in_++;+ if (i == 4)+ {+ for (i = 0; i <4; i++)+ {+ char_array_4[i] = base64_chars.find(char_array_4[i]);+ }++ char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);+ char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);+ char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];++ for (i = 0; (i < 3); i++)+ {+ ret.push_back(char_array_3[i]);+ }+ i = 0;+ }+ }++ if (i)+ {+ for (j = i; j <4; j++)+ {+ char_array_4[j] = 0;+ }++ for (j = 0; j <4; j++)+ {+ char_array_4[j] = base64_chars.find(char_array_4[j]);+ }++ char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);+ char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);+ char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];++ for (j = 0; (j < i - 1); j++)+ {+ ret.push_back(char_array_3[j]);+ }+ }++ return ret;+}++//+// random string / id+//++static std::string random_string()+{+ static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");++ std::random_device rd;+ std::mt19937 generator(rd());++ std::string result(32, ' ');++ for (int i = 0; i < 32; ++i) {+ result[i] = str[generator() % str.size()];+ }++ return result;+}++static std::string gen_chatcmplid()+{+ std::stringstream chatcmplid;+ chatcmplid << "chatcmpl-" << random_string();+ return chatcmplid.str();+}diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 4db76c0f..26283df6 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -76,8 +76,6 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");? "-DGGML_USE_TINYBLAS" \: "-DGGML_USE_CUBLAS")-int ggml_backend_cuda_reg_devices(void);-static const struct Source {const char *zip;const char *name;@@ -95,34 +93,27 @@ static const struct Source {{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last};+GGML_CALL int ggml_backend_cuda_reg_devices(void);+static struct Cuda {bool supported;atomic_uint once;- typeof(ggml_cuda_link) *GGML_CALL ggml_cuda_link;- typeof(ggml_init_cublas) *GGML_CALL ggml_init_cublas;- typeof(ggml_cublas_loaded) *GGML_CALL ggml_cublas_loaded;- typeof(ggml_cuda_host_free) *GGML_CALL ggml_cuda_host_free;- typeof(ggml_cuda_host_malloc) *GGML_CALL ggml_cuda_host_malloc;- typeof(ggml_cuda_can_mul_mat) *GGML_CALL ggml_cuda_can_mul_mat;- typeof(ggml_cuda_set_tensor_split) *GGML_CALL set_tensor_split;- typeof(ggml_cuda_transform_tensor) *GGML_CALL ggml_cuda_transform_tensor;- typeof(ggml_cuda_free_data) *GGML_CALL ggml_cuda_free_data;- typeof(ggml_cuda_assign_buffers) *GGML_CALL assign_buffers;- typeof(ggml_cuda_assign_buffers_no_scratch) *GGML_CALL assign_buffers_no_scratch;- typeof(ggml_cuda_assign_buffers_force_inplace) *GGML_CALL assign_buffers_force_inplace;- typeof(ggml_cuda_assign_buffers_no_alloc) *GGML_CALL assign_buffers_no_alloc;- typeof(ggml_cuda_assign_scratch_offset) *GGML_CALL assign_scratch_offset;- typeof(ggml_cuda_copy_to_device) *GGML_CALL copy_to_device;- typeof(ggml_cuda_set_main_device) *GGML_CALL set_main_device;- typeof(ggml_cuda_set_scratch_size) *GGML_CALL set_scratch_size;- typeof(ggml_cuda_free_scratch) *GGML_CALL free_scratch;+ typeof(ggml_cuda_link) *GGML_CALL link;+ typeof(ggml_init_cublas) *GGML_CALL init;+ typeof(ggml_cublas_loaded) *GGML_CALL loaded;+ typeof(ggml_cuda_host_free) *GGML_CALL host_free;+ typeof(ggml_cuda_host_malloc) *GGML_CALL host_malloc;+ typeof(ggml_cuda_can_mul_mat) *GGML_CALL can_mul_mat;typeof(ggml_cuda_compute_forward) *GGML_CALL compute_forward;typeof(ggml_cuda_get_device_count) *GGML_CALL get_device_count;typeof(ggml_cuda_get_device_description) *GGML_CALL get_device_description;- typeof(ggml_backend_cuda_buffer_type) *GGML_CALL ggml_backend_cuda_buffer_type;- typeof(ggml_backend_reg_cuda_init) *GGML_CALL backend_reg_init;- typeof(ggml_backend_cuda_host_buffer_type) *GGML_CALL ggml_backend_cuda_host_buffer_type;- typeof(ggml_backend_cuda_init) *GGML_CALL ggml_backend_cuda_init;+ typeof(ggml_backend_cuda_buffer_type) *GGML_CALL buffer_type;+ typeof(ggml_backend_cuda_host_buffer_type) *GGML_CALL host_buffer_type;+ typeof(ggml_backend_cuda_init) *GGML_CALL backend_init;+ typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count2;+ typeof(ggml_backend_cuda_split_buffer_type) *GGML_CALL split_buffer_type;+ typeof(ggml_backend_cuda_reg_devices) *GGML_CALL reg_devices;+ typeof(ggml_backend_cuda_get_device_memory) *GGML_CALL get_device_memory;} ggml_cuda;static const char *Dlerror(void) {@@ -633,6 +624,14 @@ static bool extract_cuda_dso(const char *dso, const char *name) {return llamafile_extract(zip, dso);}+static void *imp(void *lib, const char *sym) {+ void *fun = cosmo_dlsym(lib, sym);+ if (!fun) {+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);+ }+ return fun;+}+static bool link_cuda_dso(const char *dso, const char *dir) {// Change directory so BLAS library is more likely to be linked.@@ -660,43 +659,43 @@ static bool link_cuda_dso(const char *dso, const char *dir) {// import functionsbool ok = true;- ok &= !!(ggml_cuda.ggml_cuda_link = cosmo_dlsym(lib, "ggml_cuda_link"));- ok &= !!(ggml_cuda.ggml_init_cublas = cosmo_dlsym(lib, "ggml_init_cublas"));- ok &= !!(ggml_cuda.ggml_cublas_loaded = cosmo_dlsym(lib, "ggml_cublas_loaded"));- ok &= !!(ggml_cuda.ggml_cuda_host_free = cosmo_dlsym(lib, "ggml_cuda_host_free"));- ok &= !!(ggml_cuda.ggml_cuda_host_malloc = cosmo_dlsym(lib, "ggml_cuda_host_malloc"));- ok &= !!(ggml_cuda.ggml_cuda_can_mul_mat = cosmo_dlsym(lib, "ggml_cuda_can_mul_mat"));- ok &= !!(ggml_cuda.set_tensor_split = cosmo_dlsym(lib, "ggml_cuda_set_tensor_split"));- ok &= !!(ggml_cuda.ggml_cuda_transform_tensor = cosmo_dlsym(lib, "ggml_cuda_transform_tensor"));- ok &= !!(ggml_cuda.ggml_cuda_free_data = cosmo_dlsym(lib, "ggml_cuda_free_data"));- ok &= !!(ggml_cuda.assign_buffers = cosmo_dlsym(lib, "ggml_cuda_assign_buffers"));- ok &= !!(ggml_cuda.assign_buffers_no_scratch = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_scratch"));- ok &= !!(ggml_cuda.assign_buffers_force_inplace = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_force_inplace"));- ok &= !!(ggml_cuda.assign_buffers_no_alloc = cosmo_dlsym(lib, "ggml_cuda_assign_buffers_no_alloc"));- ok &= !!(ggml_cuda.assign_scratch_offset = cosmo_dlsym(lib, "ggml_cuda_assign_scratch_offset"));- ok &= !!(ggml_cuda.copy_to_device = cosmo_dlsym(lib, "ggml_cuda_copy_to_device"));- ok &= !!(ggml_cuda.set_main_device = cosmo_dlsym(lib, "ggml_cuda_set_main_device"));- ok &= !!(ggml_cuda.set_scratch_size = cosmo_dlsym(lib, "ggml_cuda_set_scratch_size"));- ok &= !!(ggml_cuda.free_scratch = cosmo_dlsym(lib, "ggml_cuda_free_scratch"));- ok &= !!(ggml_cuda.compute_forward = cosmo_dlsym(lib, "ggml_cuda_compute_forward"));- ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));- ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));- ok &= !!(ggml_cuda.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_cuda_init"));- ok &= !!(ggml_cuda.ggml_backend_cuda_host_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_host_buffer_type"));- ok &= !!(ggml_cuda.ggml_backend_cuda_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_buffer_type"));- ok &= !!(ggml_cuda.ggml_backend_cuda_init = cosmo_dlsym(lib, "ggml_backend_cuda_init"));+ ok &= !!(ggml_cuda.link = imp(lib, "ggml_cuda_link"));+ ok &= !!(ggml_cuda.init = imp(lib, "ggml_init_cublas"));+ ok &= !!(ggml_cuda.loaded = imp(lib, "ggml_cublas_loaded"));+ ok &= !!(ggml_cuda.host_free = imp(lib, "ggml_cuda_host_free"));+ ok &= !!(ggml_cuda.host_malloc = imp(lib, "ggml_cuda_host_malloc"));+ ok &= !!(ggml_cuda.can_mul_mat = imp(lib, "ggml_cuda_can_mul_mat"));+ ok &= !!(ggml_cuda.compute_forward = imp(lib, "ggml_cuda_compute_forward"));+ ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));+ ok &= !!(ggml_cuda.get_device_description = imp(lib, "ggml_cuda_get_device_description"));+ ok &= !!(ggml_cuda.host_buffer_type = imp(lib, "ggml_backend_cuda_host_buffer_type"));+ ok &= !!(ggml_cuda.buffer_type = imp(lib, "ggml_backend_cuda_buffer_type"));+ ok &= !!(ggml_cuda.backend_init = imp(lib, "ggml_backend_cuda_init"));+ ok &= !!(ggml_cuda.get_device_count2 = imp(lib, "ggml_backend_cuda_get_device_count"));+ ok &= !!(ggml_cuda.split_buffer_type = imp(lib, "ggml_backend_cuda_split_buffer_type"));+ ok &= !!(ggml_cuda.reg_devices = imp(lib, "ggml_backend_cuda_reg_devices"));+ ok &= !!(ggml_cuda.get_device_memory = imp(lib, "ggml_backend_cuda_get_device_memory"));if (!ok) {tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);+ cosmo_dlclose(lib);return false;}// ask the library if actual gpu devices exist- ggml_cuda.ggml_cuda_link(ggml_backend_api());- return true;+ ggml_cuda.link(ggml_backend_api());+ tinylog(__func__, ": GPU support linked\n", NULL);+ ggml_cuda.init();+ if (ggml_cuda.loaded()) {+ tinylog(__func__, ": GPU support loaded\n", NULL);+ return true;+ } else {+ tinylog(__func__, ": GPU support not possible\n", NULL);+ cosmo_dlclose(lib);+ return false;+ }}static bool import_cuda_impl(void) {- assert(FLAG_gpu != LLAMAFILE_GPU_ERROR);// No dynamic linking support on OpenBSD yet.if (IsOpenbsd()) {@@ -887,18 +886,11 @@ TryNvidia:}static void import_cuda(void) {- if (ggml_metal_supported()) {+ if (llamafile_has_metal()) {return;}if (import_cuda_impl()) {- tinylog(__func__, ": GPU support linked\n", NULL);- ggml_cuda.ggml_init_cublas(FLAG_log_disable);- if (ggml_cuda.ggml_cublas_loaded()) {- tinylog(__func__, ": GPU support loaded\n", NULL);- ggml_cuda.supported = true;- } else {- tinylog(__func__, ": GPU support not possible\n", NULL);- }+ ggml_cuda.supported = true;} else if (FLAG_gpu == LLAMAFILE_GPU_AMD ||FLAG_gpu == LLAMAFILE_GPU_NVIDIA) {tinyprint(2, "fatal error: support for --gpu ",@@ -908,144 +900,88 @@ static void import_cuda(void) {}}-bool ggml_cuda_supported(void) {+bool llamafile_has_cuda(void) {cosmo_once(&ggml_cuda.once, import_cuda);return ggml_cuda.supported;}-GGML_CALL void ggml_init_cublas(bool log_disable) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.ggml_init_cublas(log_disable);+GGML_CALL void ggml_init_cublas(void) {+ if (!llamafile_has_cuda()) return;+ return ggml_cuda.init();}GGML_CALL bool ggml_cublas_loaded(void) {- if (!ggml_cuda_supported()) return false;- return ggml_cuda.ggml_cublas_loaded();+ if (!llamafile_has_cuda()) return false;+ return ggml_cuda.loaded();}GGML_CALL void *ggml_cuda_host_malloc(size_t n) {- if (!ggml_cuda_supported()) return NULL;- return ggml_cuda.ggml_cuda_host_malloc(n);+ if (!llamafile_has_cuda()) return NULL;+ return ggml_cuda.host_malloc(n);}GGML_CALL void ggml_cuda_host_free(void *data) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.ggml_cuda_host_free(data);+ if (!llamafile_has_cuda()) return;+ return ggml_cuda.host_free(data);}GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,const struct ggml_tensor *src1,struct ggml_tensor *dst) {- if (!ggml_cuda_supported()) return false;- return ggml_cuda.ggml_cuda_can_mul_mat(src0, src1, dst);-}--GGML_CALL void ggml_cuda_set_tensor_split(const float *tensor_split) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_tensor_split(tensor_split);-}--GGML_CALL void ggml_cuda_transform_tensor(void *data, struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.ggml_cuda_transform_tensor(data, tensor);-}--GGML_CALL void ggml_cuda_free_data(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.ggml_cuda_free_data(tensor);-}--GGML_CALL void ggml_cuda_assign_buffers(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers(tensor);-}--GGML_CALL void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_no_scratch(tensor);-}--GGML_CALL void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_force_inplace(tensor);-}--GGML_CALL void ggml_cuda_assign_buffers_no_alloc(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_buffers_no_alloc(tensor);-}--GGML_CALL void ggml_cuda_assign_scratch_offset(struct ggml_tensor *tensor, size_t offset) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.assign_scratch_offset(tensor, offset);-}--GGML_CALL void ggml_cuda_copy_to_device(struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.copy_to_device(tensor);-}--GGML_CALL void ggml_cuda_set_main_device(int main_device) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_main_device(main_device);-}--GGML_CALL void ggml_cuda_set_scratch_size(size_t scratch_size) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.set_scratch_size(scratch_size);-}--GGML_CALL void ggml_cuda_free_scratch(void) {- if (!ggml_cuda_supported()) return;- return ggml_cuda.free_scratch();+ if (!llamafile_has_cuda()) return false;+ return ggml_cuda.can_mul_mat(src0, src1, dst);}GGML_CALL bool ggml_cuda_compute_forward(struct ggml_compute_params *params,struct ggml_tensor *tensor) {- if (!ggml_cuda_supported()) return false;+ if (!llamafile_has_cuda()) return false;return ggml_cuda.compute_forward(params, tensor);}GGML_CALL int ggml_cuda_get_device_count(void) {- if (!ggml_cuda_supported()) return 0;+ if (!llamafile_has_cuda()) return 0;return ggml_cuda.get_device_count();}GGML_CALL void ggml_cuda_get_device_description(int device,char *description,size_t description_size) {- if (!ggml_cuda_supported()) return;+ if (!llamafile_has_cuda()) return;return ggml_cuda.get_device_description(device, description,description_size);}GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {- if (!ggml_cuda_supported()) return 0;- return ggml_cuda.ggml_backend_cuda_buffer_type(device);+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.buffer_type(device);}-GGML_CALL ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {- if (!ggml_cuda_supported()) return 0;- return ggml_cuda.backend_reg_init(params, user_data);+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.host_buffer_type();}-GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {- if (!ggml_cuda_supported()) return 0;- return ggml_cuda.ggml_backend_cuda_host_buffer_type();+GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.backend_init(device);}-int ggml_backend_cuda_reg_devices(void) {- int device_count = ggml_cuda_get_device_count();- //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization- for (int i = 0; i < device_count; i++) {- char name[128];- snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);- ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);- }- return device_count;+GGML_CALL int ggml_backend_cuda_get_device_count(void) {+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.get_device_count();}-GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {- if (!ggml_cuda_supported()) return 0;- return ggml_cuda.ggml_backend_cuda_init(device);+GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float *tensor_split) {+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.split_buffer_type(tensor_split);+}++GGML_CALL int ggml_backend_cuda_reg_devices(void) {+ if (!llamafile_has_cuda()) return 0;+ return ggml_cuda.reg_devices();+}++GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {+ if (!llamafile_has_cuda()) return;+ return ggml_cuda.get_device_memory(device, free, total);}diff --git a/llamafile/gpu.c b/llamafile/gpu.cindex 816747ce..03884533 100644--- a/llamafile/gpu.c+++ b/llamafile/gpu.c@@ -24,11 +24,11 @@#include "llamafile/log.h"#include "llama.cpp/ggml-metal.h"+int FLAG_gpu;bool FLAG_nogpu;bool FLAG_tinyblas;bool FLAG_nocompile;bool FLAG_recompile;-int FLAG_gpu = LLAMAFILE_GPU_ERROR;const char *llamafile_describe_gpu(void) {switch (FLAG_gpu) {@@ -50,8 +50,8 @@ const char *llamafile_describe_gpu(void) {/*** Returns true if GPU support is available.*/-bool llamafile_gpu_supported(void) {- return ggml_metal_supported() || ggml_cublas_loaded();+bool llamafile_has_gpu(void) {+ return llamafile_has_metal() || llamafile_has_cuda();}/**@@ -61,24 +61,32 @@ int llamafile_gpu_layers(int n_gpu_layers) {// if user explicitly passed `--gpu KIND` but didn't specify `-ngl// LAYERS` then assume the user wants their model fully offloaded.- if (n_gpu_layers == -1 && FLAG_gpu > 0) {+ if (n_gpu_layers < 0 && FLAG_gpu > 0) {n_gpu_layers = INT_MAX;}// Apple Metal is safe enough to enable by default.- if ((n_gpu_layers < 0 || n_gpu_layers > 1) && ggml_metal_supported()) {- n_gpu_layers = 1;+ if (n_gpu_layers <= 0 && llamafile_has_metal()) {+ n_gpu_layers = INT_MAX;}// make the -ngl flag not break example code when it's impossible// if you want it to be an error just pass --gpu apple/amd/nvidia- if (n_gpu_layers > 0 && !llamafile_gpu_supported()) {+ if (n_gpu_layers > 0 && !llamafile_has_gpu()) {tinylogf("warning: --n-gpu-layers %d was passed but no GPUs were found;"" falling back to CPU inference\n", n_gpu_layers);FLAG_gpu = LLAMAFILE_GPU_DISABLE;n_gpu_layers = 0;}+ // don't bother linking gpu modules if zero layers are offloaded+ if (n_gpu_layers <= 0) {+ if (n_gpu_layers == -1) {+ tinylogf("note: if you have an AMD or NVIDIA GPU then you need to pass -ngl 9999 to enable GPU offloading\n");+ }+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;+ }+return n_gpu_layers;}@@ -103,5 +111,5 @@ int llamafile_gpu_parse(const char *s) {if (!strcasecmp(s, "rocm")) return LLAMAFILE_GPU_AMD;if (!strcasecmp(s, "hip")) return LLAMAFILE_GPU_AMD;- return INT_MIN;+ return LLAMAFILE_GPU_ERROR;}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex b52c106e..061fd4ca 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -39,8 +39,10 @@ extern int FLAG_gpu;extern bool FLAG_tinyblas;extern bool FLAG_nocompile;extern bool FLAG_recompile;+bool llamafile_has_gpu(void);int llamafile_gpu_layers(int);-bool llamafile_gpu_supported(void);+bool llamafile_has_cuda(void);+bool llamafile_has_metal(void);int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);diff --git a/llamafile/metal.c b/llamafile/metal.cindex 0faab78f..dc8bd11d 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -59,26 +59,19 @@ static const struct Source {{"/zip/llama.cpp/ggml-metal.m", "ggml-metal.m"}, // must come last};+ggml_backend_t ggml_backend_reg_metal_init(const char *, void *);+static struct Metal {bool supported;atomic_uint once;typeof(ggml_metal_link) *ggml_metal_link;- typeof(ggml_metal_add_buffer) *add_buffer;- typeof(ggml_metal_free) *free;- typeof(ggml_metal_get_concur_list) *get_concur_list;- typeof(ggml_metal_graph_compute) *graph_compute;- typeof(ggml_metal_graph_find_concurrency) *graph_find_concurrency;- typeof(ggml_metal_host_free) *host_free;- typeof(ggml_metal_host_malloc) *host_malloc;- typeof(ggml_metal_if_optimized) *if_optimized;- typeof(ggml_metal_init) *init;- typeof(ggml_metal_set_n_cb) *set_n_cb;typeof(ggml_backend_metal_init) *backend_init;typeof(ggml_backend_metal_buffer_type) *GGML_CALL backend_buffer_type;typeof(ggml_backend_metal_buffer_from_ptr) *GGML_CALL backend_buffer_from_ptr;typeof(ggml_backend_is_metal) *backend_is_metal;typeof(ggml_backend_metal_set_n_cb) *backend_set_n_cb;- typeof(ggml_metal_log_set_callback) *ggml_metal_log_set_callback;+ typeof(ggml_backend_metal_log_set_callback) *log_set_callback;+ typeof(ggml_backend_reg_metal_init) *reg_init;} ggml_metal;static const char *Dlerror(void) {@@ -209,22 +202,13 @@ static bool LinkMetal(const char *dso) {// import functionsbool ok = true;ok &= !!(ggml_metal.ggml_metal_link = cosmo_dlsym(lib, "ggml_metal_link"));- ok &= !!(ggml_metal.add_buffer = cosmo_dlsym(lib, "ggml_metal_add_buffer"));- ok &= !!(ggml_metal.free = cosmo_dlsym(lib, "ggml_metal_free"));- ok &= !!(ggml_metal.get_concur_list = cosmo_dlsym(lib, "ggml_metal_get_concur_list"));- ok &= !!(ggml_metal.graph_compute = cosmo_dlsym(lib, "ggml_metal_graph_compute"));- ok &= !!(ggml_metal.graph_find_concurrency = cosmo_dlsym(lib, "ggml_metal_graph_find_concurrency"));- ok &= !!(ggml_metal.host_free = cosmo_dlsym(lib, "ggml_metal_host_free"));- ok &= !!(ggml_metal.host_malloc = cosmo_dlsym(lib, "ggml_metal_host_malloc"));- ok &= !!(ggml_metal.if_optimized = cosmo_dlsym(lib, "ggml_metal_if_optimized"));- ok &= !!(ggml_metal.init = cosmo_dlsym(lib, "ggml_metal_init"));- ok &= !!(ggml_metal.set_n_cb = cosmo_dlsym(lib, "ggml_metal_set_n_cb"));ok &= !!(ggml_metal.backend_init = cosmo_dlsym(lib, "ggml_backend_metal_init"));ok &= !!(ggml_metal.backend_buffer_type = cosmo_dlsym(lib, "ggml_backend_metal_buffer_type"));ok &= !!(ggml_metal.backend_buffer_from_ptr = cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));ok &= !!(ggml_metal.backend_is_metal = cosmo_dlsym(lib, "ggml_backend_is_metal"));ok &= !!(ggml_metal.backend_set_n_cb = cosmo_dlsym(lib, "ggml_backend_metal_set_n_cb"));- ok &= !!(ggml_metal.ggml_metal_log_set_callback = cosmo_dlsym(lib, "ggml_metal_log_set_callback"));+ ok &= !!(ggml_metal.log_set_callback = cosmo_dlsym(lib, "ggml_backend_metal_log_set_callback"));+ ok &= !!(ggml_metal.reg_init = cosmo_dlsym(lib, "ggml_backend_reg_metal_init"));if (!ok) {tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);return false;@@ -268,7 +252,6 @@ static bool ImportMetalImpl(void) {}static void ImportMetal(void) {- assert(FLAG_gpu != LLAMAFILE_GPU_ERROR);if (ImportMetalImpl()) {ggml_metal.supported = true;tinylog("Apple Metal GPU support successfully loaded\n", NULL);@@ -280,102 +263,42 @@ static void ImportMetal(void) {}}-bool ggml_metal_supported(void) {+bool llamafile_has_metal(void) {cosmo_once(&ggml_metal.once, ImportMetal);return ggml_metal.supported;}-void *ggml_metal_host_malloc(size_t n) {- if (!ggml_metal_supported()) return NULL;- return ggml_metal.host_malloc(n);-}--void ggml_metal_host_free(void *data) {- if (!ggml_metal_supported()) return;- return ggml_metal.host_free(data);-}--struct ggml_metal_context *ggml_metal_init(int n_cb) {- struct ggml_metal_context *res;- if (!ggml_metal_supported()) return NULL;- if ((res = ggml_metal.init(n_cb))) return res;- ggml_metal.supported = false;- return NULL;-}--bool ggml_metal_add_buffer(struct ggml_metal_context *ctx,- const char *name, void *data,- size_t size, size_t max_size) {- if (!ggml_metal_supported()) return false;- return ggml_metal.add_buffer(ctx, name, data, size, max_size);-}--void ggml_metal_free(struct ggml_metal_context *ctx) {- if (!ggml_metal_supported()) return;- return ggml_metal.free(ctx);-}--int *ggml_metal_get_concur_list(struct ggml_metal_context *ctx) {- if (!ggml_metal_supported()) return NULL;- return ggml_metal.get_concur_list(ctx);-}--bool ggml_metal_graph_compute(struct ggml_metal_context *ctx,- struct ggml_cgraph *gf) {- if (!ggml_metal_supported()) return false;- return ggml_metal.graph_compute(ctx, gf);-}--void ggml_metal_graph_find_concurrency(struct ggml_metal_context *ctx,- struct ggml_cgraph *gf,- bool check_mem) {- if (!ggml_metal_supported()) return;- return ggml_metal.graph_find_concurrency(ctx, gf, check_mem);-}--int ggml_metal_if_optimized(struct ggml_metal_context *ctx) {- if (!ggml_metal_supported()) return 0;- return ggml_metal.if_optimized(ctx);-}--void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {- if (!ggml_metal_supported()) return;- return ggml_metal.set_n_cb(ctx, n_cb);-}-ggml_backend_t ggml_backend_metal_init(void) {- if (!ggml_metal_supported()) return 0;+ if (!llamafile_has_metal()) return 0;return ggml_metal.backend_init();}GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {- if (!ggml_metal_supported()) return 0;+ if (!llamafile_has_metal()) return 0;return ggml_metal.backend_buffer_type();}GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {- if (!ggml_metal_supported()) return 0;+ if (!llamafile_has_metal()) return 0;return ggml_metal.backend_buffer_from_ptr(data, size, max_size);}bool ggml_backend_is_metal(ggml_backend_t backend) {- if (!ggml_metal_supported()) return 0;+ if (!llamafile_has_metal()) return 0;return ggml_metal.backend_is_metal(backend);}void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {- if (!ggml_metal_supported()) return;- ggml_metal.backend_set_n_cb(backend, n_cb);+ if (!llamafile_has_metal()) return;+ return ggml_metal.backend_set_n_cb(backend, n_cb);}-void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {- if (!ggml_metal_supported()) return;- //- // It's not possible to pass this callback from the application to- // the metal gpu module, because ggml-metal calls it from a worker- // thread, which hasn't set Cosmo's TLS register (x28).- //- if (!log_callback) {- ggml_metal.ggml_metal_log_set_callback(log_callback, user_data);- }+void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {+ if (!llamafile_has_metal()) return;+ return ggml_metal.log_set_callback(log_callback, user_data);+}++ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {+ if (!llamafile_has_metal()) return 0;+ return ggml_metal.reg_init(params, user_data);}diff --git a/llamafile/simple.cpp b/llamafile/simple.cppindex 222acd89..d7211dea 100644--- a/llamafile/simple.cpp+++ b/llamafile/simple.cpp@@ -55,11 +55,10 @@ static bool eval_string(struct llama_context *ctx_llama, const char *str,int main(int argc, char **argv) {gpt_params params;- if (!gpt_params_parse(argc, argv, params)) return 1;- if (params.prompt.empty()) params.prompt = "The";params.n_ctx = 0;FLAG_log_disable = true;- FLAG_gpu = LLAMAFILE_GPU_AUTO;+ if (!gpt_params_parse(argc, argv, params)) return 1;+ if (params.prompt.empty()) params.prompt = "The";llama_backend_init(params.numa);llama_model_params model_params = llama_model_default_params();model_params.n_gpu_layers = llamafile_gpu_layers(35);diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex c320be95..d1785b62 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -1,7 +1,7 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi//-// Copyright 2023 Mozilla Foundation+// Copyright 2024 Mozilla Foundation//// Licensed under the Apache License, Version 2.0 (the "License");// you may not use this file except in compliance with the License.@@ -130,11 +130,10 @@ static __global__ void tinyblasS_entry(int m, int n, int k,// each thread handles a sub-matrix of size TM * TNfor (x = blockIdx.x * BM; x < m; x += jump1) {for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, float, float>(- m, n, k, x, y, //- A, lda, //- B, ldb, //- C, ldc);+ matmul_block2d<BM, BN, BK, TM, TN, float, float>(m, n, k, x, y,+ A, lda,+ B, ldb,+ C, ldc);}}}@@ -183,7 +182,7 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,return TINYBLAS_STATUS_NOT_SUPPORTED;}- tinyblasS_wrapper<48, 24, 64, 6, 3>(stream, m, n, k, A, lda, B, ldb, C, ldc);+ tinyblasS_wrapper<32, 8, 128, 1, 2>(stream, m, n, k, A, lda, B, ldb, C, ldc);return TINYBLAS_STATUS_SUCCESS;}@@ -201,10 +200,10 @@ static __global__ void tinyblasGE_entry(int m, int n, int k, const half *A,// each block handles a sub-matrix of C, of size BM * BNfor (x = blockIdx.x * BM; x < m; x += jump1) {for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, half, DST>(m, n, k, x, y, //- A, lda, //- B, ldb, //- (DST *)C, ldc);+ matmul_block2d<BM, BN, BK, TM, TN, half, DST>(m, n, k, x, y,+ A, lda,+ B, ldb,+ (DST *)C, ldc);}}}@@ -252,7 +251,7 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,}tinyblasGE_wrapper<48, 32, 64, 3, 8>(stream, m, n, k, (const half *)A, lda,- (const half *)B, ldb, C, Ctype, ldc);+ (const half *)B, ldb, C, Ctype, ldc);return TINYBLAS_STATUS_SUCCESS;}@@ -275,10 +274,10 @@ static __global__ void tinyblasGBE_entry(int m, int n, int k,for (z = blockIdx.z; z < batchCount; z += jump3) {for (x = blockIdx.x * BM; x < m; x += jump1) {for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, half, DST>(m, n, k, x, y, //- Aarray[z], lda, //- Barray[z], ldb, //- (DST *)(Carray[z]), ldc);+ matmul_block2d<BM, BN, BK, TM, TN, half, DST>(m, n, k, x, y,+ Aarray[z], lda,+ Barray[z], ldb,+ (DST *)(Carray[z]), ldc);}}}@@ -332,8 +331,8 @@ tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t stream,}tinyblasGBE_wrapper<48, 32, 64, 3, 8>(stream, m, n, k, (const half **)Aarray, lda,- (const half **)Barray, ldb, Carray, Ctype,- ldc, batchCount);+ (const half **)Barray, ldb, Carray, Ctype,+ ldc, batchCount);return TINYBLAS_STATUS_SUCCESS;}@@ -426,8 +425,8 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t stream,}tinyblasGSBE_wrapper<32, 4, 64, 1, 2>(stream, m, n, k, (const half *)A, lda, strideA,- (const half *)B, ldb, strideB, C, Ctype,- ldc, strideC, batchCount);+ (const half *)B, ldb, strideB, C, Ctype,+ ldc, strideC, batchCount);return TINYBLAS_STATUS_SUCCESS;}@@ -436,23 +435,9 @@ const char *tinyblasGetStatusString(tinyblasStatus_t err) {switch (err) {case TINYBLAS_STATUS_SUCCESS:return "TINYBLAS_STATUS_SUCCESS";- case TINYBLAS_STATUS_NOT_INITIALIZED:- return "TINYBLAS_STATUS_NOT_INITIALIZED";- case TINYBLAS_STATUS_ALLOC_FAILED:- return "TINYBLAS_STATUS_ALLOC_FAILED";- case TINYBLAS_STATUS_INVALID_VALUE:- return "TINYBLAS_STATUS_INVALID_VALUE";- case TINYBLAS_STATUS_ARCH_MISMATCH:- return "TINYBLAS_STATUS_ARCH_MISMATCH";- case TINYBLAS_STATUS_MAPPING_ERROR:- return "TINYBLAS_STATUS_MAPPING_ERROR";- case TINYBLAS_STATUS_EXECUTION_FAILED:- return "TINYBLAS_STATUS_EXECUTION_FAILED";- case TINYBLAS_STATUS_INTERNAL_ERROR:- return "TINYBLAS_STATUS_INTERNAL_ERROR";case TINYBLAS_STATUS_NOT_SUPPORTED:return "TINYBLAS_STATUS_NOT_SUPPORTED";default:- return "unknown error";+ return "unknown tinyblas error";}}diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.hindex 78d8645a..1f940547 100644--- a/llamafile/tinyblas.h+++ b/llamafile/tinyblas.h@@ -9,30 +9,23 @@#endifenum tinyblasOperation_t {- TINYBLAS_OP_N = 111,- TINYBLAS_OP_T = 112,- TINYBLAS_OP_C = 113,+ TINYBLAS_OP_N,+ TINYBLAS_OP_T,+ TINYBLAS_OP_C,};enum tinyblasStatus_t {TINYBLAS_STATUS_SUCCESS,- TINYBLAS_STATUS_NOT_INITIALIZED,- TINYBLAS_STATUS_ALLOC_FAILED,- TINYBLAS_STATUS_INVALID_VALUE,- TINYBLAS_STATUS_ARCH_MISMATCH,- TINYBLAS_STATUS_MAPPING_ERROR,- TINYBLAS_STATUS_EXECUTION_FAILED,- TINYBLAS_STATUS_INTERNAL_ERROR,TINYBLAS_STATUS_NOT_SUPPORTED,};enum tinyblasComputeType_t {- TINYBLAS_COMPUTE_16F = 150,- TINYBLAS_COMPUTE_32F = 151,+ TINYBLAS_COMPUTE_16F,+ TINYBLAS_COMPUTE_32F,};enum tinyblasGemmAlgo_t {- TINYBLAS_GEMM_DEFAULT_TENSOR_OP = 160,+ TINYBLAS_GEMM_DEFAULT_TENSOR_OP,};#define tinyblasHandle_t cudaStream_t