LLM inference in C/C++
C++ ggml
llama : place d2t by its set_rows use so it can stay on the GPU (#30912)
Aman Guptaeb182d85c0e3
diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 1b954f176c..4534be22f3 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -1110,7 +1110,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {{LLM_TENSOR_MASKED_EMBD_ORDERING, {LLM_TENSOR_LAYER_INPUT, GGML_OP_NONE}},// eagle3{LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},- {LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},+ {LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_SET_ROWS}},// dspark{LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},{LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex bc93c6f9e6..9dc366fda5 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -951,6 +951,13 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * wggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);op_tensor = ggml_get_rows(ctx, w, b);} break;+ case GGML_OP_SET_ROWS:+ {+ // w holds the row indices (e.g. d2t)+ ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, w->ne[0]);+ ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, w->ne[0]);+ op_tensor = ggml_set_rows(ctx, a, b, w);+ } break;case GGML_OP_MUL_MAT:{ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);