Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Rewrite synchronization primitives
Justine Tunney6162004caef5
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 11bdd360..2c50b729 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -7107,6 +7107,36 @@ struct ggml_tensor * ggml_cross_entropy_loss_back(////////////////////////////////////////////////////////////////////////////////+struct ggml_barrier {+ int nth;+ atomic_uint phase;+ atomic_uint count;+};++int ggml_delay(int backoff) {+ if (backoff < 13) {+ volatile int i;+ for (i = 0; i != 1 << backoff; i++) {+ }+ backoff++;+ } else {+ sched_yield();+ }+ return backoff;+}++void ggml_syncthreads(struct ggml_barrier *b) {+ unsigned phase = atomic_load_explicit(&b->phase, memory_order_relaxed);+ if (atomic_fetch_add_explicit(&b->count, 1, memory_order_acq_rel) + 1 == b->nth) {+ atomic_store_explicit(&b->count, 0, memory_order_relaxed);+ atomic_store_explicit(&b->phase, phase + 1, memory_order_release);+ } else {+ int backoff = 0;+ while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase)+ backoff = ggml_delay(backoff);+ }+}+void ggml_set_param(struct ggml_context * ctx,struct ggml_tensor * tensor) {@@ -7120,7 +7150,7 @@ void ggml_set_param(// ggml_compute_forward_dupstatic void ggml_compute_forward_dup_same_cont(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -7129,10 +7159,6 @@ static void ggml_compute_forward_dup_same_cont(GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));GGML_ASSERT(src0->type == dst->type);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const size_t nb00 = src0->nb[0];const size_t nb0 = dst->nb[0];@@ -7154,17 +7180,13 @@ static void ggml_compute_forward_dup_same_cont(}static void ggml_compute_forward_dup_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALSconst int ith = params->ith; // thread index@@ -7427,17 +7449,13 @@ static void ggml_compute_forward_dup_f16(}static void ggml_compute_forward_dup_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALSconst int ith = params->ith; // thread index@@ -7679,7 +7697,7 @@ static void ggml_compute_forward_dup_f32(// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.static void ggml_compute_forward_dup_bytes(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -7687,10 +7705,6 @@ static void ggml_compute_forward_dup_bytes(GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));GGML_ASSERT(src0->type == dst->type);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {ggml_compute_forward_dup_same_cont(params, dst);return;@@ -7830,7 +7844,7 @@ static void ggml_compute_forward_dup_bytes(}static void ggml_compute_forward_dup(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -7859,7 +7873,7 @@ static void ggml_compute_forward_dup(// ggml_compute_forward_addstatic void ggml_compute_forward_add_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -7867,10 +7881,6 @@ static void ggml_compute_forward_add_f32(GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -7949,7 +7959,7 @@ static void ggml_compute_forward_add_f32(}static void ggml_compute_forward_add_f16_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -7957,10 +7967,6 @@ static void ggml_compute_forward_add_f16_f32(GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -8028,7 +8034,7 @@ static void ggml_compute_forward_add_f16_f32(}static void ggml_compute_forward_add_f16_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8036,10 +8042,6 @@ static void ggml_compute_forward_add_f16_f16(GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -8084,7 +8086,7 @@ static void ggml_compute_forward_add_f16_f16(}static void ggml_compute_forward_add_q_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8092,10 +8094,6 @@ static void ggml_compute_forward_add_q_f32(GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int nr = ggml_nrows(src0);GGML_TENSOR_BINARY_OP_LOCALS@@ -8164,7 +8162,7 @@ static void ggml_compute_forward_add_q_f32(}static void ggml_compute_forward_add(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8224,7 +8222,7 @@ static void ggml_compute_forward_add(// ggml_compute_forward_add1static void ggml_compute_forward_add1_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8233,10 +8231,6 @@ static void ggml_compute_forward_add1_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -8278,7 +8272,7 @@ static void ggml_compute_forward_add1_f32(}static void ggml_compute_forward_add1_f16_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8287,10 +8281,6 @@ static void ggml_compute_forward_add1_f16_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// scalar to addconst float v = *(float *) src1->data;@@ -8330,7 +8320,7 @@ static void ggml_compute_forward_add1_f16_f32(}static void ggml_compute_forward_add1_f16_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8339,10 +8329,6 @@ static void ggml_compute_forward_add1_f16_f16(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// scalar to addconst float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);@@ -8382,7 +8368,7 @@ static void ggml_compute_forward_add1_f16_f16(}static void ggml_compute_forward_add1_q_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8391,10 +8377,6 @@ static void ggml_compute_forward_add1_q_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_is_scalar(src1));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// scalar to addconst float v = *(float *) src1->data;@@ -8451,7 +8433,7 @@ static void ggml_compute_forward_add1_q_f32(}static void ggml_compute_forward_add1(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8507,7 +8489,7 @@ static void ggml_compute_forward_add1(// ggml_compute_forward_accstatic void ggml_compute_forward_acc_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8524,20 +8506,16 @@ static void ggml_compute_forward_acc_f32(size_t offset = ((int32_t *) dst->op_params)[3];bool inplace = (bool) ((int32_t *) dst->op_params)[4];- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {- if (params->ith != 0) {- return;+ if (!inplace) {+ if (!params->ith) {+ // memcpy needs to be synchronized across threads to avoid race conditions.+ // => do it in INIT phase+ memcpy(+ ((char *) dst->data),+ ((char *) src0->data),+ ggml_nbytes(dst));}- // memcpy needs to be synchronized across threads to avoid race conditions.- // => do it in INIT phase- memcpy(- ((char *) dst->data),- ((char *) src0->data),- ggml_nbytes(dst));- }-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ ggml_syncthreads(params->barrier);}const int ith = params->ith;@@ -8591,7 +8569,7 @@ static void ggml_compute_forward_acc_f32(}static void ggml_compute_forward_acc(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8632,19 +8610,17 @@ static void ggml_compute_forward_acc(// ggml_compute_forward_substatic void ggml_compute_forward_sub_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- assert(params->ith == 0);assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int nr = ggml_nrows(src0);GGML_TENSOR_BINARY_OP_LOCALS@@ -8694,7 +8670,7 @@ static void ggml_compute_forward_sub_f32(}static void ggml_compute_forward_sub(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8714,7 +8690,7 @@ static void ggml_compute_forward_sub(// ggml_compute_forward_mulstatic void ggml_compute_forward_mul_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8722,9 +8698,6 @@ static void ggml_compute_forward_mul_f32(GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }const int ith = params->ith;const int nth = params->nth;@@ -8799,7 +8772,7 @@ static void ggml_compute_forward_mul_f32(}static void ggml_compute_forward_mul(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8822,7 +8795,7 @@ static void ggml_compute_forward_mul(// ggml_compute_forward_divstatic void ggml_compute_forward_div_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8830,10 +8803,6 @@ static void ggml_compute_forward_div_f32(GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -8897,7 +8866,7 @@ static void ggml_compute_forward_div_f32(}static void ggml_compute_forward_div(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8917,7 +8886,7 @@ static void ggml_compute_forward_div(// ggml_compute_forward_sqrstatic void ggml_compute_forward_sqr_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8925,10 +8894,6 @@ static void ggml_compute_forward_sqr_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -8943,9 +8908,12 @@ static void ggml_compute_forward_sqr_f32(}static void ggml_compute_forward_sqr(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -8963,7 +8931,7 @@ static void ggml_compute_forward_sqr(// ggml_compute_forward_sqrtstatic void ggml_compute_forward_sqrt_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -8971,10 +8939,6 @@ static void ggml_compute_forward_sqrt_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -8989,9 +8953,12 @@ static void ggml_compute_forward_sqrt_f32(}static void ggml_compute_forward_sqrt(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9009,7 +8976,7 @@ static void ggml_compute_forward_sqrt(// ggml_compute_forward_logstatic void ggml_compute_forward_log_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9017,10 +8984,6 @@ static void ggml_compute_forward_log_f32(GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9035,9 +8998,12 @@ static void ggml_compute_forward_log_f32(}static void ggml_compute_forward_log(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9055,7 +9021,7 @@ static void ggml_compute_forward_log(// ggml_compute_forward_sumstatic void ggml_compute_forward_sum_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9063,10 +9029,6 @@ static void ggml_compute_forward_sum_f32(assert(params->ith == 0);assert(ggml_is_scalar(dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-assert(ggml_is_scalar(dst));assert(src0->nb[0] == sizeof(float));@@ -9090,7 +9052,7 @@ static void ggml_compute_forward_sum_f32(}static void ggml_compute_forward_sum_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9098,10 +9060,6 @@ static void ggml_compute_forward_sum_f16(assert(params->ith == 0);assert(ggml_is_scalar(dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-assert(src0->nb[0] == sizeof(ggml_fp16_t));GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)@@ -9124,9 +9082,12 @@ static void ggml_compute_forward_sum_f16(}static void ggml_compute_forward_sum(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9148,17 +9109,13 @@ static void ggml_compute_forward_sum(// ggml_compute_forward_sum_rowsstatic void ggml_compute_forward_sum_rows_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));GGML_ASSERT(dst->nb[0] == sizeof(float));@@ -9183,9 +9140,12 @@ static void ggml_compute_forward_sum_rows_f32(}static void ggml_compute_forward_sum_rows(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9203,17 +9163,13 @@ static void ggml_compute_forward_sum_rows(// ggml_compute_forward_meanstatic void ggml_compute_forward_mean_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-assert(src0->nb[0] == sizeof(float));GGML_TENSOR_UNARY_OP_LOCALS@@ -9242,9 +9198,12 @@ static void ggml_compute_forward_mean_f32(}static void ggml_compute_forward_mean(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9262,17 +9221,13 @@ static void ggml_compute_forward_mean(// ggml_compute_forward_argmaxstatic void ggml_compute_forward_argmax_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-assert(src0->nb[0] == sizeof(float));assert(dst->nb[0] == sizeof(float));@@ -9292,9 +9247,12 @@ static void ggml_compute_forward_argmax_f32(}static void ggml_compute_forward_argmax(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9312,7 +9270,7 @@ static void ggml_compute_forward_argmax(// ggml_compute_forward_repeatstatic void ggml_compute_forward_repeat_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9320,10 +9278,6 @@ static void ggml_compute_forward_repeat_f32(GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALS// guaranteed to be an integer due to the check in ggml_can_repeat@@ -9357,7 +9311,7 @@ static void ggml_compute_forward_repeat_f32(}static void ggml_compute_forward_repeat_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9365,10 +9319,6 @@ static void ggml_compute_forward_repeat_f16(GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALS// guaranteed to be an integer due to the check in ggml_can_repeat@@ -9405,9 +9355,12 @@ static void ggml_compute_forward_repeat_f16(}static void ggml_compute_forward_repeat(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9431,7 +9384,7 @@ static void ggml_compute_forward_repeat(// ggml_compute_forward_repeat_backstatic void ggml_compute_forward_repeat_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9439,10 +9392,6 @@ static void ggml_compute_forward_repeat_back_f32(GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_can_repeat(dst, src0));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALS// guaranteed to be an integer due to the check in ggml_can_repeat@@ -9490,9 +9439,12 @@ static void ggml_compute_forward_repeat_back_f32(}static void ggml_compute_forward_repeat_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9510,16 +9462,12 @@ static void ggml_compute_forward_repeat_back(// ggml_compute_forward_concatstatic void ggml_compute_forward_concat_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -9559,7 +9507,7 @@ static void ggml_compute_forward_concat_f32(}static void ggml_compute_forward_concat(- const struct ggml_compute_params* params,+ struct ggml_compute_params* params,struct ggml_tensor* dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9580,7 +9528,7 @@ static void ggml_compute_forward_concat(// ggml_compute_forward_absstatic void ggml_compute_forward_abs_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9588,10 +9536,6 @@ static void ggml_compute_forward_abs_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9606,9 +9550,12 @@ static void ggml_compute_forward_abs_f32(}static void ggml_compute_forward_abs(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9626,7 +9573,7 @@ static void ggml_compute_forward_abs(// ggml_compute_forward_sgnstatic void ggml_compute_forward_sgn_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9634,10 +9581,6 @@ static void ggml_compute_forward_sgn_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9652,9 +9595,12 @@ static void ggml_compute_forward_sgn_f32(}static void ggml_compute_forward_sgn(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9672,7 +9618,7 @@ static void ggml_compute_forward_sgn(// ggml_compute_forward_negstatic void ggml_compute_forward_neg_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9680,10 +9626,6 @@ static void ggml_compute_forward_neg_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9698,9 +9640,12 @@ static void ggml_compute_forward_neg_f32(}static void ggml_compute_forward_neg(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9718,7 +9663,7 @@ static void ggml_compute_forward_neg(// ggml_compute_forward_stepstatic void ggml_compute_forward_step_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9726,10 +9671,6 @@ static void ggml_compute_forward_step_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9744,9 +9685,12 @@ static void ggml_compute_forward_step_f32(}static void ggml_compute_forward_step(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9764,7 +9708,7 @@ static void ggml_compute_forward_step(// ggml_compute_forward_tanhstatic void ggml_compute_forward_tanh_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9772,10 +9716,6 @@ static void ggml_compute_forward_tanh_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9790,9 +9730,12 @@ static void ggml_compute_forward_tanh_f32(}static void ggml_compute_forward_tanh(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9810,7 +9753,7 @@ static void ggml_compute_forward_tanh(// ggml_compute_forward_elustatic void ggml_compute_forward_elu_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9818,10 +9761,6 @@ static void ggml_compute_forward_elu_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9836,9 +9775,12 @@ static void ggml_compute_forward_elu_f32(}static void ggml_compute_forward_elu(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9856,7 +9798,7 @@ static void ggml_compute_forward_elu(// ggml_compute_forward_relustatic void ggml_compute_forward_relu_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9864,10 +9806,6 @@ static void ggml_compute_forward_relu_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -9882,9 +9820,12 @@ static void ggml_compute_forward_relu_f32(}static void ggml_compute_forward_relu(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -9902,7 +9843,7 @@ static void ggml_compute_forward_relu(// ggml_compute_forward_gelustatic void ggml_compute_forward_gelu_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9911,10 +9852,6 @@ static void ggml_compute_forward_gelu_f32(GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -9945,7 +9882,7 @@ static void ggml_compute_forward_gelu_f32(}static void ggml_compute_forward_gelu(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9965,7 +9902,7 @@ static void ggml_compute_forward_gelu(// ggml_compute_forward_gelu_quickstatic void ggml_compute_forward_gelu_quick_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -9974,10 +9911,6 @@ static void ggml_compute_forward_gelu_quick_f32(GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -10008,7 +9941,7 @@ static void ggml_compute_forward_gelu_quick_f32(}static void ggml_compute_forward_gelu_quick(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10028,7 +9961,7 @@ static void ggml_compute_forward_gelu_quick(// ggml_compute_forward_silustatic void ggml_compute_forward_silu_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10037,10 +9970,6 @@ static void ggml_compute_forward_silu_f32(GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -10071,7 +10000,7 @@ static void ggml_compute_forward_silu_f32(}static void ggml_compute_forward_silu(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10090,7 +10019,7 @@ static void ggml_compute_forward_silu(// ggml_compute_forward_leaky_relustatic void ggml_compute_forward_leaky_relu_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10098,10 +10027,6 @@ static void ggml_compute_forward_leaky_relu_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -10119,9 +10044,12 @@ static void ggml_compute_forward_leaky_relu_f32(}static void ggml_compute_forward_leaky_relu(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -10139,7 +10067,7 @@ static void ggml_compute_forward_leaky_relu(// ggml_compute_forward_silu_backstatic void ggml_compute_forward_silu_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10151,10 +10079,6 @@ static void ggml_compute_forward_silu_back_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_are_same_shape(src0, grad));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int ith = params->ith;const int nth = params->nth;@@ -10186,7 +10110,7 @@ static void ggml_compute_forward_silu_back_f32(}static void ggml_compute_forward_silu_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10205,7 +10129,7 @@ static void ggml_compute_forward_silu_back(static void ggml_compute_forward_hardswish_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10213,10 +10137,6 @@ static void ggml_compute_forward_hardswish_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -10230,9 +10150,12 @@ static void ggml_compute_forward_hardswish_f32(}}static void ggml_compute_forward_hardswish(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -10248,7 +10171,7 @@ static void ggml_compute_forward_hardswish(}static void ggml_compute_forward_hardsigmoid_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10256,10 +10179,6 @@ static void ggml_compute_forward_hardsigmoid_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -10274,9 +10193,12 @@ static void ggml_compute_forward_hardsigmoid_f32(}static void ggml_compute_forward_hardsigmoid(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -10295,17 +10217,13 @@ static void ggml_compute_forward_hardsigmoid(// ggml_compute_forward_normstatic void ggml_compute_forward_norm_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -10350,7 +10268,7 @@ static void ggml_compute_forward_norm_f32(}static void ggml_compute_forward_norm(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10370,17 +10288,13 @@ static void ggml_compute_forward_norm(// ggml_compute_forward_group_rms_normstatic void ggml_compute_forward_rms_norm_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -10422,7 +10336,7 @@ static void ggml_compute_forward_rms_norm_f32(}static void ggml_compute_forward_rms_norm(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10440,7 +10354,7 @@ static void ggml_compute_forward_rms_norm(}static void ggml_compute_forward_rms_norm_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10448,10 +10362,6 @@ static void ggml_compute_forward_rms_norm_back_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -10599,7 +10509,7 @@ static void ggml_compute_forward_rms_norm_back_f32(}static void ggml_compute_forward_rms_norm_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10619,17 +10529,13 @@ static void ggml_compute_forward_rms_norm_back(// ggml_compute_forward_group_normstatic void ggml_compute_forward_group_norm_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -10697,7 +10603,7 @@ static void ggml_compute_forward_group_norm_f32(}static void ggml_compute_forward_group_norm(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10750,7 +10656,7 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) {#endifstatic void ggml_compute_forward_mul_mat(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -10797,7 +10703,7 @@ static void ggml_compute_forward_mul_mat(#if defined(GGML_USE_CLBLAST)if (ggml_cl_can_mul_mat(src0, src1, dst)) {- if (params->ith == 0 && params->type == GGML_TASK_TYPE_COMPUTE) {+ if (!ith) {ggml_cl_mul_mat(src0, src1, dst, params->wdata, params->wsize);}return;@@ -10810,31 +10716,25 @@ static void ggml_compute_forward_mul_mat(const size_t desired_wsize = ne13*ne12*ne_plane*sizeof(float);UNUSED(desired_wsize);- if (params->type == GGML_TASK_TYPE_INIT) {- if (type != GGML_TYPE_F32) {- assert(params->wsize >= desired_wsize);- // parallelize by src0 rows- for (int64_t i13 = 0; i13 < ne13; i13++) {- for (int64_t i12 = 0; i12 < ne12; i12++) {- // broadcast src0 into src1 across 2nd,3rd dimension- const int64_t i03 = i13/r3;- const int64_t i02 = i12/r2;-- const void * x = (char *) src0->data + i02*nb02 + i03*nb03;- float * const wdata = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;- ggml_to_float_t const to_float = type_traits[type].to_float;-- for (int64_t i01 = ith; i01 < ne01; i01 += nth) {- to_float((const char *) x + i01*nb01, wdata + i01*ne00, ne00);- }+ if (type != GGML_TYPE_F32) {+ assert(params->wsize >= desired_wsize);+ // parallelize by src0 rows+ for (int64_t i13 = 0; i13 < ne13; i13++) {+ for (int64_t i12 = 0; i12 < ne12; i12++) {+ // broadcast src0 into src1 across 2nd,3rd dimension+ const int64_t i03 = i13/r3;+ const int64_t i02 = i12/r2;++ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;+ float * const wdata = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;+ ggml_to_float_t const to_float = type_traits[type].to_float;++ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {+ to_float((const char *) x + i01*nb01, wdata + i01*ne00, ne00);}}}- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ ggml_syncthreads(params->barrier);}// perform sgemm, parallelization controlled by blas lib@@ -10883,7 +10783,6 @@ static void ggml_compute_forward_mul_mat((char *)dst->data + i12*nb2 + i13*nb3,nb1/ggml_type_size(dst->type),ith, nth,- params->type,src0->type,src1->type,dst->type))@@ -10893,32 +10792,25 @@ static void ggml_compute_forward_mul_mat(UseGgmlGemm1:;#endif- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }- if (src1->type != vec_dot_type) {- char * wdata = params->wdata;- const size_t row_size = ggml_row_size(vec_dot_type, ne10);+ if (src1->type != vec_dot_type) {+ char * wdata = params->wdata;+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);- assert(params->wsize >= ne11*ne12*ne13*row_size);- GGML_ASSERT(src1->type == GGML_TYPE_F32);+ assert(params->wsize >= ne11*ne12*ne13*row_size);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);- for (int64_t i13 = 0; i13 < ne13; ++i13) {- for (int64_t i12 = 0; i12 < ne12; ++i12) {- for (int64_t i11 = 0; i11 < ne11; ++i11) {+ unsigned chore = 0;+ for (int64_t i13 = 0; i13 < ne13; ++i13) {+ for (int64_t i12 = 0; i12 < ne12; ++i12) {+ for (int64_t i11 = 0; i11 < ne11; ++i11) {+ if (chore++ % nth == ith)from_float_to_vec_dot((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), (void *) wdata, ne10);- wdata += row_size;- }+ wdata += row_size;}}}- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ ggml_syncthreads(params->barrier);}const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;@@ -10938,7 +10830,6 @@ UseGgmlGemm1:;(char *)dst->data + i12*nb2 + i13*nb3,nb1/ggml_type_size(dst->type),ith, nth,- params->type,src0->type,vec_dot_type,dst->type))@@ -11053,7 +10944,7 @@ UseGgmlGemm2:;// ggml_compute_forward_mul_mat_idstatic void ggml_compute_forward_mul_mat_id(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11102,10 +10993,7 @@ static void ggml_compute_forward_mul_mat_id(int64_t * matrix_row_counts = (int64_t *) (wdata_src1_end); // [n_as]struct mmid_row_mapping * matrix_rows = (struct mmid_row_mapping *)(matrix_row_counts + n_as); // [n_as][ne11]- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }+ if (!ith) {char * wdata = params->wdata;if (src1->type != vec_dot_type) {const size_t row_size = ggml_row_size(vec_dot_type, ne10);@@ -11139,13 +11027,8 @@ static void ggml_compute_forward_mul_mat_id(matrix_row_counts[i02] += 1;}}-- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }+ }+ ggml_syncthreads(params->barrier);// compute each matrix multiplication in sequencefor (int cur_a = 0; cur_a < n_as; ++cur_a) {@@ -11238,7 +11121,7 @@ static void ggml_compute_forward_mul_mat_id(// ggml_compute_forward_out_prodstatic void ggml_compute_forward_out_prod_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11280,26 +11163,19 @@ static void ggml_compute_forward_out_prod_f32((ggml_is_contiguous(src1) || ggml_is_transposed(src1));#endif- if (params->type == GGML_TASK_TYPE_INIT) {+ if (!ith) {#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) // gemm beta will zero dstif (use_blas) {return;}#endif- if (ith != 0) {- return;- }ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;}+ ggml_syncthreads(params->barrier);#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)if (use_blas) {- if (params->ith != 0) { // All threads other than the first do no work.+ if (params->ith) { // All threads other than the first do no work.return;}// Arguments to ggml_compute_forward_out_prod (expressed as major,minor)@@ -11431,7 +11307,7 @@ static void ggml_compute_forward_out_prod_f32(}static void ggml_compute_forward_out_prod_q_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11472,17 +11348,11 @@ static void ggml_compute_forward_out_prod_q_f32(// TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }+ if (!ith) {ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);return;}-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }+ ggml_syncthreads(params->barrier);// parallelize by last three dimensions@@ -11545,7 +11415,7 @@ static void ggml_compute_forward_out_prod_q_f32(}static void ggml_compute_forward_out_prod(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11592,7 +11462,7 @@ static void ggml_compute_forward_out_prod(// ggml_compute_forward_scalestatic void ggml_compute_forward_scale_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11601,10 +11471,6 @@ static void ggml_compute_forward_scale_f32(GGML_ASSERT(ggml_is_contiguous(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// scale factorfloat v;memcpy(&v, dst->op_params, sizeof(float));@@ -11636,9 +11502,12 @@ static void ggml_compute_forward_scale_f32(}static void ggml_compute_forward_scale(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -11656,7 +11525,7 @@ static void ggml_compute_forward_scale(// ggml_compute_forward_setstatic void ggml_compute_forward_set_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -11673,20 +11542,16 @@ static void ggml_compute_forward_set_f32(size_t offset = ((int32_t *) dst->op_params)[3];bool inplace = (bool) ((int32_t *) dst->op_params)[4];- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {- if (params->ith != 0) {- return;+ if (!inplace) {+ if (!params->ith) {+ // memcpy needs to be synchronized across threads to avoid race conditions.+ // => do it in INIT phase+ memcpy(+ ((char *) dst->data),+ ((char *) src0->data),+ ggml_nbytes(dst));}- // memcpy needs to be synchronized across threads to avoid race conditions.- // => do it in INIT phase- memcpy(- ((char *) dst->data),- ((char *) src0->data),- ggml_nbytes(dst));- }-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ ggml_syncthreads(params->barrier);}const int ith = params->ith;@@ -11731,9 +11596,12 @@ static void ggml_compute_forward_set_f32(}static void ggml_compute_forward_set(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -11772,7 +11640,7 @@ static void ggml_compute_forward_set(// ggml_compute_forward_cpystatic void ggml_compute_forward_cpy(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {ggml_compute_forward_dup(params, dst);}@@ -11780,7 +11648,7 @@ static void ggml_compute_forward_cpy(// ggml_compute_forward_contstatic void ggml_compute_forward_cont(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {ggml_compute_forward_dup(params, dst);}@@ -11788,7 +11656,7 @@ static void ggml_compute_forward_cont(// ggml_compute_forward_reshapestatic void ggml_compute_forward_reshape(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {// NOPUNUSED(params);@@ -11798,7 +11666,7 @@ static void ggml_compute_forward_reshape(// ggml_compute_forward_viewstatic void ggml_compute_forward_view(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const struct ggml_tensor * dst) {// NOPUNUSED(params);@@ -11808,7 +11676,7 @@ static void ggml_compute_forward_view(// ggml_compute_forward_permutestatic void ggml_compute_forward_permute(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const struct ggml_tensor * dst) {// NOPUNUSED(params);@@ -11818,7 +11686,7 @@ static void ggml_compute_forward_permute(// ggml_compute_forward_transposestatic void ggml_compute_forward_transpose(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const struct ggml_tensor * dst) {// NOPUNUSED(params);@@ -11828,16 +11696,12 @@ static void ggml_compute_forward_transpose(// ggml_compute_forward_get_rowsstatic void ggml_compute_forward_get_rows_q(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;@@ -11874,16 +11738,12 @@ static void ggml_compute_forward_get_rows_q(}static void ggml_compute_forward_get_rows_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;@@ -11917,16 +11777,12 @@ static void ggml_compute_forward_get_rows_f16(}static void ggml_compute_forward_get_rows_bf16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;@@ -11960,16 +11816,12 @@ static void ggml_compute_forward_get_rows_bf16(}static void ggml_compute_forward_get_rows_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_BINARY_OP_LOCALSconst int64_t nc = ne00;@@ -12003,9 +11855,14 @@ static void ggml_compute_forward_get_rows_f32(}static void ggml_compute_forward_get_rows(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ // FIXME: the cost of launching additional threads decreases performance with GPU offloading+ params->nth = MIN(params->nth, ggml_nelements(dst->src[1]));+ if (params->ith >= params->nth)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -12073,7 +11930,7 @@ static void ggml_compute_forward_get_rows(// ggml_compute_forward_get_rows_backstatic void ggml_compute_forward_get_rows_back_f32_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12084,16 +11941,10 @@ static void ggml_compute_forward_get_rows_back_f32_f16(// ggml_compute_forward_dup_same_cont(params, opt0, dst);- if (params->type == GGML_TASK_TYPE_INIT) {- if (params->ith != 0) {- return;- }+ if (!params->ith) {memset(dst->data, 0, ggml_nbytes(dst));}-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }+ ggml_syncthreads(params->barrier);const int nc = src0->ne[0];const int nr = ggml_nelements(src1);@@ -12112,27 +11963,20 @@ static void ggml_compute_forward_get_rows_back_f32_f16(}static void ggml_compute_forward_get_rows_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- GGML_ASSERT(params->ith == 0);GGML_ASSERT(ggml_is_contiguous(dst));// ggml_compute_forward_dup_same_cont(params, opt0, dst);- if (params->type == GGML_TASK_TYPE_INIT) {- if (params->ith != 0) {- return;- }- memset(dst->data, 0, ggml_nbytes(dst));- }-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }+ memset(dst->data, 0, ggml_nbytes(dst));const int nc = src0->ne[0];const int nr = ggml_nelements(src1);@@ -12151,9 +11995,12 @@ static void ggml_compute_forward_get_rows_back_f32(}static void ggml_compute_forward_get_rows_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -12193,17 +12040,13 @@ static void ggml_compute_forward_get_rows_back(// ggml_compute_forward_diagstatic void ggml_compute_forward_diag_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// TODO: handle transposed/permuted matricesGGML_TENSOR_UNARY_OP_LOCALS@@ -12235,9 +12078,12 @@ static void ggml_compute_forward_diag_f32(}static void ggml_compute_forward_diag(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -12255,7 +12101,7 @@ static void ggml_compute_forward_diag(// ggml_compute_forward_diag_mask_infstatic void ggml_compute_forward_diag_mask_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const float value) {@@ -12269,22 +12115,18 @@ static void ggml_compute_forward_diag_mask_f32(GGML_ASSERT(n_past >= 0);- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {- if (ith != 0) {- return;+ if (!inplace) {+ if (!ith) {+ // memcpy needs to be synchronized across threads to avoid race conditions.+ // => do it in INIT phase+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));+ memcpy(+ ((char *) dst->data),+ ((char *) src0->data),+ ggml_nbytes(dst));}- // memcpy needs to be synchronized across threads to avoid race conditions.- // => do it in INIT phase- GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));- GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));- memcpy(- ((char *) dst->data),- ((char *) src0->data),- ggml_nbytes(dst));- }-- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ ggml_syncthreads(params->barrier);}// TODO: handle transposed/permuted matrices@@ -12309,7 +12151,7 @@ static void ggml_compute_forward_diag_mask_f32(}static void ggml_compute_forward_diag_mask_inf(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12327,7 +12169,7 @@ static void ggml_compute_forward_diag_mask_inf(}static void ggml_compute_forward_diag_mask_zero(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12347,7 +12189,7 @@ static void ggml_compute_forward_diag_mask_zero(// ggml_compute_forward_soft_maxstatic void ggml_compute_forward_soft_max_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12357,10 +12199,6 @@ static void ggml_compute_forward_soft_max_f32(assert(ggml_is_contiguous(dst));assert(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-float scale = 1.0f;float max_bias = 0.0f;@@ -12463,11 +12301,13 @@ static void ggml_compute_forward_soft_max_f32(}static void ggml_compute_forward_soft_max(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];+ params->nth = MIN(params->nth, ggml_nrows(src0));+switch (src0->type) {case GGML_TYPE_F32:{@@ -12483,7 +12323,7 @@ static void ggml_compute_forward_soft_max(// ggml_compute_forward_soft_max_backstatic void ggml_compute_forward_soft_max_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12495,10 +12335,6 @@ static void ggml_compute_forward_soft_max_back_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));GGML_ASSERT(ggml_are_same_shape(src1, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// TODO: handle transposed/permuted matricesconst int ith = params->ith;@@ -12562,7 +12398,7 @@ static void ggml_compute_forward_soft_max_back_f32(}static void ggml_compute_forward_soft_max_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12582,17 +12418,13 @@ static void ggml_compute_forward_soft_max_back(// ggml_compute_forward_alibistatic void ggml_compute_forward_alibi_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-//const int n_past = ((int32_t *) dst->op_params)[0];const int n_head = ((int32_t *) dst->op_params)[1];float max_bias;@@ -12641,17 +12473,13 @@ static void ggml_compute_forward_alibi_f32(}static void ggml_compute_forward_alibi_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-//const int n_past = ((int32_t *) dst->op_params)[0];const int n_head = ((int32_t *) dst->op_params)[1];float max_bias;@@ -12703,9 +12531,12 @@ static void ggml_compute_forward_alibi_f16(}static void ggml_compute_forward_alibi(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return; // TODO: implement threading support+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -12753,17 +12584,13 @@ static void ggml_compute_forward_alibi(// ggml_compute_forward_clampstatic void ggml_compute_forward_clamp_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-float min;float max;memcpy(&min, (float *) dst->op_params + 0, sizeof(float));@@ -12795,9 +12622,12 @@ static void ggml_compute_forward_clamp_f32(}static void ggml_compute_forward_clamp(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return; // TODO: implement threading support+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -12898,17 +12728,13 @@ GGML_CALL void ggml_rope_yarn_corr_dims(}static void ggml_compute_forward_rope_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const bool forward) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;// these two only relevant for xPos RoPE:@@ -13076,17 +12902,13 @@ static void ggml_compute_forward_rope_f32(}static void ggml_compute_forward_rope_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const bool forward) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;//const int n_past = ((int32_t *) dst->op_params)[0];@@ -13243,7 +13065,7 @@ static void ggml_compute_forward_rope_f16(}static void ggml_compute_forward_rope(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13267,7 +13089,7 @@ static void ggml_compute_forward_rope(// ggml_compute_forward_rope_backstatic void ggml_compute_forward_rope_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13291,7 +13113,7 @@ static void ggml_compute_forward_rope_back(// ggml_compute_forward_conv_transpose_1dstatic void ggml_compute_forward_conv_transpose_1d_f16_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13314,10 +13136,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }+ if (!ith) {memset(params->wdata, 0, params->wsize);// permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)@@ -13350,13 +13169,8 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(// need to zero dst since we are accumulating into itmemset(dst->data, 0, ggml_nbytes(dst));-- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;}+ ggml_syncthreads(params->barrier);const int32_t s0 = ((const int32_t*)(dst->op_params))[0];@@ -13390,7 +13204,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(}static void ggml_compute_forward_conv_transpose_1d_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13413,10 +13227,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32(GGML_ASSERT(nb00 == sizeof(float));GGML_ASSERT(nb10 == sizeof(float));- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }+ if (!ith) {memset(params->wdata, 0, params->wsize);// prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)@@ -13449,13 +13260,8 @@ static void ggml_compute_forward_conv_transpose_1d_f32(// need to zero dst since we are accumulating into itmemset(dst->data, 0, ggml_nbytes(dst));-- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;}+ ggml_syncthreads(params->barrier);const int32_t s0 = ((const int32_t*)(dst->op_params))[0];@@ -13489,7 +13295,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32(}static void ggml_compute_forward_conv_transpose_1d(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13514,7 +13320,7 @@ static void ggml_compute_forward_conv_transpose_1d(// src1: image [N, IC, IH, IW]// dst: result [N, OH, OW, IC*KH*KW]static void ggml_compute_forward_im2col_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13557,14 +13363,6 @@ static void ggml_compute_forward_im2col_f32(GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));- if (params->type == GGML_TASK_TYPE_INIT) {- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]{float * const wdata = (float *) dst->data;@@ -13602,7 +13400,7 @@ static void ggml_compute_forward_im2col_f32(// src1: image [N, IC, IH, IW]// dst: result [N, OH, OW, IC*KH*KW]static void ggml_compute_forward_im2col_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13645,14 +13443,6 @@ static void ggml_compute_forward_im2col_f16(GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));- if (params->type == GGML_TASK_TYPE_INIT) {- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]{ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;@@ -13686,7 +13476,7 @@ static void ggml_compute_forward_im2col_f16(}static void ggml_compute_forward_im2col(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {switch (dst->type) {case GGML_TYPE_F16:@@ -13708,7 +13498,7 @@ static void ggml_compute_forward_im2col(// ggml_compute_forward_conv_transpose_2dstatic void ggml_compute_forward_conv_transpose_2d(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -13731,10 +13521,7 @@ static void ggml_compute_forward_conv_transpose_2d(GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));GGML_ASSERT(nb10 == sizeof(float));- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith != 0) {- return;- }+ if (!ith) {memset(params->wdata, 0, params->wsize);// permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)@@ -13769,13 +13556,8 @@ static void ggml_compute_forward_conv_transpose_2d(}memset(dst->data, 0, ggml_nbytes(dst));-- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;}+ ggml_syncthreads(params->barrier);const int32_t stride = ggml_get_op_params_i32(dst, 0);@@ -13815,7 +13597,7 @@ static void ggml_compute_forward_conv_transpose_2d(// ggml_compute_forward_pool_1d_sk_p0static void ggml_compute_forward_pool_1d_sk_p0(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const enum ggml_op_pool op,const int k,struct ggml_tensor * dst) {@@ -13825,10 +13607,6 @@ static void ggml_compute_forward_pool_1d_sk_p0(assert(src->type == GGML_TYPE_F32);assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const char * cdata = (const char *)src->data;const char * const data_end = cdata + ggml_nbytes(src);float * drow = (float *)dst->data;@@ -13869,9 +13647,12 @@ static void ggml_compute_forward_pool_1d_sk_p0(// ggml_compute_forward_pool_1dstatic void ggml_compute_forward_pool_1d(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const int32_t * opts = (const int32_t *)dst->op_params;enum ggml_op_pool op = opts[0];const int k0 = opts[1];@@ -13886,18 +13667,17 @@ static void ggml_compute_forward_pool_1d(// ggml_compute_forward_pool_2dstatic void ggml_compute_forward_pool_2d(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src = dst->src[0];GGML_ASSERT(src->type == GGML_TYPE_F32);GGML_ASSERT(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int32_t * opts = (const int32_t *)dst->op_params;enum ggml_op_pool op = opts[0];const int k0 = opts[1];@@ -13962,15 +13742,11 @@ static void ggml_compute_forward_pool_2d(// ggml_compute_forward_upscalestatic void ggml_compute_forward_upscale_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));const int ith = params->ith;@@ -14002,7 +13778,7 @@ static void ggml_compute_forward_upscale_f32(}static void ggml_compute_forward_upscale(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -14022,15 +13798,11 @@ static void ggml_compute_forward_upscale(// ggml_compute_forward_padstatic void ggml_compute_forward_pad_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(src0->nb[0] == sizeof(float));GGML_ASSERT( dst->nb[0] == sizeof(float));@@ -14063,7 +13835,7 @@ static void ggml_compute_forward_pad_f32(}static void ggml_compute_forward_pad(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -14084,13 +13856,9 @@ static void ggml_compute_forward_pad(// ggml_compute_forward_arangestatic void ggml_compute_forward_arange_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_ASSERT(dst->nb[0] == sizeof(float));const int ith = params->ith;@@ -14111,7 +13879,7 @@ static void ggml_compute_forward_arange_f32(}static void ggml_compute_forward_arange(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {switch (dst->type) {case GGML_TYPE_F32:@@ -14126,13 +13894,9 @@ static void ggml_compute_forward_arange(}static void ggml_compute_forward_timestep_embedding_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const struct ggml_tensor * src0 = dst->src[0];GGML_ASSERT(src0->nb[0] == sizeof(float));@@ -14163,7 +13927,7 @@ static void ggml_compute_forward_timestep_embedding_f32(}static void ggml_compute_forward_timestep_embedding(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -14183,15 +13947,11 @@ static void ggml_compute_forward_timestep_embedding(// ggml_compute_forward_argsortstatic void ggml_compute_forward_argsort_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_UNARY_OP_LOCALSGGML_ASSERT(nb0 == sizeof(float));@@ -14226,7 +13986,7 @@ static void ggml_compute_forward_argsort_f32(}static void ggml_compute_forward_argsort(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -14246,7 +14006,7 @@ static void ggml_compute_forward_argsort(// ggml_compute_forward_flash_attnstatic void ggml_compute_forward_flash_attn_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -14298,14 +14058,6 @@ static void ggml_compute_forward_flash_attn_f32(GGML_ASSERT(nb1 <= nb2);GGML_ASSERT(nb2 <= nb3);- if (params->type == GGML_TASK_TYPE_INIT) {- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// parallelize by q rows using ggml_vec_dot_f32// total rows in q@@ -14438,7 +14190,7 @@ static void ggml_compute_forward_flash_attn_f32(}static void ggml_compute_forward_flash_attn_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -14490,14 +14242,6 @@ static void ggml_compute_forward_flash_attn_f16(GGML_ASSERT(nb1 <= nb2);GGML_ASSERT(nb2 <= nb3);- if (params->type == GGML_TASK_TYPE_INIT) {- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// parallelize by q rows using ggml_vec_dot_f32// total rows in q@@ -14666,7 +14410,7 @@ static void ggml_compute_forward_flash_attn_f16(}static void ggml_compute_forward_flash_attn(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -14691,7 +14435,7 @@ static void ggml_compute_forward_flash_attn(// ggml_compute_forward_flash_ffstatic void ggml_compute_forward_flash_ff_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * a = dst->src[0]; // F16@@ -14749,14 +14493,6 @@ static void ggml_compute_forward_flash_ff_f16(GGML_ASSERT(nb1 <= nb2);GGML_ASSERT(nb2 <= nb3);- if (params->type == GGML_TASK_TYPE_INIT) {- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// parallelize by a rows using ggml_vec_dot_f32// total rows in a@@ -14826,7 +14562,7 @@ static void ggml_compute_forward_flash_ff_f16(}static void ggml_compute_forward_flash_ff(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * b0 = dst->src[1];@@ -14850,7 +14586,7 @@ static void ggml_compute_forward_flash_ff(// ggml_compute_forward_flash_attn_backstatic void ggml_compute_forward_flash_attn_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -14908,16 +14644,10 @@ static void ggml_compute_forward_flash_attn_back_f32(GGML_ASSERT(nb1 <= nb2);GGML_ASSERT(nb2 <= nb3);- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith == 0) {- memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);- }- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- return;+ if (!ith) {+ memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);}+ ggml_syncthreads(params->barrier);const int64_t elem_q = ggml_nelements(q);const int64_t elem_k = ggml_nelements(k);@@ -15205,7 +14935,7 @@ static void ggml_compute_forward_flash_attn_back_f32(}static void ggml_compute_forward_flash_attn_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -15226,12 +14956,8 @@ static void ggml_compute_forward_flash_attn_back(// ggml_compute_forward_ssm_convstatic void ggml_compute_forward_ssm_conv_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const struct ggml_tensor * src0 = dst->src[0]; // conv_stateconst struct ggml_tensor * src1 = dst->src[1]; // xconst struct ggml_tensor * src2 = dst->src[2]; // conv1d.weight@@ -15335,7 +15061,7 @@ static void ggml_compute_forward_ssm_conv_f32(}static void ggml_compute_forward_ssm_conv(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {switch (dst->src[0]->type) {case GGML_TYPE_F32:@@ -15352,12 +15078,8 @@ static void ggml_compute_forward_ssm_conv(// ggml_compute_forward_ssm_scanstatic void ggml_compute_forward_ssm_scan_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const struct ggml_tensor * src0 = dst->src[0]; // sconst struct ggml_tensor * src1 = dst->src[1]; // xconst struct ggml_tensor * src2 = dst->src[2]; // dt@@ -15460,7 +15182,7 @@ static void ggml_compute_forward_ssm_scan_f32(}static void ggml_compute_forward_ssm_scan(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {switch (dst->src[0]->type) {case GGML_TYPE_F32:@@ -15477,15 +15199,11 @@ static void ggml_compute_forward_ssm_scan(// ggml_compute_forward_win_partstatic void ggml_compute_forward_win_part_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)@@ -15523,9 +15241,12 @@ static void ggml_compute_forward_win_part_f32(}static void ggml_compute_forward_win_part(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -15543,15 +15264,11 @@ static void ggml_compute_forward_win_part(// ggml_compute_forward_win_unpartstatic void ggml_compute_forward_win_unpart_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)@@ -15587,9 +15304,12 @@ static void ggml_compute_forward_win_unpart_f32(}static void ggml_compute_forward_win_unpart(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -15607,7 +15327,7 @@ static void ggml_compute_forward_win_unpart(//gmml_compute_forward_unarystatic void ggml_compute_forward_unary(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const enum ggml_unary_op op = ggml_get_unary_op(dst);@@ -15671,15 +15391,11 @@ static void ggml_compute_forward_unary(// ggml_compute_forward_get_rel_posstatic void ggml_compute_forward_get_rel_pos_f16(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-// ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322GGML_TENSOR_UNARY_OP_LOCALS@@ -15700,9 +15416,12 @@ static void ggml_compute_forward_get_rel_pos_f16(}static void ggml_compute_forward_get_rel_pos(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -15720,7 +15439,7 @@ static void ggml_compute_forward_get_rel_pos(// ggml_compute_forward_add_rel_posstatic void ggml_compute_forward_add_rel_pos_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -15728,17 +15447,12 @@ static void ggml_compute_forward_add_rel_pos_f32(const struct ggml_tensor * src2 = dst->src[2];const bool inplace = (bool) ((int32_t *) dst->op_params)[0];- if (!inplace && params->type == GGML_TASK_TYPE_INIT) {- if (params->ith != 0) {- return;+ if (!inplace) {+ if (!params->ith) {+ memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));}- memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));- return;+ ggml_syncthreads(params->barrier);}- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -15789,7 +15503,7 @@ static void ggml_compute_forward_add_rel_pos_f32(}static void ggml_compute_forward_add_rel_pos(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -15809,7 +15523,7 @@ static void ggml_compute_forward_add_rel_pos(// ggml_compute_forward_map_unarystatic void ggml_compute_forward_map_unary_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_unary_op_f32_t fun) {@@ -15817,10 +15531,6 @@ static void ggml_compute_forward_map_unary_f32(GGML_ASSERT(ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -15835,10 +15545,13 @@ static void ggml_compute_forward_map_unary_f32(}static void ggml_compute_forward_map_unary(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_unary_op_f32_t fun) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -15856,7 +15569,7 @@ static void ggml_compute_forward_map_unary(// ggml_compute_forward_map_binarystatic void ggml_compute_forward_map_binary_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_binary_op_f32_t fun) {@@ -15866,10 +15579,6 @@ static void ggml_compute_forward_map_binary_f32(assert(params->ith == 0);assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const int n = ggml_nrows(src0);const int nc = src0->ne[0];@@ -15886,10 +15595,13 @@ static void ggml_compute_forward_map_binary_f32(}static void ggml_compute_forward_map_binary(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_binary_op_f32_t fun) {+ if (params->ith)+ return;+const struct ggml_tensor * src0 = dst->src[0];switch (src0->type) {@@ -15907,74 +15619,72 @@ static void ggml_compute_forward_map_binary(// ggml_compute_forward_map_custom1static void ggml_compute_forward_map_custom1_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_custom1_op_f32_t fun) {+ if (params->ith)+ return;+const struct ggml_tensor * a = dst->src[0];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-fun(dst, a);}// ggml_compute_forward_map_custom2static void ggml_compute_forward_map_custom2_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_custom2_op_f32_t fun) {+ if (params->ith)+ return;+const struct ggml_tensor * a = dst->src[0];const struct ggml_tensor * b = dst->src[1];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-fun(dst, a, b);}// ggml_compute_forward_map_custom3static void ggml_compute_forward_map_custom3_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst,const ggml_custom3_op_f32_t fun) {+ if (params->ith)+ return;+const struct ggml_tensor * a = dst->src[0];const struct ggml_tensor * b = dst->src[1];const struct ggml_tensor * c = dst->src[1];assert(params->ith == 0);- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-fun(dst, a, b, c);}// ggml_compute_forward_map_custom1static void ggml_compute_forward_map_custom1(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * a = dst->src[0];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-struct ggml_map_custom1_op_params p;memcpy(&p, dst->op_params, sizeof(p));+ if (p.n_tasks != GGML_N_TASKS_MAX) {+ params->nth = MIN(p.n_tasks, params->nth);+ if (params->ith >= params->nth)+ return;+ }p.fun(dst, a, params->ith, params->nth, p.userdata);}@@ -15982,18 +15692,19 @@ static void ggml_compute_forward_map_custom1(// ggml_compute_forward_map_custom2static void ggml_compute_forward_map_custom2(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * a = dst->src[0];const struct ggml_tensor * b = dst->src[1];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-struct ggml_map_custom2_op_params p;memcpy(&p, dst->op_params, sizeof(p));+ if (p.n_tasks != GGML_N_TASKS_MAX) {+ params->nth = MIN(p.n_tasks, params->nth);+ if (params->ith >= params->nth)+ return;+ }p.fun(dst, a, b, params->ith, params->nth, p.userdata);}@@ -16001,19 +15712,20 @@ static void ggml_compute_forward_map_custom2(// ggml_compute_forward_map_custom3static void ggml_compute_forward_map_custom3(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * a = dst->src[0];const struct ggml_tensor * b = dst->src[1];const struct ggml_tensor * c = dst->src[2];- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-struct ggml_map_custom3_op_params p;memcpy(&p, dst->op_params, sizeof(p));+ if (p.n_tasks != GGML_N_TASKS_MAX) {+ params->nth = MIN(p.n_tasks, params->nth);+ if (params->ith >= params->nth)+ return;+ }p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);}@@ -16021,7 +15733,7 @@ static void ggml_compute_forward_map_custom3(// ggml_compute_forward_cross_entropy_lossstatic void ggml_compute_forward_cross_entropy_loss_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -16043,21 +15755,10 @@ static void ggml_compute_forward_cross_entropy_loss_f32(GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc));- if (params->type == GGML_TASK_TYPE_INIT) {- if (ith == 0) {- memset(sums, 0, sizeof(float) * (nth + nth * nc));- }- return;- }-- if (params->type == GGML_TASK_TYPE_FINALIZE) {- if (ith == 0) {- float * dp = (float *) dst->data;- ggml_vec_sum_f32(nth, dp, sums);- dp[0] *= -1.0f / (float) nr;- }- return;+ if (!ith) {+ memset(sums, 0, sizeof(float) * (nth + nth * nc));}+ ggml_syncthreads(params->barrier);const double eps = 1e-9;@@ -16125,11 +15826,17 @@ static void ggml_compute_forward_cross_entropy_loss_f32(}#endif}+ ggml_syncthreads(params->barrier);+ if (!ith) {+ float * dp = (float *) dst->data;+ ggml_vec_sum_f32(nth, dp, sums);+ dp[0] *= -1.0f / (float) nr;+ }}static void ggml_compute_forward_cross_entropy_loss(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -16149,7 +15856,7 @@ static void ggml_compute_forward_cross_entropy_loss(// ggml_compute_forward_cross_entropy_loss_backstatic void ggml_compute_forward_cross_entropy_loss_back_f32(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -16165,10 +15872,6 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(const int64_t ith = params->ith;const int64_t nth = params->nth;- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {- return;- }-const double eps = 1e-9;// TODO: handle transposed/permuted matrices@@ -16241,7 +15944,7 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(}static void ggml_compute_forward_cross_entropy_loss_back(- const struct ggml_compute_params * params,+ struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -18091,9 +17794,7 @@ struct ggml_compute_state_shared {const int n_threads;// synchronization primitives- atomic_int n_active; // num active threads- atomic_int node_n; // active graph node- atomic_int node_task; // active graph node task phase+ struct ggml_barrier barrier;ggml_abort_callback abort_callback; // abort ggml_graph_compute when truevoid * abort_callback_data;@@ -18115,425 +17816,29 @@ static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, constnode->perf_time_us += time_us_cur;}-static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_threads) {- int n_tasks = 0;-- if (ggml_is_empty(node)) {- // no need to multi-thread a no-op- n_tasks = 1;- return n_tasks;- }-- switch (node->op) {- case GGML_OP_CPY:- case GGML_OP_DUP:- case GGML_OP_ADD:- case GGML_OP_ADD1:- case GGML_OP_ACC:- {- n_tasks = n_threads;- } break;- case GGML_OP_SUB:- case GGML_OP_SQR:- case GGML_OP_SQRT:- case GGML_OP_LOG:- case GGML_OP_SUM:- case GGML_OP_SUM_ROWS:- case GGML_OP_MEAN:- case GGML_OP_ARGMAX:- case GGML_OP_REPEAT:- case GGML_OP_REPEAT_BACK:- case GGML_OP_LEAKY_RELU:- {- n_tasks = 1;- } break;- case GGML_OP_UNARY:- switch (ggml_get_unary_op(node)) {- case GGML_UNARY_OP_ABS:- case GGML_UNARY_OP_SGN:- case GGML_UNARY_OP_NEG:- case GGML_UNARY_OP_STEP:- case GGML_UNARY_OP_TANH:- case GGML_UNARY_OP_ELU:- case GGML_UNARY_OP_RELU:- case GGML_UNARY_OP_HARDSWISH: // to opt for multiple threads- case GGML_UNARY_OP_HARDSIGMOID: // to opt for multiple threads- {- n_tasks = 1;- } break;-- case GGML_UNARY_OP_GELU:- case GGML_UNARY_OP_GELU_QUICK:- case GGML_UNARY_OP_SILU:- {- n_tasks = n_threads;- } break;- default:- GGML_ASSERT(false);- }- break;- case GGML_OP_SILU_BACK:- case GGML_OP_MUL:- case GGML_OP_DIV:- case GGML_OP_NORM:- case GGML_OP_RMS_NORM:- case GGML_OP_RMS_NORM_BACK:- case GGML_OP_GROUP_NORM:- case GGML_OP_CONCAT:- {- n_tasks = n_threads;- } break;- case GGML_OP_MUL_MAT:- {- n_tasks = n_threads;-- // TODO: use different scheduling for different matrix sizes- //const int nr0 = ggml_nrows(node->src[0]);- //const int nr1 = ggml_nrows(node->src[1]);-- //n_tasks = MIN(n_threads, MAX(1, nr0/128));- //printf("nr0 = %8d, nr1 = %8d, nr0*nr1 = %8d, n_tasks%d\n", nr0, nr1, nr0*nr1, n_tasks);- } break;- case GGML_OP_MUL_MAT_ID:- {- n_tasks = n_threads;- } break;- case GGML_OP_OUT_PROD:- {- n_tasks = n_threads;- } break;- case GGML_OP_GET_ROWS:- {- // FIXME: the cost of launching additional threads decreases performance with GPU offloading- //n_tasks = MIN(n_threads, ggml_nelements(node->src[1]));- n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1]));- } break;- case GGML_OP_SCALE:- case GGML_OP_SET:- case GGML_OP_CONT:- case GGML_OP_RESHAPE:- case GGML_OP_VIEW:- case GGML_OP_PERMUTE:- case GGML_OP_TRANSPOSE:- case GGML_OP_GET_ROWS_BACK:- case GGML_OP_DIAG:- {- n_tasks = 1;- } break;- case GGML_OP_DIAG_MASK_ZERO:- case GGML_OP_DIAG_MASK_INF:- case GGML_OP_SOFT_MAX_BACK:- case GGML_OP_ROPE:- case GGML_OP_ROPE_BACK:- case GGML_OP_ADD_REL_POS:- {- n_tasks = n_threads;- } break;- case GGML_OP_ALIBI:- {- n_tasks = 1; //TODO- } break;- case GGML_OP_CLAMP:- {- n_tasks = 1; //TODO- } break;- case GGML_OP_SOFT_MAX:- {- n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));- } break;- case GGML_OP_CONV_TRANSPOSE_1D:- {- n_tasks = n_threads;- } break;- case GGML_OP_IM2COL:- {- n_tasks = n_threads;- } break;- case GGML_OP_CONV_TRANSPOSE_2D:- {- n_tasks = n_threads;- } break;- case GGML_OP_POOL_1D:- case GGML_OP_POOL_2D:- {- n_tasks = 1;- } break;- case GGML_OP_UPSCALE:- {- n_tasks = n_threads;- } break;- case GGML_OP_PAD:- {- n_tasks = n_threads;- } break;- case GGML_OP_ARANGE:- {- n_tasks = n_threads;- } break;- case GGML_OP_TIMESTEP_EMBEDDING:- {- n_tasks = n_threads;- } break;- case GGML_OP_ARGSORT:- {- n_tasks = n_threads;- } break;- case GGML_OP_FLASH_ATTN:- {- n_tasks = n_threads;- } break;- case GGML_OP_FLASH_FF:- {- n_tasks = n_threads;- } break;- case GGML_OP_FLASH_ATTN_BACK:- {- n_tasks = n_threads;- } break;- case GGML_OP_SSM_CONV:- case GGML_OP_SSM_SCAN:- {- n_tasks = n_threads;- } break;- case GGML_OP_WIN_PART:- case GGML_OP_WIN_UNPART:- case GGML_OP_GET_REL_POS:- case GGML_OP_MAP_UNARY:- case GGML_OP_MAP_BINARY:- case GGML_OP_MAP_CUSTOM1_F32:- case GGML_OP_MAP_CUSTOM2_F32:- case GGML_OP_MAP_CUSTOM3_F32:- {- n_tasks = 1;- } break;- case GGML_OP_MAP_CUSTOM1:- {- struct ggml_map_custom1_op_params p;- memcpy(&p, node->op_params, sizeof(p));- if (p.n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p.n_tasks, n_threads);- }- } break;- case GGML_OP_MAP_CUSTOM2:- {- struct ggml_map_custom2_op_params p;- memcpy(&p, node->op_params, sizeof(p));- if (p.n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p.n_tasks, n_threads);- }- } break;- case GGML_OP_MAP_CUSTOM3:- {- struct ggml_map_custom3_op_params p;- memcpy(&p, node->op_params, sizeof(p));- if (p.n_tasks == GGML_N_TASKS_MAX) {- n_tasks = n_threads;- } else {- n_tasks = MIN(p.n_tasks, n_threads);- }- } break;- case GGML_OP_CROSS_ENTROPY_LOSS:- {- n_tasks = n_threads;- } break;- case GGML_OP_CROSS_ENTROPY_LOSS_BACK:- {- n_tasks = n_threads;- } break;- case GGML_OP_NONE:- {- n_tasks = 1;- } break;- case GGML_OP_COUNT:- {- GGML_ASSERT(false);- } break;- default:- {- fprintf(stderr, "%s: op not implemented: ", __func__);- if (node->op < GGML_OP_COUNT) {- fprintf(stderr, "%s\n", ggml_op_name(node->op));- } else {- fprintf(stderr, "%d\n", node->op);- }- GGML_ASSERT(false);- } break;- }-- assert(n_tasks > 0);-- return n_tasks;-}--static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_compute_state * state, const bool do_yield) {- // wait for other threads to finish- const int last_node_n = * node_n;-- while (true) {- if (do_yield) {- sched_yield();- }-- * node_n = atomic_load(&state->shared->node_n);- if (* node_n != last_node_n) break;- }-}--static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_compute_state * state, const bool do_yield) {- // wait for other threads to finish- const int last_task_phase = * task_phase;-- while (true) {- if (do_yield) {- sched_yield();- }-- * task_phase = atomic_load(&state->shared->node_task);- if (* task_phase != last_task_phase) break;- }-}-static thread_ret_t ggml_graph_compute_thread(void * data) {struct ggml_compute_state * state = (struct ggml_compute_state *) data;const struct ggml_cgraph * cgraph = state->shared->cgraph;const struct ggml_cplan * cplan = state->shared->cplan;-- const int n_threads = state->shared->n_threads;+ struct ggml_barrier * b = &state->shared->barrier;set_numa_thread_affinity(state->ith);- int node_n = -1;- int task_phase = GGML_TASK_TYPE_FINALIZE;-- while (true) {+ for (int node_n = 0; node_n < cgraph->n_nodes; ++node_n) {if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {- state->shared->node_n += 1;state->ec = GGML_STATUS_ABORTED;return 0;}-- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {- // all other threads are finished and spinning- // do finalize and init here so we don't have synchronize again- struct ggml_compute_params params = {- /*.type =*/ GGML_TASK_TYPE_FINALIZE,- /*.ith =*/ 0,- /*.nth =*/ 0,- /*.wsize =*/ cplan->work_size,- /*.wdata =*/ cplan->work_data,- };-- if (node_n != -1) {- /* FINALIZE */- struct ggml_tensor * node = cgraph->nodes[node_n];- if (GGML_OP_HAS_FINALIZE[node->op]) {- params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);- ggml_compute_forward(¶ms, node);- }- ggml_graph_compute_perf_stats_node(node, state->shared);- }-- // distribute new work or execute it direct if 1T- while (++node_n < cgraph->n_nodes) {- GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes);- struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);-- state->shared->perf_node_start_cycles = ggml_perf_cycles();- state->shared->perf_node_start_time_us = ggml_perf_time_us();-- params.nth = n_tasks;-- if (n_tasks == 1) {- /* INIT */- if (GGML_OP_HAS_INIT[node->op]) {- params.type = GGML_TASK_TYPE_INIT;- ggml_compute_forward(¶ms, node);- }-- // TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1,- // they do something more efficient than spinning (?)- params.type = GGML_TASK_TYPE_COMPUTE;- ggml_compute_forward(¶ms, node);-- if (GGML_OP_HAS_FINALIZE[node->op]) {- params.type = GGML_TASK_TYPE_FINALIZE;- ggml_compute_forward(¶ms, node);- }-- ggml_graph_compute_perf_stats_node(node, state->shared);- } else {- break;- }-- if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {- break;- }- }-- task_phase = GGML_TASK_TYPE_INIT;- atomic_store(&state->shared->n_active, n_threads);- atomic_store(&state->shared->node_n, node_n);- atomic_store(&state->shared->node_task, task_phase);- } else {- ggml_graph_compute_thread_sync_node(&node_n, state, false);- ggml_graph_compute_thread_sync_task(&task_phase, state, false);- }-- // check if we should stop- if (node_n >= cgraph->n_nodes) break;-- /* INIT & COMPUTE */- struct ggml_tensor * node = cgraph->nodes[node_n];- const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);-struct ggml_compute_params params = {- /*.type =*/ GGML_TASK_TYPE_INIT,- /*.ith =*/ state->ith,- /*.nth =*/ n_tasks,- /*.wsize =*/ cplan->work_size,- /*.wdata =*/ cplan->work_data,+ /*.ith =*/ state->ith,+ /*.nth =*/ state->shared->n_threads,+ /*.wsize =*/ cplan->work_size,+ /*.wdata =*/ cplan->work_data,+ /*.barrier =*/ &state->shared->barrier,};-- if (state->ith < n_tasks) {- if (GGML_OP_HAS_INIT[node->op]) {- ggml_compute_forward(¶ms, node);- }- }-- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {- task_phase = GGML_TASK_TYPE_COMPUTE;- atomic_store(&state->shared->n_active, n_threads);- atomic_store(&state->shared->node_task, task_phase);- }- else {- // TODO: this sched_yield can have significant impact on the performance - either positive or negative- // depending on the workload and the operating system.- // since it is not clear what is the best approach, it should potentially become user-configurable- // ref: https://github.com/ggerganov/ggml/issues/291- // UPD: adding the do_yield flag seems to resolve the issue universally- const bool do_yield = node_n < 0 || cgraph->nodes[node_n]->op == GGML_OP_MUL_MAT;- ggml_graph_compute_thread_sync_task(&task_phase, state, do_yield);- }-- if (state->ith < n_tasks) {- params.type = GGML_TASK_TYPE_COMPUTE;- ggml_compute_forward(¶ms, node);- }-- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {- task_phase = GGML_TASK_TYPE_FINALIZE;- atomic_store(&state->shared->n_active, n_threads);- atomic_store(&state->shared->node_task, task_phase);- }- else {- ggml_graph_compute_thread_sync_task(&task_phase, state, false);- }+ ggml_compute_forward(¶ms, cgraph->nodes[node_n]);+ ggml_syncthreads(&state->shared->barrier);}return 0;@@ -18554,9 +17859,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa// thread scheduling for the different operations + work buffer size estimationfor (int i = 0; i < cgraph->n_nodes; i++) {struct ggml_tensor * node = cgraph->nodes[i];-- const int n_tasks = ggml_get_n_tasks(node, n_threads, 1);-+ const int n_tasks = n_threads;max_tasks = MAX(max_tasks, n_tasks);size_t cur = 0;@@ -18738,7 +18041,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threawork_size += CACHE_LINE_SIZE*(n_threads - 1);}- cplan.n_threads = MIN(max_tasks, n_threads);+ cplan.n_threads = n_threads;cplan.work_size = work_size;cplan.work_data = NULL;@@ -18763,9 +18066,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl/*.perf_node_start_cycles =*/ 0,/*.perf_node_start_time_us =*/ 0,/*.n_threads =*/ n_threads,- /*.n_active =*/ n_threads,- /*.node_n =*/ -1,- /*.node_task =*/ GGML_TASK_TYPE_FINALIZE,+ /*.barrier =*/ { n_threads },/*.abort_callback =*/ NULL,/*.abort_callback_data =*/ NULL,};diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex d857c58c..2fef0d93 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -666,16 +666,9 @@ extern "C" {// compute types- // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled.- // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995.- enum ggml_task_type {- GGML_TASK_TYPE_INIT = 0,- GGML_TASK_TYPE_COMPUTE,- GGML_TASK_TYPE_FINALIZE,- };+ struct ggml_barrier;struct ggml_compute_params {- enum ggml_task_type type;// ith = thread index, nth = number of threadsint ith, nth;@@ -683,8 +676,12 @@ extern "C" {// work buffer for all threadssize_t wsize;void * wdata;++ struct ggml_barrier *barrier;};+ GGML_API void ggml_syncthreads(struct ggml_barrier *);+// numa strategiesenum ggml_numa_strategy {GGML_NUMA_STRATEGY_DISABLED = 0,diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex e429e53c..0f749ef8 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -49,7 +49,7 @@ int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);bool llamafile_sgemm(long, long, long, const void *, long, const void *, long, void *, long, int,- int, int, int, int, int);+ int, int, int, int);struct ggml_tensor;struct ggml_compute_params;diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 26d9d82e..f5ef0fef 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -107,16 +107,14 @@ static const struct GemmFuncs {* @param ldc is row stride of `C`* @param ith is thread id (must be less than `nth`)* @param nth is number of threads (must be greater than zero)- * @param task is GGML task type* @param Atype is GGML data type of `A`* @param Btype is GGML data type of `B`* @param Ctype is GGML data type of `C`* @return true if this function was able to service the matmul request*/bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,- int Ctype) {- return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);+ void *C, long ldc, int ith, int nth, int Atype, int Btype, int Ctype) {+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, Atype, Btype, Ctype);}/**diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 5bda13fe..af583485 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -8,23 +8,23 @@ struct ggml_tensor;struct ggml_compute_params;bool llamafile_sgemm_unsupported(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int);bool llamafile_sgemm_amd_avx(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int);bool llamafile_sgemm_amd_fma(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int);bool llamafile_sgemm_amd_avx2(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int);bool llamafile_sgemm_amd_avxvnni(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int);bool llamafile_sgemm_amd_avx512f(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int);bool llamafile_sgemm_amd_zen4(long, long, long, const void *, long, const void *, long, void *,- long, int, int, int, int, int, int);+ long, int, int, int, int, int);bool llamafile_sgemm_arm80(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int);bool llamafile_sgemm_arm82(long, long, long, const void *, long, const void *, long, void *, long,- int, int, int, int, int, int);+ int, int, int, int, int);bool llamafile_mixmul_unsupported(const struct ggml_compute_params *, const struct ggml_tensor *,const struct ggml_tensor *, const struct ggml_tensor *,diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppindex 3825989c..f0041c09 100644--- a/llamafile/sgemm_sss_test.cpp+++ b/llamafile/sgemm_sss_test.cpp@@ -46,8 +46,8 @@ int main(int argc, char *argv[]) {randomize(k, n, B, ldb);BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TYPE_F32, GGML_TYPE_F32,+ GGML_TYPE_F32));for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 283973ad..b153f48d 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -45,8 +45,8 @@ int main(int argc, char *argv[]) {randomize(k, n, B, ldb);BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TYPE_F32, GGML_TYPE_F32,+ GGML_TYPE_F32));double err_sum = 0;long long err_worst = 0;diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.incindex 5263d598..1b02f486 100644--- a/llamafile/tinyblas_cpu.inc+++ b/llamafile/tinyblas_cpu.inc@@ -41,6 +41,7 @@#include "llama.cpp/ggml-impl.h"#include "llama.cpp/ggml-quants.h"#include "llamafile.h"+#include "sgemm.h"#include <cosmo.h>#pragma GCC diagnostic ignored "-Wpedantic"@@ -350,6 +351,10 @@ template <>inline __m512bh load(const ggml_bf16_t *p) {return (__m512bh)_mm512_loadu_ps((const float *)p);}+template <>+inline __m512bh load(const float *p) {+ return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));+}#endif // __AVX512BF16__////////////////////////////////////////////////////////////////////////////////////////////////////@@ -363,9 +368,8 @@ class tinyBLAS {: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}- void matmul(long m, long n, int task) {- if (task == GGML_TASK_TYPE_COMPUTE)- mnpack(0, m, 0, n);+ void matmul(long m, long n) {+ mnpack(0, m, 0, n);}private:@@ -571,9 +575,8 @@ class tinyBLAS_Q0_ARM {: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}- void matmul(long m, long n, int task) {- if (task == GGML_TASK_TYPE_COMPUTE)- mnpack(0, m, 0, n);+ void matmul(long m, long n) {+ mnpack(0, m, 0, n);}private:@@ -704,9 +707,8 @@ class tinyBLAS_Q0_AVX2 {: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}- void matmul(long m, long n, int task) {- if (task == GGML_TASK_TYPE_COMPUTE)- mnpack(0, m, 0, n);+ void matmul(long m, long n) {+ mnpack(0, m, 0, n);}private:@@ -895,8 +897,7 @@ class tinyBLAS_Q0_AVX2 {** For example, for single-threaded single-precision GEMM you can say*- * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,- * 0, 1, GGML_TASK_TYPE_COMPUTE,+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,* GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);** @param m is rows in `A` and `C`@@ -910,15 +911,13 @@ class tinyBLAS_Q0_AVX2 {* @param ldc is row stride of `C`* @param ith is thread id (must be less than `nth`)* @param nth is number of threads (must be greater than zero)- * @param task is GGML task type* @param Atype is GGML data type of `A`* @param Btype is GGML data type of `B`* @param Ctype is GGML data type of `C`* @return true if this function was able to service the matmul request*/bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,- int Ctype) {+ void *C, long ldc, int ith, int nth, int Atype, int Btype, int Ctype) {assert(m >= 0);assert(n >= 0);@@ -942,14 +941,14 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__AVX__) || defined(__AVX2__)if (k % 8)return false;tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__ARM_NEON)if (n < 4)@@ -958,7 +957,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elsereturn false;@@ -969,20 +968,24 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void#if defined(__AVX512BF16__)if (k % 32)return false;+ if (Btype == GGML_TYPE_F32 && m * n < 500000) {+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, float> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+ }if (Btype != GGML_TYPE_BF16)return false;tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__AVX512F__)if (k % 16)return false;- if (Btype != GGML_TYPE_F32)- return false;tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__AVX2__)if (k % 8)@@ -991,7 +994,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__ARM_NEON) && !defined(_MSC_VER)if (k % 4)@@ -1000,7 +1003,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elsereturn false;@@ -1011,27 +1014,41 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void#if defined(__AVX512F__)if (k % 16)return false;- if (Btype != GGML_TYPE_F32)+ if (Btype == GGML_TYPE_F32 && m * n < 500000) {+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+ }+ if (Btype != GGML_TYPE_F16)return false;- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n);return true;#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)if (X86_HAVE(F16C)) {if (k % 8)return false;- if (Btype != GGML_TYPE_F32)+ if (0 && Btype == GGML_TYPE_F32 && m * n < 500000) {+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith,+ nth};+ tb.matmul(m, n);+ return true;+ }+ if (Btype != GGML_TYPE_F16)return false;- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith,+ nth};+ tb.matmul(m, n);return true;} else {return false;}#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)- if (n < 8)+ if (n < 2)return false;if (k % 8)return false;@@ -1039,7 +1056,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__ARM_NEON) && !defined(_MSC_VER)if (k % 4)@@ -1048,7 +1065,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidreturn false;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elsereturn false;@@ -1061,12 +1078,12 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void#if defined(__AVX2__) || defined(__AVX512F__)tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__ARM_FEATURE_DOTPROD)tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, float> tb{k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elsereturn false;@@ -1079,12 +1096,12 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void#if defined(__AVX2__) || defined(__AVX512F__)tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elif defined(__ARM_FEATURE_DOTPROD)tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, float> tb{k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};- tb.matmul(m, n, task);+ tb.matmul(m, n);return true;#elsereturn false;@@ -1106,7 +1123,6 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void(void)ldc;(void)ith;(void)nth;- (void)task;(void)Atype;(void)Btype;(void)Ctype;@@ -1352,31 +1368,25 @@ class MixMul {bool mixmat() {if (cols % KN)return false;- switch (params->type) {- case GGML_TASK_TYPE_INIT:- if (thought->type != ggml_type_trait<TB>::id)- quantize_thought(ggml_type_trait<TB>::id);- build_row_pointers(ggml_type_trait<TB>::id);- return true;- case GGML_TASK_TYPE_COMPUTE:- assert(!(cols % BS));- assert(!(weights->nb[1] % sizeof(TA)));- for (int expert = 0; expert < experts; ++expert) {- BLAS tb{cols / BS,- (const TA *)((const char *)weights->data + expert * weights->nb[2]),- (long)(weights->nb[1] / sizeof(TA)),- (const TB *)(rowptr_thought_ + expert * tokens * thinkers),- 0,- (TC *)(rowptr_result_ + expert * tokens * thinkers),- 0,- params->ith,- params->nth};- tb.matmul(rows, rowptr_count_[expert], GGML_TASK_TYPE_COMPUTE);- }- return true;- default:- return true;+ if (thought->type != ggml_type_trait<TB>::id)+ quantize_thought(ggml_type_trait<TB>::id);+ build_row_pointers(ggml_type_trait<TB>::id);+ ggml_syncthreads(params->barrier);+ assert(!(cols % BS));+ assert(!(weights->nb[1] % sizeof(TA)));+ for (int expert = 0; expert < experts; ++expert) {+ BLAS tb{cols / BS,+ (const TA *)((const char *)weights->data + expert * weights->nb[2]),+ (long)(weights->nb[1] / sizeof(TA)),+ (const TB *)(rowptr_thought_ + expert * tokens * thinkers),+ 0,+ (TC *)(rowptr_result_ + expert * tokens * thinkers),+ 0,+ params->ith,+ params->nth};+ tb.matmul(rows, rowptr_count_[expert]);}+ return true;}void build_row_pointers(ggml_type vec_dot_type) {