Skip to content

Distribute and run LLMs with a single file.

Rewrite synchronization primitives

Justine Tunney6162004caef5
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 11bdd360..2c50b729 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -7107,6 +7107,36 @@ struct ggml_tensor * ggml_cross_entropy_loss_back(
////////////////////////////////////////////////////////////////////////////////
+struct ggml_barrier {
+ int nth;
+ atomic_uint phase;
+ atomic_uint count;
+};
+
+int ggml_delay(int backoff) {
+ if (backoff < 13) {
+ volatile int i;
+ for (i = 0; i != 1 << backoff; i++) {
+ }
+ backoff++;
+ } else {
+ sched_yield();
+ }
+ return backoff;
+}
+
+void ggml_syncthreads(struct ggml_barrier *b) {
+ unsigned phase = atomic_load_explicit(&b->phase, memory_order_relaxed);
+ if (atomic_fetch_add_explicit(&b->count, 1, memory_order_acq_rel) + 1 == b->nth) {
+ atomic_store_explicit(&b->count, 0, memory_order_relaxed);
+ atomic_store_explicit(&b->phase, phase + 1, memory_order_release);
+ } else {
+ int backoff = 0;
+ while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase)
+ backoff = ggml_delay(backoff);
+ }
+}
+
void ggml_set_param(
struct ggml_context * ctx,
struct ggml_tensor * tensor) {
@@ -7120,7 +7150,7 @@ void ggml_set_param(
// ggml_compute_forward_dup
static void ggml_compute_forward_dup_same_cont(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -7129,10 +7159,6 @@ static void ggml_compute_forward_dup_same_cont(
GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
GGML_ASSERT(src0->type == dst->type);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const size_t nb00 = src0->nb[0];
const size_t nb0 = dst->nb[0];
@@ -7154,17 +7180,13 @@ static void ggml_compute_forward_dup_same_cont(
}
static void ggml_compute_forward_dup_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
const int ith = params->ith; // thread index
@@ -7427,17 +7449,13 @@ static void ggml_compute_forward_dup_f16(
}
static void ggml_compute_forward_dup_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
const int ith = params->ith; // thread index
@@ -7679,7 +7697,7 @@ static void ggml_compute_forward_dup_f32(
// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.
static void ggml_compute_forward_dup_bytes(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -7687,10 +7705,6 @@ static void ggml_compute_forward_dup_bytes(
GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
GGML_ASSERT(src0->type == dst->type);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
ggml_compute_forward_dup_same_cont(params, dst);
return;
@@ -7830,7 +7844,7 @@ static void ggml_compute_forward_dup_bytes(
}
static void ggml_compute_forward_dup(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -7859,7 +7873,7 @@ static void ggml_compute_forward_dup(
// ggml_compute_forward_add
static void ggml_compute_forward_add_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -7867,10 +7881,6 @@ static void ggml_compute_forward_add_f32(
GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -7949,7 +7959,7 @@ static void ggml_compute_forward_add_f32(
}
static void ggml_compute_forward_add_f16_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -7957,10 +7967,6 @@ static void ggml_compute_forward_add_f16_f32(
GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -8028,7 +8034,7 @@ static void ggml_compute_forward_add_f16_f32(
}
static void ggml_compute_forward_add_f16_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8036,10 +8042,6 @@ static void ggml_compute_forward_add_f16_f16(
GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -8084,7 +8086,7 @@ static void ggml_compute_forward_add_f16_f16(
}
static void ggml_compute_forward_add_q_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8092,10 +8094,6 @@ static void ggml_compute_forward_add_q_f32(
GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int nr = ggml_nrows(src0);
GGML_TENSOR_BINARY_OP_LOCALS
@@ -8164,7 +8162,7 @@ static void ggml_compute_forward_add_q_f32(
}
static void ggml_compute_forward_add(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8224,7 +8222,7 @@ static void ggml_compute_forward_add(
// ggml_compute_forward_add1
static void ggml_compute_forward_add1_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8233,10 +8231,6 @@ static void ggml_compute_forward_add1_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_is_scalar(src1));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -8278,7 +8272,7 @@ static void ggml_compute_forward_add1_f32(
}
static void ggml_compute_forward_add1_f16_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8287,10 +8281,6 @@ static void ggml_compute_forward_add1_f16_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_is_scalar(src1));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// scalar to add
const float v = *(float *) src1->data;
@@ -8330,7 +8320,7 @@ static void ggml_compute_forward_add1_f16_f32(
}
static void ggml_compute_forward_add1_f16_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8339,10 +8329,6 @@ static void ggml_compute_forward_add1_f16_f16(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_is_scalar(src1));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// scalar to add
const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);
@@ -8382,7 +8368,7 @@ static void ggml_compute_forward_add1_f16_f16(
}
static void ggml_compute_forward_add1_q_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8391,10 +8377,6 @@ static void ggml_compute_forward_add1_q_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_is_scalar(src1));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// scalar to add
const float v = *(float *) src1->data;
@@ -8451,7 +8433,7 @@ static void ggml_compute_forward_add1_q_f32(
}
static void ggml_compute_forward_add1(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8507,7 +8489,7 @@ static void ggml_compute_forward_add1(
// ggml_compute_forward_acc
static void ggml_compute_forward_acc_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8524,20 +8506,16 @@ static void ggml_compute_forward_acc_f32(
size_t offset = ((int32_t *) dst->op_params)[3];
bool inplace = (bool) ((int32_t *) dst->op_params)[4];
- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {
- if (params->ith != 0) {
- return;
+ if (!inplace) {
+ if (!params->ith) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
}
- // memcpy needs to be synchronized across threads to avoid race conditions.
- // => do it in INIT phase
- memcpy(
- ((char *) dst->data),
- ((char *) src0->data),
- ggml_nbytes(dst));
- }
-
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ ggml_syncthreads(params->barrier);
}
const int ith = params->ith;
@@ -8591,7 +8569,7 @@ static void ggml_compute_forward_acc_f32(
}
static void ggml_compute_forward_acc(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8632,19 +8610,17 @@ static void ggml_compute_forward_acc(
// ggml_compute_forward_sub
static void ggml_compute_forward_sub_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- assert(params->ith == 0);
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int nr = ggml_nrows(src0);
GGML_TENSOR_BINARY_OP_LOCALS
@@ -8694,7 +8670,7 @@ static void ggml_compute_forward_sub_f32(
}
static void ggml_compute_forward_sub(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8714,7 +8690,7 @@ static void ggml_compute_forward_sub(
// ggml_compute_forward_mul
static void ggml_compute_forward_mul_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8722,9 +8698,6 @@ static void ggml_compute_forward_mul_f32(
GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
const int ith = params->ith;
const int nth = params->nth;
@@ -8799,7 +8772,7 @@ static void ggml_compute_forward_mul_f32(
}
static void ggml_compute_forward_mul(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8822,7 +8795,7 @@ static void ggml_compute_forward_mul(
// ggml_compute_forward_div
static void ggml_compute_forward_div_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8830,10 +8803,6 @@ static void ggml_compute_forward_div_f32(
GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -8897,7 +8866,7 @@ static void ggml_compute_forward_div_f32(
}
static void ggml_compute_forward_div(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8917,7 +8886,7 @@ static void ggml_compute_forward_div(
// ggml_compute_forward_sqr
static void ggml_compute_forward_sqr_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8925,10 +8894,6 @@ static void ggml_compute_forward_sqr_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -8943,9 +8908,12 @@ static void ggml_compute_forward_sqr_f32(
}
static void ggml_compute_forward_sqr(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -8963,7 +8931,7 @@ static void ggml_compute_forward_sqr(
// ggml_compute_forward_sqrt
static void ggml_compute_forward_sqrt_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -8971,10 +8939,6 @@ static void ggml_compute_forward_sqrt_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -8989,9 +8953,12 @@ static void ggml_compute_forward_sqrt_f32(
}
static void ggml_compute_forward_sqrt(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9009,7 +8976,7 @@ static void ggml_compute_forward_sqrt(
// ggml_compute_forward_log
static void ggml_compute_forward_log_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9017,10 +8984,6 @@ static void ggml_compute_forward_log_f32(
GGML_ASSERT(params->ith == 0);
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9035,9 +8998,12 @@ static void ggml_compute_forward_log_f32(
}
static void ggml_compute_forward_log(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9055,7 +9021,7 @@ static void ggml_compute_forward_log(
// ggml_compute_forward_sum
static void ggml_compute_forward_sum_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9063,10 +9029,6 @@ static void ggml_compute_forward_sum_f32(
assert(params->ith == 0);
assert(ggml_is_scalar(dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
assert(ggml_is_scalar(dst));
assert(src0->nb[0] == sizeof(float));
@@ -9090,7 +9052,7 @@ static void ggml_compute_forward_sum_f32(
}
static void ggml_compute_forward_sum_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9098,10 +9060,6 @@ static void ggml_compute_forward_sum_f16(
assert(params->ith == 0);
assert(ggml_is_scalar(dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
assert(src0->nb[0] == sizeof(ggml_fp16_t));
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
@@ -9124,9 +9082,12 @@ static void ggml_compute_forward_sum_f16(
}
static void ggml_compute_forward_sum(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9148,17 +9109,13 @@ static void ggml_compute_forward_sum(
// ggml_compute_forward_sum_rows
static void ggml_compute_forward_sum_rows_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
GGML_ASSERT(dst->nb[0] == sizeof(float));
@@ -9183,9 +9140,12 @@ static void ggml_compute_forward_sum_rows_f32(
}
static void ggml_compute_forward_sum_rows(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9203,17 +9163,13 @@ static void ggml_compute_forward_sum_rows(
// ggml_compute_forward_mean
static void ggml_compute_forward_mean_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
assert(src0->nb[0] == sizeof(float));
GGML_TENSOR_UNARY_OP_LOCALS
@@ -9242,9 +9198,12 @@ static void ggml_compute_forward_mean_f32(
}
static void ggml_compute_forward_mean(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9262,17 +9221,13 @@ static void ggml_compute_forward_mean(
// ggml_compute_forward_argmax
static void ggml_compute_forward_argmax_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
assert(src0->nb[0] == sizeof(float));
assert(dst->nb[0] == sizeof(float));
@@ -9292,9 +9247,12 @@ static void ggml_compute_forward_argmax_f32(
}
static void ggml_compute_forward_argmax(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9312,7 +9270,7 @@ static void ggml_compute_forward_argmax(
// ggml_compute_forward_repeat
static void ggml_compute_forward_repeat_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9320,10 +9278,6 @@ static void ggml_compute_forward_repeat_f32(
GGML_ASSERT(params->ith == 0);
GGML_ASSERT(ggml_can_repeat(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
// guaranteed to be an integer due to the check in ggml_can_repeat
@@ -9357,7 +9311,7 @@ static void ggml_compute_forward_repeat_f32(
}
static void ggml_compute_forward_repeat_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9365,10 +9319,6 @@ static void ggml_compute_forward_repeat_f16(
GGML_ASSERT(params->ith == 0);
GGML_ASSERT(ggml_can_repeat(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
// guaranteed to be an integer due to the check in ggml_can_repeat
@@ -9405,9 +9355,12 @@ static void ggml_compute_forward_repeat_f16(
}
static void ggml_compute_forward_repeat(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9431,7 +9384,7 @@ static void ggml_compute_forward_repeat(
// ggml_compute_forward_repeat_back
static void ggml_compute_forward_repeat_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9439,10 +9392,6 @@ static void ggml_compute_forward_repeat_back_f32(
GGML_ASSERT(params->ith == 0);
GGML_ASSERT(ggml_can_repeat(dst, src0));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
// guaranteed to be an integer due to the check in ggml_can_repeat
@@ -9490,9 +9439,12 @@ static void ggml_compute_forward_repeat_back_f32(
}
static void ggml_compute_forward_repeat_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9510,16 +9462,12 @@ static void ggml_compute_forward_repeat_back(
// ggml_compute_forward_concat
static void ggml_compute_forward_concat_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -9559,7 +9507,7 @@ static void ggml_compute_forward_concat_f32(
}
static void ggml_compute_forward_concat(
- const struct ggml_compute_params* params,
+ struct ggml_compute_params* params,
struct ggml_tensor* dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9580,7 +9528,7 @@ static void ggml_compute_forward_concat(
// ggml_compute_forward_abs
static void ggml_compute_forward_abs_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9588,10 +9536,6 @@ static void ggml_compute_forward_abs_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9606,9 +9550,12 @@ static void ggml_compute_forward_abs_f32(
}
static void ggml_compute_forward_abs(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9626,7 +9573,7 @@ static void ggml_compute_forward_abs(
// ggml_compute_forward_sgn
static void ggml_compute_forward_sgn_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9634,10 +9581,6 @@ static void ggml_compute_forward_sgn_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9652,9 +9595,12 @@ static void ggml_compute_forward_sgn_f32(
}
static void ggml_compute_forward_sgn(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9672,7 +9618,7 @@ static void ggml_compute_forward_sgn(
// ggml_compute_forward_neg
static void ggml_compute_forward_neg_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9680,10 +9626,6 @@ static void ggml_compute_forward_neg_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9698,9 +9640,12 @@ static void ggml_compute_forward_neg_f32(
}
static void ggml_compute_forward_neg(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9718,7 +9663,7 @@ static void ggml_compute_forward_neg(
// ggml_compute_forward_step
static void ggml_compute_forward_step_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9726,10 +9671,6 @@ static void ggml_compute_forward_step_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9744,9 +9685,12 @@ static void ggml_compute_forward_step_f32(
}
static void ggml_compute_forward_step(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9764,7 +9708,7 @@ static void ggml_compute_forward_step(
// ggml_compute_forward_tanh
static void ggml_compute_forward_tanh_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9772,10 +9716,6 @@ static void ggml_compute_forward_tanh_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9790,9 +9730,12 @@ static void ggml_compute_forward_tanh_f32(
}
static void ggml_compute_forward_tanh(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9810,7 +9753,7 @@ static void ggml_compute_forward_tanh(
// ggml_compute_forward_elu
static void ggml_compute_forward_elu_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9818,10 +9761,6 @@ static void ggml_compute_forward_elu_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9836,9 +9775,12 @@ static void ggml_compute_forward_elu_f32(
}
static void ggml_compute_forward_elu(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9856,7 +9798,7 @@ static void ggml_compute_forward_elu(
// ggml_compute_forward_relu
static void ggml_compute_forward_relu_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9864,10 +9806,6 @@ static void ggml_compute_forward_relu_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -9882,9 +9820,12 @@ static void ggml_compute_forward_relu_f32(
}
static void ggml_compute_forward_relu(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -9902,7 +9843,7 @@ static void ggml_compute_forward_relu(
// ggml_compute_forward_gelu
static void ggml_compute_forward_gelu_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9911,10 +9852,6 @@ static void ggml_compute_forward_gelu_f32(
GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -9945,7 +9882,7 @@ static void ggml_compute_forward_gelu_f32(
}
static void ggml_compute_forward_gelu(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9965,7 +9902,7 @@ static void ggml_compute_forward_gelu(
// ggml_compute_forward_gelu_quick
static void ggml_compute_forward_gelu_quick_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -9974,10 +9911,6 @@ static void ggml_compute_forward_gelu_quick_f32(
GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -10008,7 +9941,7 @@ static void ggml_compute_forward_gelu_quick_f32(
}
static void ggml_compute_forward_gelu_quick(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10028,7 +9961,7 @@ static void ggml_compute_forward_gelu_quick(
// ggml_compute_forward_silu
static void ggml_compute_forward_silu_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10037,10 +9970,6 @@ static void ggml_compute_forward_silu_f32(
GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -10071,7 +10000,7 @@ static void ggml_compute_forward_silu_f32(
}
static void ggml_compute_forward_silu(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10090,7 +10019,7 @@ static void ggml_compute_forward_silu(
// ggml_compute_forward_leaky_relu
static void ggml_compute_forward_leaky_relu_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10098,10 +10027,6 @@ static void ggml_compute_forward_leaky_relu_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -10119,9 +10044,12 @@ static void ggml_compute_forward_leaky_relu_f32(
}
static void ggml_compute_forward_leaky_relu(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -10139,7 +10067,7 @@ static void ggml_compute_forward_leaky_relu(
// ggml_compute_forward_silu_back
static void ggml_compute_forward_silu_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10151,10 +10079,6 @@ static void ggml_compute_forward_silu_back_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_are_same_shape(src0, grad));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int ith = params->ith;
const int nth = params->nth;
@@ -10186,7 +10110,7 @@ static void ggml_compute_forward_silu_back_f32(
}
static void ggml_compute_forward_silu_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10205,7 +10129,7 @@ static void ggml_compute_forward_silu_back(
static void ggml_compute_forward_hardswish_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10213,10 +10137,6 @@ static void ggml_compute_forward_hardswish_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -10230,9 +10150,12 @@ static void ggml_compute_forward_hardswish_f32(
}
}
static void ggml_compute_forward_hardswish(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -10248,7 +10171,7 @@ static void ggml_compute_forward_hardswish(
}
static void ggml_compute_forward_hardsigmoid_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10256,10 +10179,6 @@ static void ggml_compute_forward_hardsigmoid_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -10274,9 +10193,12 @@ static void ggml_compute_forward_hardsigmoid_f32(
}
static void ggml_compute_forward_hardsigmoid(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -10295,17 +10217,13 @@ static void ggml_compute_forward_hardsigmoid(
// ggml_compute_forward_norm
static void ggml_compute_forward_norm_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -10350,7 +10268,7 @@ static void ggml_compute_forward_norm_f32(
}
static void ggml_compute_forward_norm(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10370,17 +10288,13 @@ static void ggml_compute_forward_norm(
// ggml_compute_forward_group_rms_norm
static void ggml_compute_forward_rms_norm_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -10422,7 +10336,7 @@ static void ggml_compute_forward_rms_norm_f32(
}
static void ggml_compute_forward_rms_norm(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10440,7 +10354,7 @@ static void ggml_compute_forward_rms_norm(
}
static void ggml_compute_forward_rms_norm_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10448,10 +10362,6 @@ static void ggml_compute_forward_rms_norm_back_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -10599,7 +10509,7 @@ static void ggml_compute_forward_rms_norm_back_f32(
}
static void ggml_compute_forward_rms_norm_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10619,17 +10529,13 @@ static void ggml_compute_forward_rms_norm_back(
// ggml_compute_forward_group_norm
static void ggml_compute_forward_group_norm_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -10697,7 +10603,7 @@ static void ggml_compute_forward_group_norm_f32(
}
static void ggml_compute_forward_group_norm(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10750,7 +10656,7 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) {
#endif
static void ggml_compute_forward_mul_mat(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -10797,7 +10703,7 @@ static void ggml_compute_forward_mul_mat(
#if defined(GGML_USE_CLBLAST)
if (ggml_cl_can_mul_mat(src0, src1, dst)) {
- if (params->ith == 0 && params->type == GGML_TASK_TYPE_COMPUTE) {
+ if (!ith) {
ggml_cl_mul_mat(src0, src1, dst, params->wdata, params->wsize);
}
return;
@@ -10810,31 +10716,25 @@ static void ggml_compute_forward_mul_mat(
const size_t desired_wsize = ne13*ne12*ne_plane*sizeof(float);
UNUSED(desired_wsize);
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (type != GGML_TYPE_F32) {
- assert(params->wsize >= desired_wsize);
- // parallelize by src0 rows
- for (int64_t i13 = 0; i13 < ne13; i13++) {
- for (int64_t i12 = 0; i12 < ne12; i12++) {
- // broadcast src0 into src1 across 2nd,3rd dimension
- const int64_t i03 = i13/r3;
- const int64_t i02 = i12/r2;
-
- const void * x = (char *) src0->data + i02*nb02 + i03*nb03;
- float * const wdata = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;
- ggml_to_float_t const to_float = type_traits[type].to_float;
-
- for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
- to_float((const char *) x + i01*nb01, wdata + i01*ne00, ne00);
- }
+ if (type != GGML_TYPE_F32) {
+ assert(params->wsize >= desired_wsize);
+ // parallelize by src0 rows
+ for (int64_t i13 = 0; i13 < ne13; i13++) {
+ for (int64_t i12 = 0; i12 < ne12; i12++) {
+ // broadcast src0 into src1 across 2nd,3rd dimension
+ const int64_t i03 = i13/r3;
+ const int64_t i02 = i12/r2;
+
+ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;
+ float * const wdata = (float *) params->wdata + i13*ne12*ne_plane + i12*ne_plane;
+ ggml_to_float_t const to_float = type_traits[type].to_float;
+
+ for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
+ to_float((const char *) x + i01*nb01, wdata + i01*ne00, ne00);
}
}
}
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ ggml_syncthreads(params->barrier);
}
// perform sgemm, parallelization controlled by blas lib
@@ -10883,7 +10783,6 @@ static void ggml_compute_forward_mul_mat(
(char *)dst->data + i12*nb2 + i13*nb3,
nb1/ggml_type_size(dst->type),
ith, nth,
- params->type,
src0->type,
src1->type,
dst->type))
@@ -10893,32 +10792,25 @@ static void ggml_compute_forward_mul_mat(
UseGgmlGemm1:;
#endif
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
- if (src1->type != vec_dot_type) {
- char * wdata = params->wdata;
- const size_t row_size = ggml_row_size(vec_dot_type, ne10);
+ if (src1->type != vec_dot_type) {
+ char * wdata = params->wdata;
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
- assert(params->wsize >= ne11*ne12*ne13*row_size);
- GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ assert(params->wsize >= ne11*ne12*ne13*row_size);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
- for (int64_t i13 = 0; i13 < ne13; ++i13) {
- for (int64_t i12 = 0; i12 < ne12; ++i12) {
- for (int64_t i11 = 0; i11 < ne11; ++i11) {
+ unsigned chore = 0;
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
+ for (int64_t i11 = 0; i11 < ne11; ++i11) {
+ if (chore++ % nth == ith)
from_float_to_vec_dot((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), (void *) wdata, ne10);
- wdata += row_size;
- }
+ wdata += row_size;
}
}
}
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ ggml_syncthreads(params->barrier);
}
const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
@@ -10938,7 +10830,6 @@ UseGgmlGemm1:;
(char *)dst->data + i12*nb2 + i13*nb3,
nb1/ggml_type_size(dst->type),
ith, nth,
- params->type,
src0->type,
vec_dot_type,
dst->type))
@@ -11053,7 +10944,7 @@ UseGgmlGemm2:;
// ggml_compute_forward_mul_mat_id
static void ggml_compute_forward_mul_mat_id(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11102,10 +10993,7 @@ static void ggml_compute_forward_mul_mat_id(
int64_t * matrix_row_counts = (int64_t *) (wdata_src1_end); // [n_as]
struct mmid_row_mapping * matrix_rows = (struct mmid_row_mapping *)(matrix_row_counts + n_as); // [n_as][ne11]
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
+ if (!ith) {
char * wdata = params->wdata;
if (src1->type != vec_dot_type) {
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
@@ -11139,13 +11027,8 @@ static void ggml_compute_forward_mul_mat_id(
matrix_row_counts[i02] += 1;
}
}
-
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
+ }
+ ggml_syncthreads(params->barrier);
// compute each matrix multiplication in sequence
for (int cur_a = 0; cur_a < n_as; ++cur_a) {
@@ -11238,7 +11121,7 @@ static void ggml_compute_forward_mul_mat_id(
// ggml_compute_forward_out_prod
static void ggml_compute_forward_out_prod_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11280,26 +11163,19 @@ static void ggml_compute_forward_out_prod_f32(
(ggml_is_contiguous(src1) || ggml_is_transposed(src1));
#endif
- if (params->type == GGML_TASK_TYPE_INIT) {
+ if (!ith) {
#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) // gemm beta will zero dst
if (use_blas) {
return;
}
#endif
- if (ith != 0) {
- return;
- }
ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
}
+ ggml_syncthreads(params->barrier);
#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)
if (use_blas) {
- if (params->ith != 0) { // All threads other than the first do no work.
+ if (params->ith) { // All threads other than the first do no work.
return;
}
// Arguments to ggml_compute_forward_out_prod (expressed as major,minor)
@@ -11431,7 +11307,7 @@ static void ggml_compute_forward_out_prod_f32(
}
static void ggml_compute_forward_out_prod_q_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11472,17 +11348,11 @@ static void ggml_compute_forward_out_prod_q_f32(
// TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
+ if (!ith) {
ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);
return;
}
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
+ ggml_syncthreads(params->barrier);
// parallelize by last three dimensions
@@ -11545,7 +11415,7 @@ static void ggml_compute_forward_out_prod_q_f32(
}
static void ggml_compute_forward_out_prod(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11592,7 +11462,7 @@ static void ggml_compute_forward_out_prod(
// ggml_compute_forward_scale
static void ggml_compute_forward_scale_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11601,10 +11471,6 @@ static void ggml_compute_forward_scale_f32(
GGML_ASSERT(ggml_is_contiguous(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// scale factor
float v;
memcpy(&v, dst->op_params, sizeof(float));
@@ -11636,9 +11502,12 @@ static void ggml_compute_forward_scale_f32(
}
static void ggml_compute_forward_scale(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -11656,7 +11525,7 @@ static void ggml_compute_forward_scale(
// ggml_compute_forward_set
static void ggml_compute_forward_set_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -11673,20 +11542,16 @@ static void ggml_compute_forward_set_f32(
size_t offset = ((int32_t *) dst->op_params)[3];
bool inplace = (bool) ((int32_t *) dst->op_params)[4];
- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {
- if (params->ith != 0) {
- return;
+ if (!inplace) {
+ if (!params->ith) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
}
- // memcpy needs to be synchronized across threads to avoid race conditions.
- // => do it in INIT phase
- memcpy(
- ((char *) dst->data),
- ((char *) src0->data),
- ggml_nbytes(dst));
- }
-
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ ggml_syncthreads(params->barrier);
}
const int ith = params->ith;
@@ -11731,9 +11596,12 @@ static void ggml_compute_forward_set_f32(
}
static void ggml_compute_forward_set(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -11772,7 +11640,7 @@ static void ggml_compute_forward_set(
// ggml_compute_forward_cpy
static void ggml_compute_forward_cpy(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
ggml_compute_forward_dup(params, dst);
}
@@ -11780,7 +11648,7 @@ static void ggml_compute_forward_cpy(
// ggml_compute_forward_cont
static void ggml_compute_forward_cont(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
ggml_compute_forward_dup(params, dst);
}
@@ -11788,7 +11656,7 @@ static void ggml_compute_forward_cont(
// ggml_compute_forward_reshape
static void ggml_compute_forward_reshape(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
// NOP
UNUSED(params);
@@ -11798,7 +11666,7 @@ static void ggml_compute_forward_reshape(
// ggml_compute_forward_view
static void ggml_compute_forward_view(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const struct ggml_tensor * dst) {
// NOP
UNUSED(params);
@@ -11808,7 +11676,7 @@ static void ggml_compute_forward_view(
// ggml_compute_forward_permute
static void ggml_compute_forward_permute(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const struct ggml_tensor * dst) {
// NOP
UNUSED(params);
@@ -11818,7 +11686,7 @@ static void ggml_compute_forward_permute(
// ggml_compute_forward_transpose
static void ggml_compute_forward_transpose(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const struct ggml_tensor * dst) {
// NOP
UNUSED(params);
@@ -11828,16 +11696,12 @@ static void ggml_compute_forward_transpose(
// ggml_compute_forward_get_rows
static void ggml_compute_forward_get_rows_q(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_BINARY_OP_LOCALS
const int64_t nc = ne00;
@@ -11874,16 +11738,12 @@ static void ggml_compute_forward_get_rows_q(
}
static void ggml_compute_forward_get_rows_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_BINARY_OP_LOCALS
const int64_t nc = ne00;
@@ -11917,16 +11777,12 @@ static void ggml_compute_forward_get_rows_f16(
}
static void ggml_compute_forward_get_rows_bf16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_BINARY_OP_LOCALS
const int64_t nc = ne00;
@@ -11960,16 +11816,12 @@ static void ggml_compute_forward_get_rows_bf16(
}
static void ggml_compute_forward_get_rows_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_BINARY_OP_LOCALS
const int64_t nc = ne00;
@@ -12003,9 +11855,14 @@ static void ggml_compute_forward_get_rows_f32(
}
static void ggml_compute_forward_get_rows(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ // FIXME: the cost of launching additional threads decreases performance with GPU offloading
+ params->nth = MIN(params->nth, ggml_nelements(dst->src[1]));
+ if (params->ith >= params->nth)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -12073,7 +11930,7 @@ static void ggml_compute_forward_get_rows(
// ggml_compute_forward_get_rows_back
static void ggml_compute_forward_get_rows_back_f32_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12084,16 +11941,10 @@ static void ggml_compute_forward_get_rows_back_f32_f16(
// ggml_compute_forward_dup_same_cont(params, opt0, dst);
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (params->ith != 0) {
- return;
- }
+ if (!params->ith) {
memset(dst->data, 0, ggml_nbytes(dst));
}
-
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
+ ggml_syncthreads(params->barrier);
const int nc = src0->ne[0];
const int nr = ggml_nelements(src1);
@@ -12112,27 +11963,20 @@ static void ggml_compute_forward_get_rows_back_f32_f16(
}
static void ggml_compute_forward_get_rows_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- GGML_ASSERT(params->ith == 0);
GGML_ASSERT(ggml_is_contiguous(dst));
// ggml_compute_forward_dup_same_cont(params, opt0, dst);
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (params->ith != 0) {
- return;
- }
- memset(dst->data, 0, ggml_nbytes(dst));
- }
-
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
+ memset(dst->data, 0, ggml_nbytes(dst));
const int nc = src0->ne[0];
const int nr = ggml_nelements(src1);
@@ -12151,9 +11995,12 @@ static void ggml_compute_forward_get_rows_back_f32(
}
static void ggml_compute_forward_get_rows_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -12193,17 +12040,13 @@ static void ggml_compute_forward_get_rows_back(
// ggml_compute_forward_diag
static void ggml_compute_forward_diag_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// TODO: handle transposed/permuted matrices
GGML_TENSOR_UNARY_OP_LOCALS
@@ -12235,9 +12078,12 @@ static void ggml_compute_forward_diag_f32(
}
static void ggml_compute_forward_diag(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -12255,7 +12101,7 @@ static void ggml_compute_forward_diag(
// ggml_compute_forward_diag_mask_inf
static void ggml_compute_forward_diag_mask_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const float value) {
@@ -12269,22 +12115,18 @@ static void ggml_compute_forward_diag_mask_f32(
GGML_ASSERT(n_past >= 0);
- if (!inplace && (params->type == GGML_TASK_TYPE_INIT)) {
- if (ith != 0) {
- return;
+ if (!inplace) {
+ if (!ith) {
+ // memcpy needs to be synchronized across threads to avoid race conditions.
+ // => do it in INIT phase
+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
+ memcpy(
+ ((char *) dst->data),
+ ((char *) src0->data),
+ ggml_nbytes(dst));
}
- // memcpy needs to be synchronized across threads to avoid race conditions.
- // => do it in INIT phase
- GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
- GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
- memcpy(
- ((char *) dst->data),
- ((char *) src0->data),
- ggml_nbytes(dst));
- }
-
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ ggml_syncthreads(params->barrier);
}
// TODO: handle transposed/permuted matrices
@@ -12309,7 +12151,7 @@ static void ggml_compute_forward_diag_mask_f32(
}
static void ggml_compute_forward_diag_mask_inf(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12327,7 +12169,7 @@ static void ggml_compute_forward_diag_mask_inf(
}
static void ggml_compute_forward_diag_mask_zero(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12347,7 +12189,7 @@ static void ggml_compute_forward_diag_mask_zero(
// ggml_compute_forward_soft_max
static void ggml_compute_forward_soft_max_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12357,10 +12199,6 @@ static void ggml_compute_forward_soft_max_f32(
assert(ggml_is_contiguous(dst));
assert(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
float scale = 1.0f;
float max_bias = 0.0f;
@@ -12463,11 +12301,13 @@ static void ggml_compute_forward_soft_max_f32(
}
static void ggml_compute_forward_soft_max(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
+ params->nth = MIN(params->nth, ggml_nrows(src0));
+
switch (src0->type) {
case GGML_TYPE_F32:
{
@@ -12483,7 +12323,7 @@ static void ggml_compute_forward_soft_max(
// ggml_compute_forward_soft_max_back
static void ggml_compute_forward_soft_max_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12495,10 +12335,6 @@ static void ggml_compute_forward_soft_max_back_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
GGML_ASSERT(ggml_are_same_shape(src1, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// TODO: handle transposed/permuted matrices
const int ith = params->ith;
@@ -12562,7 +12398,7 @@ static void ggml_compute_forward_soft_max_back_f32(
}
static void ggml_compute_forward_soft_max_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12582,17 +12418,13 @@ static void ggml_compute_forward_soft_max_back(
// ggml_compute_forward_alibi
static void ggml_compute_forward_alibi_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
//const int n_past = ((int32_t *) dst->op_params)[0];
const int n_head = ((int32_t *) dst->op_params)[1];
float max_bias;
@@ -12641,17 +12473,13 @@ static void ggml_compute_forward_alibi_f32(
}
static void ggml_compute_forward_alibi_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
//const int n_past = ((int32_t *) dst->op_params)[0];
const int n_head = ((int32_t *) dst->op_params)[1];
float max_bias;
@@ -12703,9 +12531,12 @@ static void ggml_compute_forward_alibi_f16(
}
static void ggml_compute_forward_alibi(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return; // TODO: implement threading support
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -12753,17 +12584,13 @@ static void ggml_compute_forward_alibi(
// ggml_compute_forward_clamp
static void ggml_compute_forward_clamp_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
float min;
float max;
memcpy(&min, (float *) dst->op_params + 0, sizeof(float));
@@ -12795,9 +12622,12 @@ static void ggml_compute_forward_clamp_f32(
}
static void ggml_compute_forward_clamp(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return; // TODO: implement threading support
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -12898,17 +12728,13 @@ GGML_CALL void ggml_rope_yarn_corr_dims(
}
static void ggml_compute_forward_rope_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const bool forward) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
// these two only relevant for xPos RoPE:
@@ -13076,17 +12902,13 @@ static void ggml_compute_forward_rope_f32(
}
static void ggml_compute_forward_rope_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const bool forward) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
//const int n_past = ((int32_t *) dst->op_params)[0];
@@ -13243,7 +13065,7 @@ static void ggml_compute_forward_rope_f16(
}
static void ggml_compute_forward_rope(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13267,7 +13089,7 @@ static void ggml_compute_forward_rope(
// ggml_compute_forward_rope_back
static void ggml_compute_forward_rope_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13291,7 +13113,7 @@ static void ggml_compute_forward_rope_back(
// ggml_compute_forward_conv_transpose_1d
static void ggml_compute_forward_conv_transpose_1d_f16_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13314,10 +13136,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
+ if (!ith) {
memset(params->wdata, 0, params->wsize);
// permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
@@ -13350,13 +13169,8 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
// need to zero dst since we are accumulating into it
memset(dst->data, 0, ggml_nbytes(dst));
-
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
}
+ ggml_syncthreads(params->barrier);
const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
@@ -13390,7 +13204,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
}
static void ggml_compute_forward_conv_transpose_1d_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13413,10 +13227,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32(
GGML_ASSERT(nb00 == sizeof(float));
GGML_ASSERT(nb10 == sizeof(float));
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
+ if (!ith) {
memset(params->wdata, 0, params->wsize);
// prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
@@ -13449,13 +13260,8 @@ static void ggml_compute_forward_conv_transpose_1d_f32(
// need to zero dst since we are accumulating into it
memset(dst->data, 0, ggml_nbytes(dst));
-
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
}
+ ggml_syncthreads(params->barrier);
const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
@@ -13489,7 +13295,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32(
}
static void ggml_compute_forward_conv_transpose_1d(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13514,7 +13320,7 @@ static void ggml_compute_forward_conv_transpose_1d(
// src1: image [N, IC, IH, IW]
// dst: result [N, OH, OW, IC*KH*KW]
static void ggml_compute_forward_im2col_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13557,14 +13363,6 @@ static void ggml_compute_forward_im2col_f32(
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
- if (params->type == GGML_TASK_TYPE_INIT) {
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
{
float * const wdata = (float *) dst->data;
@@ -13602,7 +13400,7 @@ static void ggml_compute_forward_im2col_f32(
// src1: image [N, IC, IH, IW]
// dst: result [N, OH, OW, IC*KH*KW]
static void ggml_compute_forward_im2col_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13645,14 +13443,6 @@ static void ggml_compute_forward_im2col_f16(
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
- if (params->type == GGML_TASK_TYPE_INIT) {
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
{
ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
@@ -13686,7 +13476,7 @@ static void ggml_compute_forward_im2col_f16(
}
static void ggml_compute_forward_im2col(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
switch (dst->type) {
case GGML_TYPE_F16:
@@ -13708,7 +13498,7 @@ static void ggml_compute_forward_im2col(
// ggml_compute_forward_conv_transpose_2d
static void ggml_compute_forward_conv_transpose_2d(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -13731,10 +13521,7 @@ static void ggml_compute_forward_conv_transpose_2d(
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith != 0) {
- return;
- }
+ if (!ith) {
memset(params->wdata, 0, params->wsize);
// permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)
@@ -13769,13 +13556,8 @@ static void ggml_compute_forward_conv_transpose_2d(
}
memset(dst->data, 0, ggml_nbytes(dst));
-
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
}
+ ggml_syncthreads(params->barrier);
const int32_t stride = ggml_get_op_params_i32(dst, 0);
@@ -13815,7 +13597,7 @@ static void ggml_compute_forward_conv_transpose_2d(
// ggml_compute_forward_pool_1d_sk_p0
static void ggml_compute_forward_pool_1d_sk_p0(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const enum ggml_op_pool op,
const int k,
struct ggml_tensor * dst) {
@@ -13825,10 +13607,6 @@ static void ggml_compute_forward_pool_1d_sk_p0(
assert(src->type == GGML_TYPE_F32);
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const char * cdata = (const char *)src->data;
const char * const data_end = cdata + ggml_nbytes(src);
float * drow = (float *)dst->data;
@@ -13869,9 +13647,12 @@ static void ggml_compute_forward_pool_1d_sk_p0(
// ggml_compute_forward_pool_1d
static void ggml_compute_forward_pool_1d(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const int32_t * opts = (const int32_t *)dst->op_params;
enum ggml_op_pool op = opts[0];
const int k0 = opts[1];
@@ -13886,18 +13667,17 @@ static void ggml_compute_forward_pool_1d(
// ggml_compute_forward_pool_2d
static void ggml_compute_forward_pool_2d(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src = dst->src[0];
GGML_ASSERT(src->type == GGML_TYPE_F32);
GGML_ASSERT(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int32_t * opts = (const int32_t *)dst->op_params;
enum ggml_op_pool op = opts[0];
const int k0 = opts[1];
@@ -13962,15 +13742,11 @@ static void ggml_compute_forward_pool_2d(
// ggml_compute_forward_upscale
static void ggml_compute_forward_upscale_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -14002,7 +13778,7 @@ static void ggml_compute_forward_upscale_f32(
}
static void ggml_compute_forward_upscale(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -14022,15 +13798,11 @@ static void ggml_compute_forward_upscale(
// ggml_compute_forward_pad
static void ggml_compute_forward_pad_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(src0->nb[0] == sizeof(float));
GGML_ASSERT( dst->nb[0] == sizeof(float));
@@ -14063,7 +13835,7 @@ static void ggml_compute_forward_pad_f32(
}
static void ggml_compute_forward_pad(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -14084,13 +13856,9 @@ static void ggml_compute_forward_pad(
// ggml_compute_forward_arange
static void ggml_compute_forward_arange_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_ASSERT(dst->nb[0] == sizeof(float));
const int ith = params->ith;
@@ -14111,7 +13879,7 @@ static void ggml_compute_forward_arange_f32(
}
static void ggml_compute_forward_arange(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
switch (dst->type) {
case GGML_TYPE_F32:
@@ -14126,13 +13894,9 @@ static void ggml_compute_forward_arange(
}
static void ggml_compute_forward_timestep_embedding_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const struct ggml_tensor * src0 = dst->src[0];
GGML_ASSERT(src0->nb[0] == sizeof(float));
@@ -14163,7 +13927,7 @@ static void ggml_compute_forward_timestep_embedding_f32(
}
static void ggml_compute_forward_timestep_embedding(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -14183,15 +13947,11 @@ static void ggml_compute_forward_timestep_embedding(
// ggml_compute_forward_argsort
static void ggml_compute_forward_argsort_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_UNARY_OP_LOCALS
GGML_ASSERT(nb0 == sizeof(float));
@@ -14226,7 +13986,7 @@ static void ggml_compute_forward_argsort_f32(
}
static void ggml_compute_forward_argsort(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -14246,7 +14006,7 @@ static void ggml_compute_forward_argsort(
// ggml_compute_forward_flash_attn
static void ggml_compute_forward_flash_attn_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -14298,14 +14058,6 @@ static void ggml_compute_forward_flash_attn_f32(
GGML_ASSERT(nb1 <= nb2);
GGML_ASSERT(nb2 <= nb3);
- if (params->type == GGML_TASK_TYPE_INIT) {
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// parallelize by q rows using ggml_vec_dot_f32
// total rows in q
@@ -14438,7 +14190,7 @@ static void ggml_compute_forward_flash_attn_f32(
}
static void ggml_compute_forward_flash_attn_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -14490,14 +14242,6 @@ static void ggml_compute_forward_flash_attn_f16(
GGML_ASSERT(nb1 <= nb2);
GGML_ASSERT(nb2 <= nb3);
- if (params->type == GGML_TASK_TYPE_INIT) {
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// parallelize by q rows using ggml_vec_dot_f32
// total rows in q
@@ -14666,7 +14410,7 @@ static void ggml_compute_forward_flash_attn_f16(
}
static void ggml_compute_forward_flash_attn(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -14691,7 +14435,7 @@ static void ggml_compute_forward_flash_attn(
// ggml_compute_forward_flash_ff
static void ggml_compute_forward_flash_ff_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * a = dst->src[0]; // F16
@@ -14749,14 +14493,6 @@ static void ggml_compute_forward_flash_ff_f16(
GGML_ASSERT(nb1 <= nb2);
GGML_ASSERT(nb2 <= nb3);
- if (params->type == GGML_TASK_TYPE_INIT) {
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// parallelize by a rows using ggml_vec_dot_f32
// total rows in a
@@ -14826,7 +14562,7 @@ static void ggml_compute_forward_flash_ff_f16(
}
static void ggml_compute_forward_flash_ff(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * b0 = dst->src[1];
@@ -14850,7 +14586,7 @@ static void ggml_compute_forward_flash_ff(
// ggml_compute_forward_flash_attn_back
static void ggml_compute_forward_flash_attn_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -14908,16 +14644,10 @@ static void ggml_compute_forward_flash_attn_back_f32(
GGML_ASSERT(nb1 <= nb2);
GGML_ASSERT(nb2 <= nb3);
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith == 0) {
- memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);
- }
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
+ if (!ith) {
+ memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);
}
+ ggml_syncthreads(params->barrier);
const int64_t elem_q = ggml_nelements(q);
const int64_t elem_k = ggml_nelements(k);
@@ -15205,7 +14935,7 @@ static void ggml_compute_forward_flash_attn_back_f32(
}
static void ggml_compute_forward_flash_attn_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -15226,12 +14956,8 @@ static void ggml_compute_forward_flash_attn_back(
// ggml_compute_forward_ssm_conv
static void ggml_compute_forward_ssm_conv_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const struct ggml_tensor * src0 = dst->src[0]; // conv_state
const struct ggml_tensor * src1 = dst->src[1]; // x
const struct ggml_tensor * src2 = dst->src[2]; // conv1d.weight
@@ -15335,7 +15061,7 @@ static void ggml_compute_forward_ssm_conv_f32(
}
static void ggml_compute_forward_ssm_conv(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
switch (dst->src[0]->type) {
case GGML_TYPE_F32:
@@ -15352,12 +15078,8 @@ static void ggml_compute_forward_ssm_conv(
// ggml_compute_forward_ssm_scan
static void ggml_compute_forward_ssm_scan_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const struct ggml_tensor * src0 = dst->src[0]; // s
const struct ggml_tensor * src1 = dst->src[1]; // x
const struct ggml_tensor * src2 = dst->src[2]; // dt
@@ -15460,7 +15182,7 @@ static void ggml_compute_forward_ssm_scan_f32(
}
static void ggml_compute_forward_ssm_scan(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
switch (dst->src[0]->type) {
case GGML_TYPE_F32:
@@ -15477,15 +15199,11 @@ static void ggml_compute_forward_ssm_scan(
// ggml_compute_forward_win_part
static void ggml_compute_forward_win_part_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
@@ -15523,9 +15241,12 @@ static void ggml_compute_forward_win_part_f32(
}
static void ggml_compute_forward_win_part(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -15543,15 +15264,11 @@ static void ggml_compute_forward_win_part(
// ggml_compute_forward_win_unpart
static void ggml_compute_forward_win_unpart_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
@@ -15587,9 +15304,12 @@ static void ggml_compute_forward_win_unpart_f32(
}
static void ggml_compute_forward_win_unpart(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -15607,7 +15327,7 @@ static void ggml_compute_forward_win_unpart(
//gmml_compute_forward_unary
static void ggml_compute_forward_unary(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const enum ggml_unary_op op = ggml_get_unary_op(dst);
@@ -15671,15 +15391,11 @@ static void ggml_compute_forward_unary(
// ggml_compute_forward_get_rel_pos
static void ggml_compute_forward_get_rel_pos_f16(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
// ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322
GGML_TENSOR_UNARY_OP_LOCALS
@@ -15700,9 +15416,12 @@ static void ggml_compute_forward_get_rel_pos_f16(
}
static void ggml_compute_forward_get_rel_pos(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -15720,7 +15439,7 @@ static void ggml_compute_forward_get_rel_pos(
// ggml_compute_forward_add_rel_pos
static void ggml_compute_forward_add_rel_pos_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -15728,17 +15447,12 @@ static void ggml_compute_forward_add_rel_pos_f32(
const struct ggml_tensor * src2 = dst->src[2];
const bool inplace = (bool) ((int32_t *) dst->op_params)[0];
- if (!inplace && params->type == GGML_TASK_TYPE_INIT) {
- if (params->ith != 0) {
- return;
+ if (!inplace) {
+ if (!params->ith) {
+ memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));
}
- memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));
- return;
+ ggml_syncthreads(params->barrier);
}
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
int64_t t0 = ggml_perf_time_us();
UNUSED(t0);
@@ -15789,7 +15503,7 @@ static void ggml_compute_forward_add_rel_pos_f32(
}
static void ggml_compute_forward_add_rel_pos(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -15809,7 +15523,7 @@ static void ggml_compute_forward_add_rel_pos(
// ggml_compute_forward_map_unary
static void ggml_compute_forward_map_unary_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_unary_op_f32_t fun) {
@@ -15817,10 +15531,6 @@ static void ggml_compute_forward_map_unary_f32(
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -15835,10 +15545,13 @@ static void ggml_compute_forward_map_unary_f32(
}
static void ggml_compute_forward_map_unary(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_unary_op_f32_t fun) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -15856,7 +15569,7 @@ static void ggml_compute_forward_map_unary(
// ggml_compute_forward_map_binary
static void ggml_compute_forward_map_binary_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_binary_op_f32_t fun) {
@@ -15866,10 +15579,6 @@ static void ggml_compute_forward_map_binary_f32(
assert(params->ith == 0);
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const int n = ggml_nrows(src0);
const int nc = src0->ne[0];
@@ -15886,10 +15595,13 @@ static void ggml_compute_forward_map_binary_f32(
}
static void ggml_compute_forward_map_binary(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_binary_op_f32_t fun) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * src0 = dst->src[0];
switch (src0->type) {
@@ -15907,74 +15619,72 @@ static void ggml_compute_forward_map_binary(
// ggml_compute_forward_map_custom1
static void ggml_compute_forward_map_custom1_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_custom1_op_f32_t fun) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * a = dst->src[0];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
fun(dst, a);
}
// ggml_compute_forward_map_custom2
static void ggml_compute_forward_map_custom2_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_custom2_op_f32_t fun) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * a = dst->src[0];
const struct ggml_tensor * b = dst->src[1];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
fun(dst, a, b);
}
// ggml_compute_forward_map_custom3
static void ggml_compute_forward_map_custom3_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst,
const ggml_custom3_op_f32_t fun) {
+ if (params->ith)
+ return;
+
const struct ggml_tensor * a = dst->src[0];
const struct ggml_tensor * b = dst->src[1];
const struct ggml_tensor * c = dst->src[1];
assert(params->ith == 0);
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
fun(dst, a, b, c);
}
// ggml_compute_forward_map_custom1
static void ggml_compute_forward_map_custom1(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * a = dst->src[0];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
struct ggml_map_custom1_op_params p;
memcpy(&p, dst->op_params, sizeof(p));
+ if (p.n_tasks != GGML_N_TASKS_MAX) {
+ params->nth = MIN(p.n_tasks, params->nth);
+ if (params->ith >= params->nth)
+ return;
+ }
p.fun(dst, a, params->ith, params->nth, p.userdata);
}
@@ -15982,18 +15692,19 @@ static void ggml_compute_forward_map_custom1(
// ggml_compute_forward_map_custom2
static void ggml_compute_forward_map_custom2(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * a = dst->src[0];
const struct ggml_tensor * b = dst->src[1];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
struct ggml_map_custom2_op_params p;
memcpy(&p, dst->op_params, sizeof(p));
+ if (p.n_tasks != GGML_N_TASKS_MAX) {
+ params->nth = MIN(p.n_tasks, params->nth);
+ if (params->ith >= params->nth)
+ return;
+ }
p.fun(dst, a, b, params->ith, params->nth, p.userdata);
}
@@ -16001,19 +15712,20 @@ static void ggml_compute_forward_map_custom2(
// ggml_compute_forward_map_custom3
static void ggml_compute_forward_map_custom3(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * a = dst->src[0];
const struct ggml_tensor * b = dst->src[1];
const struct ggml_tensor * c = dst->src[2];
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
struct ggml_map_custom3_op_params p;
memcpy(&p, dst->op_params, sizeof(p));
+ if (p.n_tasks != GGML_N_TASKS_MAX) {
+ params->nth = MIN(p.n_tasks, params->nth);
+ if (params->ith >= params->nth)
+ return;
+ }
p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);
}
@@ -16021,7 +15733,7 @@ static void ggml_compute_forward_map_custom3(
// ggml_compute_forward_cross_entropy_loss
static void ggml_compute_forward_cross_entropy_loss_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -16043,21 +15755,10 @@ static void ggml_compute_forward_cross_entropy_loss_f32(
GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc));
- if (params->type == GGML_TASK_TYPE_INIT) {
- if (ith == 0) {
- memset(sums, 0, sizeof(float) * (nth + nth * nc));
- }
- return;
- }
-
- if (params->type == GGML_TASK_TYPE_FINALIZE) {
- if (ith == 0) {
- float * dp = (float *) dst->data;
- ggml_vec_sum_f32(nth, dp, sums);
- dp[0] *= -1.0f / (float) nr;
- }
- return;
+ if (!ith) {
+ memset(sums, 0, sizeof(float) * (nth + nth * nc));
}
+ ggml_syncthreads(params->barrier);
const double eps = 1e-9;
@@ -16125,11 +15826,17 @@ static void ggml_compute_forward_cross_entropy_loss_f32(
}
#endif
}
+ ggml_syncthreads(params->barrier);
+ if (!ith) {
+ float * dp = (float *) dst->data;
+ ggml_vec_sum_f32(nth, dp, sums);
+ dp[0] *= -1.0f / (float) nr;
+ }
}
static void ggml_compute_forward_cross_entropy_loss(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -16149,7 +15856,7 @@ static void ggml_compute_forward_cross_entropy_loss(
// ggml_compute_forward_cross_entropy_loss_back
static void ggml_compute_forward_cross_entropy_loss_back_f32(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -16165,10 +15872,6 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(
const int64_t ith = params->ith;
const int64_t nth = params->nth;
- if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
- return;
- }
-
const double eps = 1e-9;
// TODO: handle transposed/permuted matrices
@@ -16241,7 +15944,7 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(
}
static void ggml_compute_forward_cross_entropy_loss_back(
- const struct ggml_compute_params * params,
+ struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -18091,9 +17794,7 @@ struct ggml_compute_state_shared {
const int n_threads;
// synchronization primitives
- atomic_int n_active; // num active threads
- atomic_int node_n; // active graph node
- atomic_int node_task; // active graph node task phase
+ struct ggml_barrier barrier;
ggml_abort_callback abort_callback; // abort ggml_graph_compute when true
void * abort_callback_data;
@@ -18115,425 +17816,29 @@ static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const
node->perf_time_us += time_us_cur;
}
-static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_threads) {
- int n_tasks = 0;
-
- if (ggml_is_empty(node)) {
- // no need to multi-thread a no-op
- n_tasks = 1;
- return n_tasks;
- }
-
- switch (node->op) {
- case GGML_OP_CPY:
- case GGML_OP_DUP:
- case GGML_OP_ADD:
- case GGML_OP_ADD1:
- case GGML_OP_ACC:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_SUB:
- case GGML_OP_SQR:
- case GGML_OP_SQRT:
- case GGML_OP_LOG:
- case GGML_OP_SUM:
- case GGML_OP_SUM_ROWS:
- case GGML_OP_MEAN:
- case GGML_OP_ARGMAX:
- case GGML_OP_REPEAT:
- case GGML_OP_REPEAT_BACK:
- case GGML_OP_LEAKY_RELU:
- {
- n_tasks = 1;
- } break;
- case GGML_OP_UNARY:
- switch (ggml_get_unary_op(node)) {
- case GGML_UNARY_OP_ABS:
- case GGML_UNARY_OP_SGN:
- case GGML_UNARY_OP_NEG:
- case GGML_UNARY_OP_STEP:
- case GGML_UNARY_OP_TANH:
- case GGML_UNARY_OP_ELU:
- case GGML_UNARY_OP_RELU:
- case GGML_UNARY_OP_HARDSWISH: // to opt for multiple threads
- case GGML_UNARY_OP_HARDSIGMOID: // to opt for multiple threads
- {
- n_tasks = 1;
- } break;
-
- case GGML_UNARY_OP_GELU:
- case GGML_UNARY_OP_GELU_QUICK:
- case GGML_UNARY_OP_SILU:
- {
- n_tasks = n_threads;
- } break;
- default:
- GGML_ASSERT(false);
- }
- break;
- case GGML_OP_SILU_BACK:
- case GGML_OP_MUL:
- case GGML_OP_DIV:
- case GGML_OP_NORM:
- case GGML_OP_RMS_NORM:
- case GGML_OP_RMS_NORM_BACK:
- case GGML_OP_GROUP_NORM:
- case GGML_OP_CONCAT:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_MUL_MAT:
- {
- n_tasks = n_threads;
-
- // TODO: use different scheduling for different matrix sizes
- //const int nr0 = ggml_nrows(node->src[0]);
- //const int nr1 = ggml_nrows(node->src[1]);
-
- //n_tasks = MIN(n_threads, MAX(1, nr0/128));
- //printf("nr0 = %8d, nr1 = %8d, nr0*nr1 = %8d, n_tasks%d\n", nr0, nr1, nr0*nr1, n_tasks);
- } break;
- case GGML_OP_MUL_MAT_ID:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_OUT_PROD:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_GET_ROWS:
- {
- // FIXME: the cost of launching additional threads decreases performance with GPU offloading
- //n_tasks = MIN(n_threads, ggml_nelements(node->src[1]));
- n_tasks = MIN(n_cur_threads, ggml_nelements(node->src[1]));
- } break;
- case GGML_OP_SCALE:
- case GGML_OP_SET:
- case GGML_OP_CONT:
- case GGML_OP_RESHAPE:
- case GGML_OP_VIEW:
- case GGML_OP_PERMUTE:
- case GGML_OP_TRANSPOSE:
- case GGML_OP_GET_ROWS_BACK:
- case GGML_OP_DIAG:
- {
- n_tasks = 1;
- } break;
- case GGML_OP_DIAG_MASK_ZERO:
- case GGML_OP_DIAG_MASK_INF:
- case GGML_OP_SOFT_MAX_BACK:
- case GGML_OP_ROPE:
- case GGML_OP_ROPE_BACK:
- case GGML_OP_ADD_REL_POS:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_ALIBI:
- {
- n_tasks = 1; //TODO
- } break;
- case GGML_OP_CLAMP:
- {
- n_tasks = 1; //TODO
- } break;
- case GGML_OP_SOFT_MAX:
- {
- n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
- } break;
- case GGML_OP_CONV_TRANSPOSE_1D:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_IM2COL:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_CONV_TRANSPOSE_2D:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_POOL_1D:
- case GGML_OP_POOL_2D:
- {
- n_tasks = 1;
- } break;
- case GGML_OP_UPSCALE:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_PAD:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_ARANGE:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_TIMESTEP_EMBEDDING:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_ARGSORT:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_FLASH_ATTN:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_FLASH_FF:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_FLASH_ATTN_BACK:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_SSM_CONV:
- case GGML_OP_SSM_SCAN:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_WIN_PART:
- case GGML_OP_WIN_UNPART:
- case GGML_OP_GET_REL_POS:
- case GGML_OP_MAP_UNARY:
- case GGML_OP_MAP_BINARY:
- case GGML_OP_MAP_CUSTOM1_F32:
- case GGML_OP_MAP_CUSTOM2_F32:
- case GGML_OP_MAP_CUSTOM3_F32:
- {
- n_tasks = 1;
- } break;
- case GGML_OP_MAP_CUSTOM1:
- {
- struct ggml_map_custom1_op_params p;
- memcpy(&p, node->op_params, sizeof(p));
- if (p.n_tasks == GGML_N_TASKS_MAX) {
- n_tasks = n_threads;
- } else {
- n_tasks = MIN(p.n_tasks, n_threads);
- }
- } break;
- case GGML_OP_MAP_CUSTOM2:
- {
- struct ggml_map_custom2_op_params p;
- memcpy(&p, node->op_params, sizeof(p));
- if (p.n_tasks == GGML_N_TASKS_MAX) {
- n_tasks = n_threads;
- } else {
- n_tasks = MIN(p.n_tasks, n_threads);
- }
- } break;
- case GGML_OP_MAP_CUSTOM3:
- {
- struct ggml_map_custom3_op_params p;
- memcpy(&p, node->op_params, sizeof(p));
- if (p.n_tasks == GGML_N_TASKS_MAX) {
- n_tasks = n_threads;
- } else {
- n_tasks = MIN(p.n_tasks, n_threads);
- }
- } break;
- case GGML_OP_CROSS_ENTROPY_LOSS:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
- {
- n_tasks = n_threads;
- } break;
- case GGML_OP_NONE:
- {
- n_tasks = 1;
- } break;
- case GGML_OP_COUNT:
- {
- GGML_ASSERT(false);
- } break;
- default:
- {
- fprintf(stderr, "%s: op not implemented: ", __func__);
- if (node->op < GGML_OP_COUNT) {
- fprintf(stderr, "%s\n", ggml_op_name(node->op));
- } else {
- fprintf(stderr, "%d\n", node->op);
- }
- GGML_ASSERT(false);
- } break;
- }
-
- assert(n_tasks > 0);
-
- return n_tasks;
-}
-
-static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_compute_state * state, const bool do_yield) {
- // wait for other threads to finish
- const int last_node_n = * node_n;
-
- while (true) {
- if (do_yield) {
- sched_yield();
- }
-
- * node_n = atomic_load(&state->shared->node_n);
- if (* node_n != last_node_n) break;
- }
-}
-
-static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_compute_state * state, const bool do_yield) {
- // wait for other threads to finish
- const int last_task_phase = * task_phase;
-
- while (true) {
- if (do_yield) {
- sched_yield();
- }
-
- * task_phase = atomic_load(&state->shared->node_task);
- if (* task_phase != last_task_phase) break;
- }
-}
-
static thread_ret_t ggml_graph_compute_thread(void * data) {
struct ggml_compute_state * state = (struct ggml_compute_state *) data;
const struct ggml_cgraph * cgraph = state->shared->cgraph;
const struct ggml_cplan * cplan = state->shared->cplan;
-
- const int n_threads = state->shared->n_threads;
+ struct ggml_barrier * b = &state->shared->barrier;
set_numa_thread_affinity(state->ith);
- int node_n = -1;
- int task_phase = GGML_TASK_TYPE_FINALIZE;
-
- while (true) {
+ for (int node_n = 0; node_n < cgraph->n_nodes; ++node_n) {
if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {
- state->shared->node_n += 1;
state->ec = GGML_STATUS_ABORTED;
return 0;
}
-
- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {
- // all other threads are finished and spinning
- // do finalize and init here so we don't have synchronize again
- struct ggml_compute_params params = {
- /*.type =*/ GGML_TASK_TYPE_FINALIZE,
- /*.ith =*/ 0,
- /*.nth =*/ 0,
- /*.wsize =*/ cplan->work_size,
- /*.wdata =*/ cplan->work_data,
- };
-
- if (node_n != -1) {
- /* FINALIZE */
- struct ggml_tensor * node = cgraph->nodes[node_n];
- if (GGML_OP_HAS_FINALIZE[node->op]) {
- params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);
- ggml_compute_forward(&params, node);
- }
- ggml_graph_compute_perf_stats_node(node, state->shared);
- }
-
- // distribute new work or execute it direct if 1T
- while (++node_n < cgraph->n_nodes) {
- GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, node_n, cgraph->n_nodes);
- struct ggml_tensor * node = cgraph->nodes[node_n];
- const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);
-
- state->shared->perf_node_start_cycles = ggml_perf_cycles();
- state->shared->perf_node_start_time_us = ggml_perf_time_us();
-
- params.nth = n_tasks;
-
- if (n_tasks == 1) {
- /* INIT */
- if (GGML_OP_HAS_INIT[node->op]) {
- params.type = GGML_TASK_TYPE_INIT;
- ggml_compute_forward(&params, node);
- }
-
- // TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1,
- // they do something more efficient than spinning (?)
- params.type = GGML_TASK_TYPE_COMPUTE;
- ggml_compute_forward(&params, node);
-
- if (GGML_OP_HAS_FINALIZE[node->op]) {
- params.type = GGML_TASK_TYPE_FINALIZE;
- ggml_compute_forward(&params, node);
- }
-
- ggml_graph_compute_perf_stats_node(node, state->shared);
- } else {
- break;
- }
-
- if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {
- break;
- }
- }
-
- task_phase = GGML_TASK_TYPE_INIT;
- atomic_store(&state->shared->n_active, n_threads);
- atomic_store(&state->shared->node_n, node_n);
- atomic_store(&state->shared->node_task, task_phase);
- } else {
- ggml_graph_compute_thread_sync_node(&node_n, state, false);
- ggml_graph_compute_thread_sync_task(&task_phase, state, false);
- }
-
- // check if we should stop
- if (node_n >= cgraph->n_nodes) break;
-
- /* INIT & COMPUTE */
- struct ggml_tensor * node = cgraph->nodes[node_n];
- const int n_tasks = ggml_get_n_tasks(node, n_threads, state->shared->n_threads);
-
struct ggml_compute_params params = {
- /*.type =*/ GGML_TASK_TYPE_INIT,
- /*.ith =*/ state->ith,
- /*.nth =*/ n_tasks,
- /*.wsize =*/ cplan->work_size,
- /*.wdata =*/ cplan->work_data,
+ /*.ith =*/ state->ith,
+ /*.nth =*/ state->shared->n_threads,
+ /*.wsize =*/ cplan->work_size,
+ /*.wdata =*/ cplan->work_data,
+ /*.barrier =*/ &state->shared->barrier,
};
-
- if (state->ith < n_tasks) {
- if (GGML_OP_HAS_INIT[node->op]) {
- ggml_compute_forward(&params, node);
- }
- }
-
- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {
- task_phase = GGML_TASK_TYPE_COMPUTE;
- atomic_store(&state->shared->n_active, n_threads);
- atomic_store(&state->shared->node_task, task_phase);
- }
- else {
- // TODO: this sched_yield can have significant impact on the performance - either positive or negative
- // depending on the workload and the operating system.
- // since it is not clear what is the best approach, it should potentially become user-configurable
- // ref: https://github.com/ggerganov/ggml/issues/291
- // UPD: adding the do_yield flag seems to resolve the issue universally
- const bool do_yield = node_n < 0 || cgraph->nodes[node_n]->op == GGML_OP_MUL_MAT;
- ggml_graph_compute_thread_sync_task(&task_phase, state, do_yield);
- }
-
- if (state->ith < n_tasks) {
- params.type = GGML_TASK_TYPE_COMPUTE;
- ggml_compute_forward(&params, node);
- }
-
- if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) {
- task_phase = GGML_TASK_TYPE_FINALIZE;
- atomic_store(&state->shared->n_active, n_threads);
- atomic_store(&state->shared->node_task, task_phase);
- }
- else {
- ggml_graph_compute_thread_sync_task(&task_phase, state, false);
- }
+ ggml_compute_forward(&params, cgraph->nodes[node_n]);
+ ggml_syncthreads(&state->shared->barrier);
}
return 0;
@@ -18554,9 +17859,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
// thread scheduling for the different operations + work buffer size estimation
for (int i = 0; i < cgraph->n_nodes; i++) {
struct ggml_tensor * node = cgraph->nodes[i];
-
- const int n_tasks = ggml_get_n_tasks(node, n_threads, 1);
-
+ const int n_tasks = n_threads;
max_tasks = MAX(max_tasks, n_tasks);
size_t cur = 0;
@@ -18738,7 +18041,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
work_size += CACHE_LINE_SIZE*(n_threads - 1);
}
- cplan.n_threads = MIN(max_tasks, n_threads);
+ cplan.n_threads = n_threads;
cplan.work_size = work_size;
cplan.work_data = NULL;
@@ -18763,9 +18066,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
/*.perf_node_start_cycles =*/ 0,
/*.perf_node_start_time_us =*/ 0,
/*.n_threads =*/ n_threads,
- /*.n_active =*/ n_threads,
- /*.node_n =*/ -1,
- /*.node_task =*/ GGML_TASK_TYPE_FINALIZE,
+ /*.barrier =*/ { n_threads },
/*.abort_callback =*/ NULL,
/*.abort_callback_data =*/ NULL,
};
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index d857c58c..2fef0d93 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -666,16 +666,9 @@ extern "C" {
// compute types
- // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled.
- // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995.
- enum ggml_task_type {
- GGML_TASK_TYPE_INIT = 0,
- GGML_TASK_TYPE_COMPUTE,
- GGML_TASK_TYPE_FINALIZE,
- };
+ struct ggml_barrier;
struct ggml_compute_params {
- enum ggml_task_type type;
// ith = thread index, nth = number of threads
int ith, nth;
@@ -683,8 +676,12 @@ extern "C" {
// work buffer for all threads
size_t wsize;
void * wdata;
+
+ struct ggml_barrier *barrier;
};
+ GGML_API void ggml_syncthreads(struct ggml_barrier *);
+
// numa strategies
enum ggml_numa_strategy {
GGML_NUMA_STRATEGY_DISABLED = 0,
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index e429e53c..0f749ef8 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -49,7 +49,7 @@ int llamafile_gpu_parse(const char *);
const char *llamafile_describe_gpu(void);
bool llamafile_sgemm(long, long, long, const void *, long, const void *, long, void *, long, int,
- int, int, int, int, int);
+ int, int, int, int);
struct ggml_tensor;
struct ggml_compute_params;
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 26d9d82e..f5ef0fef 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -107,16 +107,14 @@ static const struct GemmFuncs {
* @param ldc is row stride of `C`
* @param ith is thread id (must be less than `nth`)
* @param nth is number of threads (must be greater than zero)
- * @param task is GGML task type
* @param Atype is GGML data type of `A`
* @param Btype is GGML data type of `B`
* @param Ctype is GGML data type of `C`
* @return true if this function was able to service the matmul request
*/
bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
- int Ctype) {
- return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype, Ctype);
+ void *C, long ldc, int ith, int nth, int Atype, int Btype, int Ctype) {
+ return funcs.sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, Atype, Btype, Ctype);
}
/**
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index 5bda13fe..af583485 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -8,23 +8,23 @@ struct ggml_tensor;
struct ggml_compute_params;
bool llamafile_sgemm_unsupported(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int);
bool llamafile_sgemm_amd_avx(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int);
bool llamafile_sgemm_amd_fma(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int);
bool llamafile_sgemm_amd_avx2(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int);
bool llamafile_sgemm_amd_avxvnni(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int);
bool llamafile_sgemm_amd_avx512f(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int);
bool llamafile_sgemm_amd_zen4(long, long, long, const void *, long, const void *, long, void *,
- long, int, int, int, int, int, int);
+ long, int, int, int, int, int);
bool llamafile_sgemm_arm80(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int);
bool llamafile_sgemm_arm82(long, long, long, const void *, long, const void *, long, void *, long,
- int, int, int, int, int, int);
+ int, int, int, int, int);
bool llamafile_mixmul_unsupported(const struct ggml_compute_params *, const struct ggml_tensor *,
const struct ggml_tensor *, const struct ggml_tensor *,
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
index 3825989c..f0041c09 100644
--- a/llamafile/sgemm_sss_test.cpp
+++ b/llamafile/sgemm_sss_test.cpp
@@ -46,8 +46,8 @@ int main(int argc, char *argv[]) {
randomize(k, n, B, ldb);
BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TYPE_F32, GGML_TYPE_F32,
+ GGML_TYPE_F32));
for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 283973ad..b153f48d 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -45,8 +45,8 @@ int main(int argc, char *argv[]) {
randomize(k, n, B, ldb);
BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
+ BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TYPE_F32, GGML_TYPE_F32,
+ GGML_TYPE_F32));
double err_sum = 0;
long long err_worst = 0;
diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.inc
index 5263d598..1b02f486 100644
--- a/llamafile/tinyblas_cpu.inc
+++ b/llamafile/tinyblas_cpu.inc
@@ -41,6 +41,7 @@
#include "llama.cpp/ggml-impl.h"
#include "llama.cpp/ggml-quants.h"
#include "llamafile.h"
+#include "sgemm.h"
#include <cosmo.h>
#pragma GCC diagnostic ignored "-Wpedantic"
@@ -350,6 +351,10 @@ template <>
inline __m512bh load(const ggml_bf16_t *p) {
return (__m512bh)_mm512_loadu_ps((const float *)p);
}
+template <>
+inline __m512bh load(const float *p) {
+ return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));
+}
#endif // __AVX512BF16__
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -363,9 +368,8 @@ class tinyBLAS {
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
- void matmul(long m, long n, int task) {
- if (task == GGML_TASK_TYPE_COMPUTE)
- mnpack(0, m, 0, n);
+ void matmul(long m, long n) {
+ mnpack(0, m, 0, n);
}
private:
@@ -571,9 +575,8 @@ class tinyBLAS_Q0_ARM {
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
- void matmul(long m, long n, int task) {
- if (task == GGML_TASK_TYPE_COMPUTE)
- mnpack(0, m, 0, n);
+ void matmul(long m, long n) {
+ mnpack(0, m, 0, n);
}
private:
@@ -704,9 +707,8 @@ class tinyBLAS_Q0_AVX2 {
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
}
- void matmul(long m, long n, int task) {
- if (task == GGML_TASK_TYPE_COMPUTE)
- mnpack(0, m, 0, n);
+ void matmul(long m, long n) {
+ mnpack(0, m, 0, n);
}
private:
@@ -895,8 +897,7 @@ class tinyBLAS_Q0_AVX2 {
*
* For example, for single-threaded single-precision GEMM you can say
*
- * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,
- * 0, 1, GGML_TASK_TYPE_COMPUTE,
+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1,
* GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);
*
* @param m is rows in `A` and `C`
@@ -910,15 +911,13 @@ class tinyBLAS_Q0_AVX2 {
* @param ldc is row stride of `C`
* @param ith is thread id (must be less than `nth`)
* @param nth is number of threads (must be greater than zero)
- * @param task is GGML task type
* @param Atype is GGML data type of `A`
* @param Btype is GGML data type of `B`
* @param Ctype is GGML data type of `C`
* @return true if this function was able to service the matmul request
*/
bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
- void *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
- int Ctype) {
+ void *C, long ldc, int ith, int nth, int Atype, int Btype, int Ctype) {
assert(m >= 0);
assert(n >= 0);
@@ -942,14 +941,14 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{
k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__AVX__) || defined(__AVX2__)
if (k % 8)
return false;
tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{
k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__ARM_NEON)
if (n < 4)
@@ -958,7 +957,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{
k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#else
return false;
@@ -969,20 +968,24 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
#if defined(__AVX512BF16__)
if (k % 32)
return false;
+ if (Btype == GGML_TYPE_F32 && m * n < 500000) {
+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, float, float> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+ }
if (Btype != GGML_TYPE_BF16)
return false;
tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{
k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__AVX512F__)
if (k % 16)
return false;
- if (Btype != GGML_TYPE_F32)
- return false;
tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__AVX2__)
if (k % 8)
@@ -991,7 +994,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
if (k % 4)
@@ -1000,7 +1003,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#else
return false;
@@ -1011,27 +1014,41 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
#if defined(__AVX512F__)
if (k % 16)
return false;
- if (Btype != GGML_TYPE_F32)
+ if (Btype == GGML_TYPE_F32 && m * n < 500000) {
+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+ }
+ if (Btype != GGML_TYPE_F16)
return false;
- tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n);
return true;
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
if (X86_HAVE(F16C)) {
if (k % 8)
return false;
- if (Btype != GGML_TYPE_F32)
+ if (0 && Btype == GGML_TYPE_F32 && m * n < 500000) {
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith,
+ nth};
+ tb.matmul(m, n);
+ return true;
+ }
+ if (Btype != GGML_TYPE_F16)
return false;
- tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
- k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith,
+ nth};
+ tb.matmul(m, n);
return true;
} else {
return false;
}
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
- if (n < 8)
+ if (n < 2)
return false;
if (k % 8)
return false;
@@ -1039,7 +1056,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{
k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
if (k % 4)
@@ -1048,7 +1065,7 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
return false;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#else
return false;
@@ -1061,12 +1078,12 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
#if defined(__AVX2__) || defined(__AVX512F__)
tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{
k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__ARM_FEATURE_DOTPROD)
tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, float> tb{
k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#else
return false;
@@ -1079,12 +1096,12 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
#if defined(__AVX2__) || defined(__AVX512F__)
tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{
k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#elif defined(__ARM_FEATURE_DOTPROD)
tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, float> tb{
k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
- tb.matmul(m, n, task);
+ tb.matmul(m, n);
return true;
#else
return false;
@@ -1106,7 +1123,6 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
(void)ldc;
(void)ith;
(void)nth;
- (void)task;
(void)Atype;
(void)Btype;
(void)Ctype;
@@ -1352,31 +1368,25 @@ class MixMul {
bool mixmat() {
if (cols % KN)
return false;
- switch (params->type) {
- case GGML_TASK_TYPE_INIT:
- if (thought->type != ggml_type_trait<TB>::id)
- quantize_thought(ggml_type_trait<TB>::id);
- build_row_pointers(ggml_type_trait<TB>::id);
- return true;
- case GGML_TASK_TYPE_COMPUTE:
- assert(!(cols % BS));
- assert(!(weights->nb[1] % sizeof(TA)));
- for (int expert = 0; expert < experts; ++expert) {
- BLAS tb{cols / BS,
- (const TA *)((const char *)weights->data + expert * weights->nb[2]),
- (long)(weights->nb[1] / sizeof(TA)),
- (const TB *)(rowptr_thought_ + expert * tokens * thinkers),
- 0,
- (TC *)(rowptr_result_ + expert * tokens * thinkers),
- 0,
- params->ith,
- params->nth};
- tb.matmul(rows, rowptr_count_[expert], GGML_TASK_TYPE_COMPUTE);
- }
- return true;
- default:
- return true;
+ if (thought->type != ggml_type_trait<TB>::id)
+ quantize_thought(ggml_type_trait<TB>::id);
+ build_row_pointers(ggml_type_trait<TB>::id);
+ ggml_syncthreads(params->barrier);
+ assert(!(cols % BS));
+ assert(!(weights->nb[1] % sizeof(TA)));
+ for (int expert = 0; expert < experts; ++expert) {
+ BLAS tb{cols / BS,
+ (const TA *)((const char *)weights->data + expert * weights->nb[2]),
+ (long)(weights->nb[1] / sizeof(TA)),
+ (const TB *)(rowptr_thought_ + expert * tokens * thinkers),
+ 0,
+ (TC *)(rowptr_result_ + expert * tokens * thinkers),
+ 0,
+ params->ith,
+ params->nth};
+ tb.matmul(rows, rowptr_count_[expert]);
}
+ return true;
}
void build_row_pointers(ggml_type vec_dot_type) {