LLM inference in C/C++
C++ ggml
[SYCL]: fix memory errors in mul_mat, split buffer, host pool (#29889)
Michael Taylora30fb4924dcf
diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cppindex 894006949d..f01d0ff955 100644--- a/ggml/src/ggml-sycl/common.cpp+++ b/ggml/src/ggml-sycl/common.cpp@@ -115,7 +115,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t#elseze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0};#endif- ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr);+ // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms+ ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr);if (r == ZE_RESULT_SUCCESS && ptr) {ggml_sycl_memtrace_add(type, ptr, size);return ptr;diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cppindex ef7413abd8..8e1f1d50a9 100644--- a/ggml/src/ggml-sycl/cpy.cpp+++ b/ggml/src/ggml-sycl/cpy.cpp@@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, coscope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0));const int64_t ne = ggml_nelements(src0);GGML_ASSERT(ne == ggml_nelements(src1));+ // the cpy kernels index in 32-bit+ GGML_ASSERT(ne <= INT32_MAX);GGML_TENSOR_BINARY_OP_LOCALS01;diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex 1cfc841264..b380cbedfd 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -899,6 +899,10 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) {static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst,const void *ptr_src, size_t size) {char *host_buf = (char *)malloc(size);+ if (host_buf == nullptr) {+ GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size);+ GGML_ABORT("fatal error");+ }q_src.memcpy(host_buf, (const char *)ptr_src, size).wait();q_dst.memcpy((char *)ptr_dst, host_buf, size).wait();free(host_buf);@@ -1177,12 +1181,13 @@ bool is_bmg_g31_arch(int device) {}static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {- size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size();+ ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;+ int device = ctx->device;+ // use the buft device, the current device may differ on multi-GPU systems+ size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size();if (g_ggml_sycl_host_pinned_mem_2g) {return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024);} else {- ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;- int device = ctx->device;if(is_bmg_g31_arch(device)) {//Todo, it's workaround for BMG-G31, which has a known issue with large allocations.//The max alloc size is reduced to 60% of the reported max alloc size.@@ -1398,7 +1403,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};ctx->tensor_extras.push_back(extra);- ctx->streams.push_back(&(dpct::get_current_device().default_queue()));+ // consumers index the queues by device id, so size the vector once with one queue per device+ if (ctx->streams.empty()) {+ for (int i = 0; i < ggml_sycl_info().device_count; ++i) {+ ctx->streams.push_back(&(dpct::get_device(i).default_queue()));+ }+ }for (int i = 0; i < ggml_sycl_info().device_count; ++i) {int64_t row_low, row_high;@@ -2089,8 +2099,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {queue_ptr qptr;int device;- inline static int counter{ 0 };-struct ggml_sycl_buffer {void * ptr = nullptr;size_t size = 0;@@ -2100,6 +2108,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {static constexpr int MAX_POOL_SIZE{ 64 };std::vector<ggml_sycl_buffer> buffer_pool = std::vector<ggml_sycl_buffer>(MAX_POOL_SIZE);size_t pool_size = 0;+ int counter = 0;explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}@@ -2113,41 +2122,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {b.size = 0;}}- counter = 0;}void * alloc(size_t size, size_t * actual_size) override {if (counter == MAX_POOL_SIZE) {- ggml_sycl_buffer b = buffer_pool[0];- void * ptr = b.ptr;- *actual_size = b.size;- counter = 1;- return ptr;- }- ggml_sycl_buffer & b = buffer_pool[counter];-- if (b.ptr == nullptr) {- void * ptr;-+ // the pool wrapped around, no slot is reserved for this allocation+ void * ptr = nullptr;SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));if (!ptr) {GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);return nullptr;}- pool_size += size;+ pool_size += size;*actual_size = size;- counter = counter + 1;+ counter = 0;return ptr;- } else {- ++counter;- b.size = size;- return b.ptr;}+ ggml_sycl_buffer & b = buffer_pool[counter];+ ++counter;++ if (b.ptr != nullptr) {+ if (b.size >= size) {+ // hand out the cached buffer, free() puts it back into the pool+ void * ptr = b.ptr;+ *actual_size = b.size;+ b.ptr = nullptr;+ b.size = 0;+ return ptr;+ }+ // the cached buffer is too small, replace it+ SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));+ pool_size -= b.size;+ b.ptr = nullptr;+ b.size = 0;+ }++ void * ptr;+ SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));+ if (!ptr) {+ GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);+ return nullptr;+ }+ pool_size += size;+ *actual_size = size;+ return ptr;}void free(void * ptr, size_t size) override {// if the pool is not completed add the pointer to it in place of the first nullptr found.- // Otherwise do nothing, pointers will be freed once the pool is deallocated.+ // Otherwise free the buffer, all slots are cached.for (int i = 0; i < MAX_POOL_SIZE; ++i) {ggml_sycl_buffer & b = buffer_pool[i];if (b.ptr == nullptr) {@@ -2156,6 +2179,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {return;}}+ SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));+ pool_size -= size;}};@@ -3154,8 +3179,8 @@ inline void ggml_sycl_op_mul_mat_sycl(to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream);}const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16- ? (const sycl::half *)src1->data + src1_padded_row_size- : src1_as_f16.get();+ ? (const sycl::half *)src1_ddf_i+ : src1_as_f16.get();ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());if (src0->type != GGML_TYPE_F16) {@@ -3252,6 +3277,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst)float * dst_dd = static_cast<float *>(dst->data);const int64_t ne = ggml_nelements(dst->src[0]);+ // the kernel indexes in 32-bit+ GGML_ASSERT(ne <= INT32_MAX);sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream);}@@ -3266,6 +3293,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor *const int64_t ncols = dst->src[0]->ne[0];const int64_t nrows = ggml_nrows(dst->src[0]);+ // the kernel indexes in 32-bit+ GGML_ASSERT(ncols * nrows <= INT32_MAX);sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);}@@ -3282,6 +3311,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dstconst int64_t ncols = dst->src[0]->ne[0];const int64_t nrows = ggml_nrows(dst->src[0]);+ // the kernel indexes in 32-bit+ GGML_ASSERT(ncols * nrows <= INT32_MAX);sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);@@ -3351,6 +3382,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dconst int64_t ncols = dst->src[0]->ne[0];const int64_t nrows = ggml_nrows(dst->src[0]);+ // the kernel indexes in 32-bit+ GGML_ASSERT(ncols * nrows <= INT32_MAX);argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);}@@ -3366,6 +3399,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_tenconst int64_t ne00 = dst->src[0]->ne[0];const int64_t ne01 = dst->src[0]->ne[1];const int nrows0 = ggml_nrows(dst->src[0]);+ // the kernel indexes in 32-bit+ GGML_ASSERT(ne00 * nrows0 <= INT32_MAX);const int n_past = ((int32_t *) dst->op_params)[0];@@ -3444,6 +3479,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * dsmemcpy(&scale, (float *) dst->op_params + 0, sizeof(float));memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));+ // the kernel indexes in 32-bit+ GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX);+scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream);/*DPCT1010:87: SYCL uses exceptions to report errors and does not use the@@ -3524,7 +3562,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tenconst size_t q8_1_bs = QK8_1;ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;const bool src0_is_contiguous = ggml_is_contiguous(src0);const bool src1_is_contiguous = ggml_is_contiguous(src1);@@ -3606,7 +3643,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tenqueue_ptr stream = ctx.stream(i, 0);if (src0_is_contiguous) {- dev[i].src0_dd = (char *) src0->data;+ // a split tensor keeps its rows in the per-device slices of the tensor extra+ dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data;} else {dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0));}@@ -3702,7 +3740,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensrc1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs).wait()));} else {- const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] ++ // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf+ const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf +(i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type);SYCL_CHECK(@@ -5269,6 +5308,7 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg(int) ne11,/*stride_col_y=*/src1_padded_cols / QK8_1,/*stride_col_dst=*/(int) glu->ne[0], stream);+ GGML_UNUSED(gate);}// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.@@ -6860,7 +6900,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, conscase GGML_GLU_OP_GEGLU_ERF:case GGML_GLU_OP_GEGLU_QUICK:case GGML_GLU_OP_SWIGLU_CLAMP:- return ggml_is_contiguous_1(op->src[0]);+ // the fused GLU kernels only have f32/f16 instantiations+ return ggml_is_contiguous_1(op->src[0]) &&+ (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);default:return false;}@@ -7123,7 +7165,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, conscase GGML_OP_RMS_NORM_BACK:return ggml_is_contiguous(op->src[0]);case GGML_OP_SCALE:- return true;+ // the scale kernel is f32 only+ return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;case GGML_OP_CONT:return true;case GGML_OP_TRI:@@ -7156,12 +7199,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons#endif) &&op->src[0]->type == op->type;- case GGML_OP_CONV_3D:+ case GGML_OP_CONV_3D: {+ const int32_t * opts = (const int32_t *) op->op_params;+ // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0)+ // and MKL BLAS rejects it; let those fall back to the CPU backend+ const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9];+ const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10];return op->type == GGML_TYPE_F32 &&(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&op->src[1]->type == GGML_TYPE_F32 &&ggml_is_contiguous(op->src[0]) &&- ggml_is_contiguous(op->src[1]);+ ggml_is_contiguous(op->src[1]) &&+ knl_n_total > 0 && patch_total > 0;+ }case GGML_OP_SUM:case GGML_OP_SUM_ROWS:case GGML_OP_MEAN:diff --git a/ggml/src/ggml-sycl/roll.cpp b/ggml/src/ggml-sycl/roll.cppindex 1e05181789..a95cee9a97 100644--- a/ggml/src/ggml-sycl/roll.cpp+++ b/ggml/src/ggml-sycl/roll.cpp@@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {const ggml_tensor *src = dst->src[0];GGML_ASSERT(src && src->type == GGML_TYPE_F32);+ // the kernel indexes in 32-bit+ GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX);const int ne0 = (int) dst->ne[0];const int ne1 = (int) dst->ne[1];const int ne2 = (int) dst->ne[2];