LLM inference in C/C++
C++ ggml
vulkan : add command-buffer debug labels for GPU profilers (#28101)
Cordell Blanchard72797e89198a
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 235b35c6ce..cefe186fee 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -844,6 +844,8 @@ struct vk_device_struct {std::mutex compile_mutex;std::condition_variable compile_cv;+ uint32_t debug_cmdbuf_idx {};+vk::PhysicalDevice physical_device;vk::PhysicalDeviceProperties properties;std::string name;@@ -2209,6 +2211,8 @@ struct vk_context_struct {std::vector<vk_staging_memcpy> out_memcpys;std::vector<vk_staging_memset> memsets;+ std::vector<std::string> debug_labels;+vk_command_pool * p {};};typedef std::shared_ptr<vk_context_struct> vk_context;@@ -8332,6 +8336,95 @@ template <typename T, uint32_t N> const T *push_constant_data(const std::array<Treturn t.data();}+static void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) {+ vk::DebugUtilsLabelEXT label = {};+ label.pLabelName = name;+ label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};+ vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));+}++// no-op unless GGML_VK_DEBUG_MARKERS is set+struct ggml_vk_debug_label {+ // at most one of these is set, depending on the scope the label was opened in+ vk_context_struct * subctx {};+ vk_queue_handle * qhandle {};++ // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)".+ // RGP cannot recover the pipeline name on its own, it only has the hash+ ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {+ return;+ }+ begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")");+ }++ // one region per graph node+ // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19"+ ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {+ return;+ }+ std::string name = ggml_op_name(cgraph->nodes[node_idx]->op);+ for (int i = 1; i <= n_fused; i++) {+ name += "+";+ name += ggml_op_name(cgraph->nodes[node_idx + i]->op);+ }+ name += " ";+ name += cgraph->nodes[node_idx]->name;+ begin(ctx, name);+ }++ // one region per graph evaluation, opened on the queue instead of a command buffer+ // so it spans every submit the evaluation makes+ ggml_vk_debug_label(vk_queue_handle * handle, const char * name) {+ if (!vk_instance.debug_utils_support || handle == nullptr) {+ return;+ }+ vk::DebugUtilsLabelEXT label = {};+ label.pLabelName = name;+ label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};++ qhandle = handle;+ std::lock_guard<vk_queue_handle> guard(*qhandle);+ vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));+ }++ // call before the command buffer can end, the destructor covers the rest+ void close() {+ if (subctx != nullptr) {+ // close on the current command buffer, which may differ from the one begin used+ if (subctx->s != nullptr) {+ vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf);+ }+ subctx->debug_labels.pop_back();+ subctx = nullptr;+ }+ if (qhandle != nullptr) {+ std::lock_guard<vk_queue_handle> guard(*qhandle);+ vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue);+ qhandle = nullptr;+ }+ }++ ~ggml_vk_debug_label() {+ close();+ }++ ggml_vk_debug_label(const ggml_vk_debug_label &) = delete;+ ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete;++private:+ // the constructors check this too, so the name is not built when markers are off+ void begin(vk_context & ctx, const std::string & name) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {+ return;+ }+ subctx = ctx.get();+ subctx->debug_labels.push_back(name);+ ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str());+ }+};+template <typename T>static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);@@ -8361,7 +8454,10 @@ static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context&0,{ descriptor_set },{});- subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);+ {+ ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2);+ subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);+ }}static void ggml_vk_ctx_end(vk_context& ctx) {@@ -8370,6 +8466,15 @@ static void ggml_vk_ctx_end(vk_context& ctx) {return;}+ // close open labels so this buffer is balanced; reopened in ggml_vk_ctx_begin+ if (vk_instance.debug_utils_support) {+ for (size_t i = 0; i < ctx->debug_labels.size(); i++) {+ vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf);+ }+ // the enclosing per-command-buffer region+ vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(ctx->s->buffer->buf);+ }+ctx->s->buffer->buf.end();ctx->s = nullptr;}@@ -8382,6 +8487,17 @@ static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {subctx->seqs.push_back({ ggml_vk_begin_submission(device, *subctx->p) });subctx->s = subctx->seqs[subctx->seqs.size() - 1].data();++ if (vk_instance.debug_utils_support) {+ // outermost region, one per command buffer, so the gaps between submits stand out+ const std::string name = "submit " + std::to_string(device->debug_cmdbuf_idx++);+ ggml_vk_cmd_label_begin(subctx->s->buffer->buf, name.c_str());++ // reopen labels left open when the previous command buffer was submitted+ for (const std::string & label : subctx->debug_labels) {+ ggml_vk_cmd_label_begin(subctx->s->buffer->buf, label.c_str());+ }+ }}static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {@@ -15971,6 +16087,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr}}+ // closed explicitly below, and by the destructor on the early returns+ ggml_vk_debug_label dbg(compute_ctx, cgraph, node_idx, ctx->num_additional_fused_ops);+switch (node->op) {case GGML_OP_REPEAT:ggml_vk_repeat(ctx, compute_ctx, src0, node);@@ -16375,6 +16494,9 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgrreturn false;}+ // the submit path below can end the command buffer, so close the region first+ dbg.close();+ctx->tensor_ctxs[node_idx] = compute_ctx;#if defined(GGML_VULKAN_CHECK_RESULTS)@@ -17841,14 +17963,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgctx->device->diag_prev_end = -1;if (vk_instance.debug_utils_support) {- vk::DebugUtilsLabelEXT dul = {};- dul.pLabelName = "ggml_backend_vk_graph_compute";- dul.color = std::array<float,4>{1.0f, 1.0f, 1.0f, 1.0f};-- std::lock_guard<vk_queue_handle> guard(*ctx->device->compute_queue->handle);- vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(ctx->device->compute_queue->handle->queue, reinterpret_cast<VkDebugUtilsLabelEXT*>(&dul));+ ctx->device->debug_cmdbuf_idx = 0;}+ // queue scope, so it encloses every submit this evaluation makes.+ // closed when the function returns+ ggml_vk_debug_label queue_dbg(ctx->device->compute_queue->handle.get(), "ggml_backend_vk_graph_compute");+ctx->prealloc_size_add_rms_partials_offset = 0;ctx->do_add_rms_partials = false;ctx->do_add_rms_partials_offset_calculation = false;