LLM inference in C/C++
C++ ggml
vulkan: fix submission batching size, add debug tools for diagnosing causes of DeviceLost drivers errors (#26371)
Ruben Ortlam803b7fcae893
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 2418ba6d17..a923755f9e 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -186,13 +186,22 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256-#define VK_CHECK(err, msg) \+#define VK_CHECK(err, msg, dev) \do { \- vk::Result err_ = (err); \+ vk::Result err_; \+ try { \+ err_ = (err); \+ } catch (vk::DeviceLostError &) { \+ ggml_vk_print_device_lost_info(dev); \+ GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \+ #err, __FILE__, __LINE__); \+ throw; \+ } \if (err_ != vk::Result::eSuccess) { \- fprintf(stderr, "ggml_vulkan: %s error %s at %s:%d\n", \+ GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \#err, to_string(err_).c_str(), __FILE__, __LINE__); \- exit(1); \+ throw vk::SystemError(vk::make_error_code(err_), \+ "ggml_vulkan: " msg); \} \} while (0)@@ -302,9 +311,13 @@ struct vk_command_pool {}};+static void ggml_vk_print_device_fault_info(const vk_device& device);+static void ggml_vk_print_device_lost_info(const vk_device& device);+// Prevent simultaneous submissions to the same queue.struct vk_queue_handle {vk::Queue queue;+ vk_device_ref device;virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;virtual void lock() {} // no-op by default (internally synchronized case)virtual void unlock() {}@@ -315,7 +328,14 @@ struct vk_queue_handle_synchronized : vk_queue_handle {std::mutex mutex;void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {std::lock_guard<std::mutex> guard(mutex);- queue.submit(submits, fence);+ try {+ queue.submit(submits, fence);+ } catch (vk::DeviceLostError &) {+ if (auto dev = device.lock()) {+ ggml_vk_print_device_lost_info(dev);+ }+ throw;+ }}void lock() override { mutex.lock(); }void unlock() override { mutex.unlock(); }@@ -324,7 +344,14 @@ struct vk_queue_handle_synchronized : vk_queue_handle {struct vk_queue_handle_unsynchronized : vk_queue_handle {void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {// Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues- queue.submit(submits, fence);+ try {+ queue.submit(submits, fence);+ } catch (vk::DeviceLostError &) {+ if (auto dev = device.lock()) {+ ggml_vk_print_device_lost_info(dev);+ }+ throw;+ }}// lock()/unlock() inherited no-ops};@@ -835,6 +862,15 @@ struct vk_device_struct {bool pipeline_executable_properties_support {};+ bool device_fault {};+ PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};++ bool serialize_submissions {};++ const ggml_cgraph * diag_cgraph {};+ int diag_prev_start = -1;+ int diag_prev_end = -1;+size_t idx;bool mul_mat_l[GGML_TYPE_COUNT];@@ -1118,6 +1154,57 @@ void vk_command_pool::destroy(vk::Device& device) {cmd_buffers.clear();}+static void ggml_vk_print_device_fault_info(const vk_device& device) {+ if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {+ return;+ }++ VkDeviceFaultCountsEXT fault_counts {};+ fault_counts.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT;+ VkResult res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, nullptr);+ if (res != VK_SUCCESS) {+ GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (counts) failed: %d\n", res);+ return;+ }++ std::vector<VkDeviceFaultAddressInfoEXT> address_infos(fault_counts.addressInfoCount);+ std::vector<VkDeviceFaultVendorInfoEXT> vendor_infos(fault_counts.vendorInfoCount);++ VkDeviceFaultInfoEXT fault_info {};+ fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT;+ fault_info.pAddressInfos = address_infos.data();+ fault_info.pVendorInfos = vendor_infos.data();++ res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info);+ if (res != VK_SUCCESS) {+ GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);+ return;+ }++ if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {+ return;+ }++ if (fault_info.description[0] != '\0') {+ GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);+ }++ for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {+ const auto& info = address_infos[i];+ GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",+ i, (int)info.addressType,+ (unsigned long long)info.reportedAddress,+ (unsigned long long)info.addressPrecision);+ }+ for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {+ const auto& info = vendor_infos[i];+ GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",+ i, info.description,+ (unsigned long long)info.vendorFaultCode,+ (unsigned long long)info.vendorFaultData);+ }+}+struct vk_buffer_struct {vk::Buffer buffer = VK_NULL_HANDLE;vk::DeviceMemory device_memory = VK_NULL_HANDLE;@@ -2059,6 +2146,36 @@ static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {return 0;}+static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {+ uint64_t total_flops = 0;+ int n_ops = 0;+ for (int j = start; j <= end && j < cgraph->n_nodes; j++) {+ uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]);+ total_flops += flops;+ n_ops++;+ if (flops > 0) {+ GGML_LOG_CONT(" node %d: %s (%s) [%.2f GFLOP]\n",+ j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op),+ flops / 1e9);+ } else {+ GGML_LOG_CONT(" node %d: %s (%s)\n",+ j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op));+ }+ }+ GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);+}++static void ggml_vk_print_device_lost_info(const vk_device& device) {+ ggml_vk_print_device_fault_info(device);+ if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",+ device->name.c_str(), device->diag_prev_start, device->diag_prev_end);+ ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end);+ } else {+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());+ }+}+class vk_perf_logger {public:void print_timings(bool force = false) {@@ -2471,17 +2588,27 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {// Use waitForFences while most of the graph executes. Hopefully the CPU can sleep// during this wait.if (ctx->almost_ready_fence_pending) {- VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence");+ VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device);ctx->device->device.resetFences({ ctx->almost_ready_fence });ctx->almost_ready_fence_pending = false;}// Spin (w/pause) waiting for the graph to finish executing.vk::Result result;- while ((result = ctx->device->device.getFenceStatus(ctx->fence)) != vk::Result::eSuccess) {+ for (;;) {+ try {+ result = ctx->device->device.getFenceStatus(ctx->fence);+ } catch (vk::DeviceLostError &) {+ ggml_vk_print_device_lost_info(ctx->device);+ GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__);+ throw;+ }+ if (result == vk::Result::eSuccess) {+ break;+ }if (result != vk::Result::eNotReady) {- fprintf(stderr, "ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);- exit(1);+ GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);+ throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus");}for (uint32_t i = 0; i < 100; ++i) {YIELD();@@ -3172,6 +3299,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_}h->queue = device->device.getQueue2(queue_info2);+ h->device = device;q->handle = h;q->cmd_pool.init(device, q.get());@@ -6117,6 +6245,8 @@ static vk_device ggml_vk_get_device(size_t idx) {#endif} else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) {internally_sync_support = true;+ } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) {+ device->device_fault = true;}}@@ -6471,8 +6601,18 @@ static vk_device ggml_vk_get_device(size_t idx) {}#endif+ VkPhysicalDeviceFaultFeaturesEXT fault_features {};+ fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT;+ if (device->device_fault) {+ last_struct->pNext = (VkBaseOutStructure *)&fault_features;+ last_struct = (VkBaseOutStructure *)&fault_features;+ device_extensions.push_back("VK_EXT_device_fault");+ }+vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);+ device->device_fault = device->device_fault && fault_features.deviceFault;+device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues;// Build queue create infos only after querying whether internally synchronized queues are enabled.@@ -6771,6 +6911,11 @@ static vk_device ggml_vk_get_device(size_t idx) {device_create_info.setPNext(&device_features2);device->device = device->physical_device.createDevice(device_create_info);+ if (device->device_fault) {+ device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)+ vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT");+ }+// Queuesdevice->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false);@@ -6893,6 +7038,8 @@ static vk_device ggml_vk_get_device(size_t idx) {device->idx = idx;+ device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;+device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;device->add_rms_fusion = !device->disable_fusion &&@@ -8319,7 +8466,7 @@ static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void *}ggml_vk_submit(subctx, dst->device->fence);- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences");+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);dst->device->device.resetFences({ dst->device->fence });ggml_vk_queue_command_pools_cleanup(dst->device);}@@ -8431,7 +8578,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, siggml_vk_ctx_end(subctx);ggml_vk_submit(subctx, src->device->fence);VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),- "vk_buffer_read_2d uma waitForFences");+ "vk_buffer_read_2d uma waitForFences", src->device);src->device->device.resetFences({ src->device->fence });ggml_vk_queue_command_pools_cleanup(src->device);@@ -8452,7 +8599,7 @@ static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, siggml_vk_ctx_end(subctx);ggml_vk_submit(subctx, src->device->fence);- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences");+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);src->device->device.resetFences({ src->device->fence });ggml_vk_queue_command_pools_cleanup(src->device);@@ -8487,7 +8634,7 @@ static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& srggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);ggml_vk_ctx_end(subctx);ggml_vk_submit(subctx, src->device->fence);- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences");+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);src->device->device.resetFences({ src->device->fence });ggml_vk_queue_command_pools_cleanup(src->device);} else {@@ -8531,7 +8678,7 @@ static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, sizggml_vk_ctx_end(subctx);ggml_vk_submit(subctx, dst->device->fence);- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences");+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);dst->device->device.resetFences({ dst->device->fence });ggml_vk_queue_command_pools_cleanup(dst->device);}@@ -14266,7 +14413,7 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_tauto begin = std::chrono::high_resolution_clock::now();ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences");+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);ctx->device->device.resetFences({ ctx->fence });ggml_vk_queue_command_pools_cleanup(ctx->device);@@ -14468,7 +14615,7 @@ static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_auto begin = std::chrono::high_resolution_clock::now();ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);ctx->device->device.resetFences({ ctx->fence });ggml_vk_queue_command_pools_cleanup(ctx->device);@@ -14754,7 +14901,7 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m,auto begin = std::chrono::high_resolution_clock::now();ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);ctx->device->device.resetFences({ ctx->fence });ggml_vk_queue_command_pools_cleanup(ctx->device);@@ -15553,7 +15700,9 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph *memset(mset.dst, mset.val, mset.n);}- if (almost_ready && !ctx->almost_ready_fence_pending) {+ if (ctx->device->serialize_submissions) {+ ggml_vk_submit(subctx, ctx->fence);+ } else if (almost_ready && !ctx->almost_ready_fence_pending) {ggml_vk_submit(subctx, ctx->almost_ready_fence);ctx->almost_ready_fence_pending = true;} else {@@ -16164,12 +16313,20 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {memcpy(cpy.dst, cpy.src, cpy.n);}- ggml_vk_submit(compute_ctx, {});+ if (ctx->device->serialize_submissions) {+ ggml_vk_submit(compute_ctx, ctx->fence);+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "synchronize waitForFences", ctx->device);+ ctx->device->device.resetFences({ ctx->fence });+ } else {+ ggml_vk_submit(compute_ctx, {});+ }ctx->submit_pending = true;}if (ctx->submit_pending) {- if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {+ if (ctx->device->serialize_submissions) {+ ctx->submit_pending = false;+ } else if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {vk::TimelineSemaphoreSubmitInfo tl_info{1, &ctx->transfer_semaphore.value,0, nullptr,@@ -16186,7 +16343,9 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {} else {ctx->device->compute_queue->handle->submit({}, ctx->fence);}- ggml_vk_wait_for_fence(ctx);+ if (!ctx->device->serialize_submissions) {+ ggml_vk_wait_for_fence(ctx);+ }ctx->submit_pending = false;if (cmd_buf) {cmd_buf->in_use = false;@@ -16758,6 +16917,10 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgVK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;+ ctx->device->diag_cgraph = nullptr;+ ctx->device->diag_prev_start = -1;+ ctx->device->diag_prev_end = -1;+if (vk_instance.debug_utils_support) {vk::DebugUtilsLabelEXT dul = {};dul.pLabelName = "ggml_backend_vk_graph_compute";@@ -16849,6 +17012,36 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg}uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);+ auto const submit_after = [&](int start, int end) {+ if (ctx->device->serialize_submissions) {+ try {+ auto res = ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX);+ if (res != vk::Result::eSuccess) {+ GGML_LOG_ERROR("ggml_vulkan: waitForFences error during serialized submission\n");+ throw vk::SystemError(vk::make_error_code(res), "ggml_vulkan: waitForFences during serialized submission");+ }+ } catch (vk::DeviceLostError &) {+ ggml_vk_print_device_fault_info(ctx->device);+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s waiting for submission (nodes %d to %d):\n",+ ctx->device->name.c_str(), start, end);+ ggml_vk_print_node_list(cgraph, start, end);+ throw;+ }+ ctx->device->device.resetFences({ ctx->fence });+ ctx->submit_pending = false;+ ctx->device->diag_cgraph = cgraph;+ ctx->device->diag_prev_start = start;+ ctx->device->diag_prev_end = end;+ }+ first_node_in_batch = true;+ submitted_nodes = 0;+ batch_flops = 0;+ if (submit_count < 3) {+ flops_per_submit *= 2;+ }+ submit_count++;+ };+for (int i = 0; i < cgraph->n_nodes; i++) {if (first_node_in_batch) {submit_node_idx = i;@@ -16856,8 +17049,20 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg{auto node_flops = ggml_vk_get_node_flops(cgraph->nodes[i]);- batch_flops += node_flops;total_flops += node_flops;++ // Flush the current batch before recording a node that would push it over the flop threshold+ if (flops_per_submit != 0 && submitted_nodes > 0 && batch_flops + node_flops >= flops_per_submit) {+ vk_context flush_ctx = ggml_vk_get_compute_ctx(ctx);+ ggml_vk_ctx_end(flush_ctx);+ flush_ctx->exit_tensor_idx = -1;+ ctx->compute_ctx.reset();+ ggml_vk_compute_forward(ctx, cgraph, cgraph->nodes[submit_node_idx], submit_node_idx, false);+ submit_after(submit_node_idx, i - 1);+ submit_node_idx = i;+ }++ batch_flops += node_flops;}// op_srcs_fused_elementwise indicates whether an op's srcs all contribute to@@ -17111,13 +17316,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg}if (submit && enqueued) {- first_node_in_batch = true;- submitted_nodes = 0;- batch_flops = 0;- if (submit_count < 3) {- flops_per_submit *= 2;- }- submit_count++;+ submit_after(submit_node_idx, i + (int)ctx->num_additional_fused_ops);}i += ctx->num_additional_fused_ops;ctx->num_additional_fused_ops = 0;@@ -17133,13 +17332,13 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgggml_vk_ctx_end(compute_ctx);ggml_vk_submit(compute_ctx, ctx->device->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences");+ VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences", ctx->device);ctx->device->device.resetFences({ ctx->device->fence });ctx->compute_ctx.reset();// Get the results and pass them to the loggerstd::vector<uint64_t> timestamps(cgraph->n_nodes + 1);- VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results");+ VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results", ctx->device);if (!vk_perf_logger_concurrent) {// Log each op separatelyfor (int i = 1; i < ctx->query_idx; i++) {@@ -18366,7 +18565,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggmvk::Semaphore sem = vkev->tl_semaphore.s;uint64_t val = vkev->tl_semaphore.value;vk::SemaphoreWaitInfo swi{vk::SemaphoreWaitFlags{}, sem, val};- VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize");+ VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize", device);// Reset and move submitted eventsfor (auto& event : vkev->events_submitted) {