LLM inference in C/C++
C++ ggml
vulkan: split buffers and debug code into separate files, add shared headers (#28732)
Ruben Ortlamf172be756ae6
diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txtindex e733ad5cc9..af951a2bd5 100644--- a/ggml/src/ggml-vulkan/CMakeLists.txt+++ b/ggml/src/ggml-vulkan/CMakeLists.txt@@ -62,6 +62,11 @@ if (Vulkan_FOUND)ggml_add_backend_library(ggml-vulkanggml-vulkan.cpp../../include/ggml-vulkan.h+ ggml-vulkan-types.h+ ggml-vulkan-push-constants.h+ ggml-vulkan-common.h+ ggml-vulkan-buffers.cpp+ ggml-vulkan-debug.cpp)set(VULKAN_SHADER_GEN_CMAKE_ARGS "")diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cppnew file mode 100644index 0000000000..4d4c849513--- /dev/null+++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp@@ -0,0 +1,783 @@+#include "ggml-vulkan-common.h"++ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {+ /* .get_name = */ ggml_backend_vk_buffer_type_name,+ /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,+ /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,+ /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,+ /* .is_host = */ NULL,+};++static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {+ std::vector<uint32_t> indices;++ for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {+ vk::MemoryType memory_type = mem_props->memoryTypes[i];+ if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&+ (flags & memory_type.propertyFlags) == flags &&+ mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {+ indices.push_back(i);+ }+ }+ return indices;+}++static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list,+ void *import_ptr = nullptr) {+ VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");+ if (size > device->max_buffer_size) {+ throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit");+ }++ vk_buffer buf = std::make_shared<vk_buffer_struct>();++ if (size == 0) {+ buf->size = 0;+ return buf;+ }++ vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst;+ vk::MemoryAllocateFlags mem_flags {};+ if (device->buffer_device_address) {+ usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress;+ mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress;+ }++ vk::BufferCreateInfo buffer_create_info{+ vk::BufferCreateFlags(),+ size,+ usage_flags,+ vk::SharingMode::eExclusive,+ 0,+ nullptr,+ };++ vk::ExternalMemoryBufferCreateInfo external_memory_bci;+ if (import_ptr) {+ external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;+ buffer_create_info.setPNext(&external_memory_bci);+ }++ buf->buffer = device->device.createBuffer(buffer_create_info);++ vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);++ vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();++ const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f };++ vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags };++ if (device->memory_priority) {+ mem_flags_info.setPNext(&mem_priority_info);+ }++ if (import_ptr) {+ vk::MemoryHostPointerPropertiesEXT host_pointer_props;+ try {+ host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr);+ } catch (vk::SystemError& e) {+ GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what());+ device->device.destroyBuffer(buf->buffer);+ return {};+ }+ vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();++ uint32_t memory_type_idx;+ vk::MemoryPropertyFlags property_flags = *req_flags_list.begin();+ for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) {+ if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) {+ continue;+ }+ if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) {+ continue;+ }++ vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx];+ // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed+ if ((memory_type.propertyFlags & property_flags) == property_flags) {+ property_flags = memory_type.propertyFlags;+ break;+ }+ }+ if (memory_type_idx == 32) {+ GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n");+ device->device.destroyBuffer(buf->buffer);+ return {};+ }++ buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags;+ try {+ vk::ImportMemoryHostPointerInfoEXT import_info;+ import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;+ import_info.pHostPointer = import_ptr;+ import_info.setPNext(&mem_flags_info);+ buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info });+ } catch (const vk::SystemError& e) {+ }+ } else {+ for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) {+ const auto & req_flags = *it;++ const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags);++ if (memory_type_indices.empty()) {+ continue;+ }++ bool done = false;++ for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) {+ try {+ buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info });+ buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags;+ done = true;+ break;+ } catch (const vk::SystemError& e) {+ // loop and retry+ // during last attempt throw the exception+ if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) {+ device->device.destroyBuffer(buf->buffer);+ throw e;+ }+ }+ }++ if (done) {+ break;+ }+ }+ }++ if (!buf->device_memory) {+ device->device.destroyBuffer(buf->buffer);+ throw vk::OutOfDeviceMemoryError("No suitable memory type found");+ }++ buf->ptr = nullptr;++ if (import_ptr) {+ buf->ptr = import_ptr;+ } else {+ if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {+ buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE);+ }+ }++ device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);++ buf->device = device;+ buf->size = size;++ if (device->buffer_device_address) {+ const vk::BufferDeviceAddressInfo addressInfo(buf->buffer);+ buf->bda_addr = device->device.getBufferAddress(addressInfo);+ }++ device->memory_logger->log_allocation(buf, size);++ return buf;+}++vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags) {+ try {+ return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});+ } catch (const vk::SystemError& e) {+ std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;+ std::cerr << "ggml_vulkan: " << e.what() << std::endl;+ throw e;+ }+}++vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {+ vk_buffer buf;+ try {+ if (device->prefer_host_memory) {+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,+ vk::MemoryPropertyFlagBits::eDeviceLocal});+ } else if (device->uma) {+ // On UMA, prefer host-visible memory so direct tensor borrowing works.+ // If unavailable, fall back to device-local memory.+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,+ vk::MemoryPropertyFlagBits::eDeviceLocal,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});+ } else if (device->disable_host_visible_vidmem) {+ if (device->allow_sysmem_fallback) {+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});+ } else {+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ }+ } else {+ // use rebar if available, otherwise fallback to device only visible memory+ if (device->allow_sysmem_fallback) {+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,+ vk::MemoryPropertyFlagBits::eDeviceLocal,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});+ } else {+ buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,+ vk::MemoryPropertyFlagBits::eDeviceLocal});+ }+ }+ } catch (const vk::SystemError& e) {+ std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;+ std::cerr << "ggml_vulkan: " << e.what() << std::endl;+ throw e;+ }++ return buf;+}++void ggml_vk_destroy_buffer(vk_buffer& buf) {+ if (buf == nullptr) {+ return;+ }++ if (buf->device != nullptr) {+ buf->device->memory_logger->log_deallocation(buf);+ }++ buf.reset();+}++void * ggml_vk_host_malloc(vk_device& device, size_t size) {+ VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");+ vk_buffer buf = ggml_vk_create_buffer(device, size,+ {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});++ if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {+ fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",+ size/1024.0/1024.0);+ device->device.freeMemory(buf->device_memory);+ device->device.destroyBuffer(buf->buffer);+ return nullptr;+ }++ std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);+ device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));++ return buf->ptr;+}++void ggml_vk_host_free(vk_device& device, void* ptr) {+ if (ptr == nullptr) {+ return;+ }+ VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");+ std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);++ vk_buffer buf;+ size_t index;+ for (size_t i = 0; i < device->pinned_memory.size(); i++) {+ const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);+ const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);+ if (ptr >= addr && ptr < endr) {+ buf = std::get<2>(device->pinned_memory[i]);+ index = i;+ break;+ }+ }+ if (buf == nullptr) {+ fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n");+ return;+ }++ ggml_vk_destroy_buffer(buf);++ device->pinned_memory.erase(device->pinned_memory.begin() + index);+}++void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {+ std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);+ buf = nullptr;+ buf_offset = 0;+ for (size_t i = 0; i < device->pinned_memory.size(); i++) {+ const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);+ const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);+ if (ptr >= addr && ptr < endr) {+ buf = std::get<2>(device->pinned_memory[i]);+ buf_offset = ((const uint8_t *)ptr) - addr;+ break;+ }+ }+}++void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) {+ if (device->sync_staging == nullptr || device->sync_staging->size < size) {+ VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");+ ggml_vk_destroy_buffer(device->sync_staging);+ device->sync_staging = ggml_vk_create_buffer_check(device, size,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);+ }+}++void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) {+ if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) {+ VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");+ ggml_vk_destroy_buffer(ctx->sync_staging);+ ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,+ vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);+ }+}++static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {+ VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");+ GGML_ASSERT(!ggml_is_contiguous(tensor));+ // Buffer is already mapped+ if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {+ std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl;+ GGML_ABORT("fatal error");+ }+ // Check if src is pinned memory+ vk_buffer buf = nullptr;+ size_t buf_offset = 0;+ ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset);++ const uint64_t ne0 = tensor->ne[0];+ const uint64_t ne1 = tensor->ne[1];+ const uint64_t ne2 = tensor->ne[2];+ const uint64_t ne3 = tensor->ne[3];+ const uint64_t nb0 = tensor->nb[0];+ const uint64_t nb1 = tensor->nb[1];+ const uint64_t nb2 = tensor->nb[2];+ const uint64_t nb3 = tensor->nb[3];+ const ggml_type type = tensor->type;+ const uint64_t ts = ggml_type_size(type);+ const uint64_t bs = ggml_blck_size(type);++ const uint64_t dstnb0 = ts;+ const uint64_t dstnb1 = dstnb0*(ne0/bs);+ const uint64_t dstnb2 = dstnb1*ne1;+ const uint64_t dstnb3 = dstnb2*ne2;++ const uint64_t ne = ggml_nelements(tensor);++ if (buf != nullptr) {+ // Memory is pinned, use as staging buffer+ std::vector<vk::BufferCopy> slices;++ for (uint64_t i3 = 0; i3 < ne3; i3++) {+ for (uint64_t i2 = 0; i2 < ne2; i2++) {+ // Find longest contiguous slice+ if (ne1*nb1 == dstnb2) {+ slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 });+ } else {+ for (uint64_t i1 = 0; i1 < ne1; i1++) {+ if (ne0*nb0/bs == dstnb1) {+ slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 });+ } else {+ const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;+ const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1;+ for (uint64_t i0 = 0; i0 < ne0; i0++) {+ slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 });+ }+ }+ }+ }+ }+ }++ ggml_vk_sync_buffers(ctx, subctx);+ subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);+ return;+ }++ if (!sync_staging) {+ GGML_ABORT("Asynchronous write to non-pinned memory not supported");+ }++ // Staging buffer required+ vk_buffer& staging = ctx->device->sync_staging;+ const uint64_t copy_size = ts*ne/bs;+ ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size);+ VkBufferCopy buf_copy{ 0, offset, copy_size };++ ggml_vk_sync_buffers(ctx, subctx);+ vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy);++ for (uint64_t i3 = 0; i3 < ne3; i3++) {+ for (uint64_t i2 = 0; i2 < ne2; i2++) {+ // Find longest contiguous slice+ if (ne1*nb1 == dstnb2) {+ deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys);+ } else {+ for (uint64_t i1 = 0; i1 < ne1; i1++) {+ if (ne0*nb0/bs == dstnb1) {+ deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys);+ } else {+ const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;+ const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1;+ for (uint64_t i0 = 0; i0 < ne0; i0++) {+ deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys);+ }+ }+ }+ }+ }+ }+}++bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {+ VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")");+ // Check if src is pinned memory+ vk_buffer buf = nullptr;+ size_t buf_offset = 0;+ ggml_vk_host_get(dst->device, src, buf, buf_offset);++ if (buf != nullptr) {+ // Memory is pinned, use as staging buffer+ std::vector<vk::BufferCopy> slices(1);+ if (width == spitch && width == dpitch) {+ // Only do single write if stride is equal+ slices[0].srcOffset = buf_offset;+ slices[0].dstOffset = offset;+ slices[0].size = width * height;+ } else {+ slices.resize(height);+ for (size_t i = 0; i < height; i++) {+ slices[i].srcOffset = buf_offset + i * spitch;+ slices[i].dstOffset = offset + i * dpitch;+ slices[i].size = width;+ }+ }++ ggml_vk_sync_buffers(nullptr, subctx);+ subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);+ return true;+ }+ VK_LOG_DEBUG("STAGING");++ if (!sync_staging) {+ // copy was not handled caller needs to fall back+ return false;+ }++ // Staging buffer required+ const size_t staging_size = width * height;+ ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size);++ vk_buffer& staging_buffer = dst->device->sync_staging;++ std::vector<vk::BufferCopy> slices(1);+ if (width == dpitch) {+ slices[0].srcOffset = 0;+ slices[0].dstOffset = offset;+ slices[0].size = staging_size;+ } else {+ slices.resize(height);+ for (size_t i = 0; i < height; i++) {+ slices[i].srcOffset = i * width;+ slices[i].dstOffset = offset + i * dpitch;+ slices[i].size = width;+ }+ }++ ggml_vk_sync_buffers(nullptr, subctx);+ subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices);++ if (width == spitch) {+ deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys);+ } else {+ for (size_t i = 0; i < height; i++) {+ deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys);+ }+ }+ return true;+}++bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging) {+ VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")");+ return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging);+}++void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) {+ VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")");+ // Buffer is already mapped+ if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {+ GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);++ if (width == spitch && width == dpitch) {+ memcpy((uint8_t *)dst->ptr + offset, src, width * height);+ } else {+ for (size_t i = 0; i < height; i++) {+ memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width);+ }+ }+ } else {+ std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);++ vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);+ ggml_vk_ctx_begin(dst->device, subctx);+ bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);+ GGML_ASSERT(ret);+ ggml_vk_ctx_end(subctx);++ for (auto& cpy : subctx->in_memcpys) {+ memcpy(cpy.dst, cpy.src, cpy.n);+ }++ for (auto& mset : subctx->memsets) {+ memset(mset.dst, mset.val, mset.n);+ }++ ggml_vk_submit(subctx, dst->device->fence);+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);+ dst->device->device.resetFences({ dst->device->fence });+ ggml_vk_queue_command_pools_cleanup(dst->device);+ }+}++void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) {+ VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")");+ ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1);+}++bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging) {+ VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")");+ GGML_ASSERT(width > 0);+ GGML_ASSERT(height > 0);+ GGML_ASSERT(src != nullptr);++ // TODO: staging_offset is not used++ // Check if dst is pinned memory+ vk_buffer buf = nullptr;+ size_t buf_offset = 0;+ ggml_vk_host_get(src->device, dst, buf, buf_offset);++ std::vector<vk::BufferCopy> slices(1);+ if (width == spitch && width == dpitch) {+ // Only do single write if stride is equal+ slices[0].srcOffset = offset;+ slices[0].dstOffset = buf_offset;+ slices[0].size = width * height;+ } else {+ slices.resize(height);+ for (size_t i = 0; i < height; i++) {+ slices[i].srcOffset = offset + i * spitch;+ slices[i].dstOffset = buf_offset + i * dpitch;+ slices[i].size = width;+ }+ }++ if (buf != nullptr) {+ // Memory is pinned, use as staging buffer+ ggml_vk_sync_buffers(nullptr, subctx);+ subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices);++ return true;+ }+ VK_LOG_DEBUG("STAGING");++ if (!sync_staging) {+ // copy was not handled caller needs to fall back+ return false;+ }++ // Fall back to staging buffer+ const size_t staging_size = width * height;+ ggml_vk_ensure_sync_staging_buffer(src->device, staging_size);++ vk_buffer& staging_buffer = src->device->sync_staging;++ std::vector<vk::BufferCopy> staging_slices(1);+ if (width == spitch) {+ staging_slices[0].srcOffset = offset;+ staging_slices[0].dstOffset = 0;+ staging_slices[0].size = staging_size;+ } else {+ staging_slices.resize(height);+ for (size_t i = 0; i < height; i++) {+ staging_slices[i].srcOffset = offset + i * spitch;+ staging_slices[i].dstOffset = i * width;+ staging_slices[i].size = width;+ }+ }++ ggml_vk_sync_buffers(nullptr, subctx);+ subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices);++ if (width == dpitch) {+ deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys);+ } else {+ for (size_t i = 0; i < height; i++) {+ deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys);+ }+ }+ return true;+}++static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) {+ return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);+}++void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {+ VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");++ // If the device is not an UMA device the memory is host-accessible through rebar. While writing+ // through PCIe is sufficient fast reading back data from PCIe is slower than going through+ // the HW device to host copy path.+ if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) {+ GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);++ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);+ vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);+ ggml_vk_ctx_begin(src->device, subctx);+ subctx->s->buffer->buf.pipelineBarrier(+ vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,+ vk::PipelineStageFlagBits::eHost,+ {},+ { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,+ vk::AccessFlagBits::eHostRead } },+ {}, {});+ ggml_vk_ctx_end(subctx);+ ggml_vk_submit(subctx, src->device->fence);+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),+ "vk_buffer_read_2d uma waitForFences", src->device);+ src->device->device.resetFences({ src->device->fence });+ ggml_vk_queue_command_pools_cleanup(src->device);++ if (width == spitch && width == dpitch) {+ memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);+ } else {+ for (size_t i = 0; i < height; i++) {+ memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width);+ }+ }+ } else {+ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);++ vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);+ ggml_vk_ctx_begin(src->device, subctx);+ bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);+ GGML_ASSERT(ret);+ ggml_vk_ctx_end(subctx);++ ggml_vk_submit(subctx, src->device->fence);+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);+ src->device->device.resetFences({ src->device->fence });+ ggml_vk_queue_command_pools_cleanup(src->device);++ for (auto& cpy : subctx->out_memcpys) {+ memcpy(cpy.dst, cpy.src, cpy.n);+ }+ }+}++void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) {+ VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")");+ ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1);+}++void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {+ VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")");+ // Make sure both buffers are on same device+ GGML_ASSERT(src->device == dst->device);++ VkBufferCopy bc{ src_offset, dst_offset, size };++ vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc);+}++void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {+ if (src->device == dst->device) {+ std::lock_guard<std::recursive_mutex> guard(src->device->mutex);+ VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");+ // Copy within the device+ vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);+ ggml_vk_ctx_begin(src->device, subctx);+ ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);+ ggml_vk_ctx_end(subctx);+ ggml_vk_submit(subctx, src->device->fence);+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);+ src->device->device.resetFences({ src->device->fence });+ ggml_vk_queue_command_pools_cleanup(src->device);+ } else {+ VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");+ // Copy device to device+ ggml_vk_ensure_sync_staging_buffer(src->device, size);++ // Copy to src staging buffer+ ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size);+ // Copy to dst buffer+ ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size);+ }+}++void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) {+ VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")");++ if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&+ dst->device->uma) {+ deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets);+ return;+ }++ // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers+ ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);+}++void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) {+ VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")");++ if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&+ dst->device->uma) {+ memset((uint8_t*)dst->ptr + offset, c, size);+ return;+ }++ std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);+ vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);+ ggml_vk_ctx_begin(dst->device, subctx);+ subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);+ ggml_vk_ctx_end(subctx);++ ggml_vk_submit(subctx, dst->device->fence);+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);+ dst->device->device.resetFences({ dst->device->fence });+ ggml_vk_queue_command_pools_cleanup(dst->device);+}++ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {+ /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer,+ /* .get_base = */ ggml_backend_vk_buffer_get_base,+ /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor,+ /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor,+ /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor,+ /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor,+ /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d,+ /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d,+ /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,+ /* .clear = */ ggml_backend_vk_buffer_clear,+ /* .reset = */ NULL,+};++vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {+ if (!device->external_memory_host) {+ return {};+ }++ uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr);+ if (uptr & (device->min_imported_host_pointer_alignment - 1)) {+ return {};+ }+ if (size & (device->min_imported_host_pointer_alignment - 1)) {+ return {};+ }++ const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;++ vk_buffer buf {};+ try {+ buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr);+ } catch (vk::SystemError& e) {+ GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what());+ }++ return buf;+}+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.hnew file mode 100644index 0000000000..4ae5fea7a8--- /dev/null+++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h@@ -0,0 +1,282 @@+#pragma once+#include "ggml-vulkan-push-constants.h"++// shared globals+extern ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface;+extern bool vk_memory_logger_enabled;+extern bool vk_perf_logger_enabled;+extern bool vk_perf_logger_concurrent;+extern bool vk_enable_sync_logger;+extern uint32_t vk_perf_logger_frequency;+extern std::string vk_pipeline_stats_filter;+extern void * const vk_ptr_base;+extern vk_instance_t vk_instance;+extern ggml_backend_buffer_i ggml_backend_vk_buffer_interface;++// instance+vk_device ggml_vk_get_device(size_t idx);+DispatchLoaderDynamic & ggml_vk_default_dispatcher();+void ggml_vk_instance_init();+void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx);+int ggml_vk_get_device_count();+void ggml_vk_get_device_description(int device, char * description, size_t description_size);+bool ggml_vk_instance_layer_settings_available();+bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions);+bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions);+bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev);+bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch);+uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev);+bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor);++// shaders+void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);+vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc);+vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc, bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type);+uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch);+void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);+bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);+bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16);++// buffers+vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0));+vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size);+void ggml_vk_destroy_buffer(vk_buffer& buf);+void * ggml_vk_host_malloc(vk_device& device, size_t size);+void ggml_vk_host_free(vk_device& device, void* ptr);+void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset);+void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size);+void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size);+bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false);+bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false);+void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height);+void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size);+bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false);+void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height);+void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size);+void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);+void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size);+void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size);+void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size);+vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size);++// pipelines+uint64_t vk_tensor_offset(const ggml_tensor * tensor);+uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);+void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx);+void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n);+void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);+void ggml_vk_submit(vk_context& ctx, vk::Fence fence);+uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues);+std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only);+std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source);+vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p);+vk_context ggml_vk_create_temporary_context(vk_command_pool& p);+void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p);+void ggml_vk_queue_command_pools_cleanup(vk_device& device);+vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0);+void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx);+void ggml_vk_set_event(vk_context& ctx, vk::Event& event);+void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events);+vk_subbuffer ggml_vk_tensor_subbuffer(const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false);+void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name);+void ggml_vk_ctx_end(vk_context& ctx);+void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx);+vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx);+vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx);+bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx);+size_t ggml_vk_align_size(size_t width, size_t align);+void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr);+void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr);++// matmul+vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type);+void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n);+bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor);+vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to);+vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type);+void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne);+void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst);+void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst);+void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst);+void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);+bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx);+void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);++// flash-attn+void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst);++// operators+void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out);+bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst);+void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst);+void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);+void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+int ggml_vk_unary_mul_op_index(ggml_unary_op op);+void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);+void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);+void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx);+void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);+void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node);+void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params);+void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst);+void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);+void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop);+void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx);+void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst);+void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx);+void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);+void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);+void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst);++// graph+void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);+bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit);+void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready);+void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx);+void ggml_vk_cleanup(ggml_backend_vk_context * ctx);+void ggml_vk_synchronize(ggml_backend_vk_context * ctx);+bool ggml_vk_is_empty(ggml_tensor * node);+bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops);+bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, int num_extra);+bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, topk_moe_mode mode);+bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise);+bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);+void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params);++// backend+bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer);+void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer);+void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer);+enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor);+void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);+void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size);+void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size);+void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst);+void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value);+const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);+ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);+size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);+size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);+size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);+void ggml_backend_vk_free(ggml_backend_t backend);+ggml_backend_reg_t ggml_backend_vk_reg();++// debug+int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op);++// ggml-vulkan.cpp (residual)+bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type);+void ggml_vk_print_device_fault_info(const vk_device& device);+uint64_t ggml_vk_get_node_flops(const ggml_tensor * node);+void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end);+void ggml_vk_print_device_lost_info(const vk_device& device);+size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t);+size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size);+uint32_t ggml_vk_concat_unit_size(ggml_type type);+bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst);++template <typename T>+inline void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {+ const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);+ const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);+ const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);+ VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {";+ for (auto& buffer : descriptor_buffer_infos) {+ std::cerr << "(" << buffer.buffer << ", " << buffer.offset << ", " << buffer.range << "), ";+ }+ std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))");+ GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] &&+ wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&+ wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]);+ GGML_ASSERT(ctx->descriptor_set_idx < ctx->descriptor_sets.size());+ GGML_ASSERT(descriptor_buffer_infos.size() <= MAX_PARAMETER_COUNT);+ GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size());+ GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants));++ vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[ctx->descriptor_set_idx++];+ vk::WriteDescriptorSet write_descriptor_set{ descriptor_set, 0, 0, pipeline->parameter_count, vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.begin() };+ ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {});++ subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants));+ subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline);+ subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute,+ pipeline->layout,+ 0,+ { descriptor_set },+ {});+ {+ ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2);+ subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);+ }+}+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cppnew file mode 100644index 0000000000..15abd54681--- /dev/null+++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp@@ -0,0 +1,1561 @@+#include "ggml-vulkan-common.h"++bool vk_memory_logger_enabled = false;++bool vk_perf_logger_enabled = false;++bool vk_perf_logger_concurrent = false;++bool vk_enable_sync_logger = false;++uint32_t vk_perf_logger_frequency = 1;++std::string vk_pipeline_stats_filter;++void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {+ if (!vk_memory_logger_enabled) {+ return;+ }+ std::lock_guard<std::mutex> guard(log_mutex);+ vk_buffer buf = buf_ref.lock();+ const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);+ const std::string type = device ? "device" : "host";+ allocations[buf->buffer] = size;+ total_device += device ? size : 0;+ total_host += device ? 0 : size;+ VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));+}++void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) {+ if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) {+ return;+ }++ std::lock_guard<std::mutex> guard(log_mutex);+ vk_buffer buf = buf_ref.lock();+ const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);+ std::string type = device ? "device" : "host";+ auto it = allocations.find(buf->buffer);+ if (it != allocations.end()) {+ total_device -= device ? it->second : 0;+ total_host -= device ? 0 : it->second;+ VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));+ allocations.erase(it);+ } else {+ VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer);+ }+}++#ifdef GGML_VULKAN_CHECK_RESULTS+static size_t vk_skip_checks;+static size_t vk_output_tensor;++static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name);+static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);+static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);+#endif++#ifdef GGML_VULKAN_RUN_TESTS+static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) {+ if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) {+ return;+ }+ i0 = std::max(i0, 5);+ i1 = std::max(i1, 5);+ i2 = std::max(i2, 0);+ fprintf(stderr, " ");+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ fprintf(stderr, "%7d ", idx1);+ }+ fprintf(stderr, "\n");+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {+ fprintf(stderr, "%7d: ", idx0);+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) {+ float val;+ if (type == GGML_TYPE_F32) {+ val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0);+ } else if (type == GGML_TYPE_F16) {+ val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0));+ } else {+ GGML_ABORT("fatal error");+ }+ fprintf(stderr, "% 7.2f ", val);+ } else {+ fprintf(stderr, " ");+ }+ }+ fprintf(stderr, "\n");+ }+}++template <typename X_TYPE, typename Y_TYPE>+static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) {+ VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")");+ const size_t x_ne = m * k * batch;+ const size_t y_ne = k * n * batch;+ const size_t d_ne = m * n * batch;++ ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;+ ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;+ vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false};+ auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key);+ GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty());+ auto& mm_test_configs = mm_test_it->second;+ GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size());++ std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size);+ vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned;++ const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align);++ if (k != kpad) {+ p = mm_test_configs[shader_size].unaligned;+ shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size);+ }++ if (split_k > 1) {+ ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);++ if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {+ // Resize buffer+ if (ctx->prealloc_split_k != nullptr) {+ ggml_vk_destroy_buffer(ctx->prealloc_split_k);+ }+ ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ }+ }++ ggml_pipeline_allocate_descriptor_sets(ctx);++ vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});++ X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne);+ Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne);+ float* d = (float *) malloc(sizeof(float) * d_ne);++ for (size_t i = 0; i < x_ne; i++) {+ if (std::is_same<float, X_TYPE>()) {+ x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;+ // x[i] = 1.0f;+ // x[i] = i + 1;+ // x[i] = (i % k == i / k) ? 1.0f : 0.0f;+ } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {+ x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);+ // x[i] = ggml_fp32_to_fp16(1.0f);+ // x[i] = ggml_fp32_to_fp16(i + 1);+ // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);+ } else {+ GGML_ABORT("fatal error");+ }+ }+ for (size_t i = 0; i < y_ne; i++) {+ if (std::is_same<float, Y_TYPE>()) {+ y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;+ // y[i] = (i % k == i / k) ? 1.0f : 0.0f;+ // y[i] = i + 1;+ } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {+ y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);+ // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);+ // y[i] = ggml_fp32_to_fp16(i + 1);+ } else {+ GGML_ABORT("fatal error");+ }+ }++ ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch);+ ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch);++ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);+ ggml_vk_ctx_begin(ctx->device, subctx);+ for (size_t i = 0; i < num_it; i++) {+ ggml_vk_matmul(+ ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),+ m, n, k,+ k, k, m, k*m, k*n, m*n,+ split_k, batch, batch, batch, 1, 1, n+ );+ }+ ggml_vk_ctx_end(subctx);++ auto begin = std::chrono::high_resolution_clock::now();+ ggml_vk_submit(subctx, ctx->fence);+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);+ ctx->device->device.resetFences({ ctx->fence });+ ggml_vk_queue_command_pools_cleanup(ctx->device);++ auto end = std::chrono::high_resolution_clock::now();+ double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;++ // copy dst to host+ ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne);++ float * d_chk = (float *) malloc(sizeof(float) * d_ne);++ ggml_init_params iparams = {+ /*.mem_size =*/ 1024*1024*1024,+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,+ };++ ggml_context * ggml_ctx = ggml_init(iparams);++ ggml_type src0_type;+ ggml_type src1_type;++ if (std::is_same<float, X_TYPE>()) {+ src0_type = GGML_TYPE_F32;+ } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {+ src0_type = GGML_TYPE_F16;+ } else {+ GGML_ABORT("fatal error");+ }+ if (std::is_same<float, Y_TYPE>()) {+ src1_type = GGML_TYPE_F32;+ } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {+ src1_type = GGML_TYPE_F16;+ } else {+ GGML_ABORT("fatal error");+ }++ ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch);+ ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch);+ ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);++ src0_ggml->data = x;+ src1_ggml->data = y;+ tensor_ggml->data = d_chk;++ ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);+ ggml_build_forward_expand(cgraph, tensor_ggml);++ ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);++ ggml_free(ggml_ctx);++ double avg_err = 0.0;+ int first_err_n = -1;+ int first_err_m = -1;+ int first_err_b = -1;++ for (size_t i = 0; i < m*n*batch; i++) {+ double err = std::fabs(d[i] - d_chk[i]);+ avg_err += err;++ if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {+ first_err_b = i / (m * n);+ first_err_n = (i % (m * n)) / m;+ first_err_m = (i % (m * n)) % m;+ }+ }++ avg_err /= m * n;++ double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0);++ std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;++ if (avg_err > 0.1 || std::isnan(avg_err)) {+ std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;+ std::cerr << "Actual result: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+ std::cerr << "Expected result: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ if (split_k > 1) {+ float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);+ ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);++ std::cerr << "d_buf0: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf1: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf2: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf3: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ free(split_k_buf);+ }+ }++ free(d_chk);++ ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool);++ ggml_vk_destroy_buffer(d_X);+ ggml_vk_destroy_buffer(d_Y);+ ggml_vk_destroy_buffer(d_D);++ free(x);+ free(y);+ free(d);+}++static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) {+ if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) {+ return;+ }+ i0 = std::max(i0, 5);+ i1 = std::max(i1, 5);+ i2 = std::max(i2, 0);+ i3 = std::max(i3, 0);+ fprintf(stderr, " ");+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ fprintf(stderr, "%7d ", idx1);+ }+ fprintf(stderr, "\n");+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {+ fprintf(stderr, "%7d: ", idx0);+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {+ float val;+ if (tensor->type == GGML_TYPE_F32) {+ val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);+ } else if (tensor->type == GGML_TYPE_F16) {+ val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));+ } else {+ GGML_ABORT("fatal error");+ }+ fprintf(stderr, "% 7.2f ", val);+ } else {+ fprintf(stderr, " ");+ }+ }+ fprintf(stderr, "\n");+ }+}++static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) {+ ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr);+}++static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) {+ if (quant == GGML_TYPE_F32) {+ memcpy(to, from, sizeof(float) * ne);+ return;+ }++ const auto * tt = ggml_get_type_traits(quant);++ ggml_to_float_t dequant_fn = tt->to_float;++ dequant_fn(from, to, ne);+}++static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {+ VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")");+ const size_t x_sz = sizeof(float) * ne;+ const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne;+ const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);+ float * x = (float *) malloc(x_sz);+ void * qx = malloc(qx_sz);+ vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ float * x_ref = (float *) malloc(x_sz);+ ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16);++ for (size_t i = 0; i < ne; i++) {+ x[i] = rand() / (float)RAND_MAX;+ }++ vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant);++ ggml_vk_quantize_data(x, qx, ne, quant);+ ggml_vk_dequantize_data(qx, x_ref, ne, quant);++ ggml_pipeline_request_descriptor_sets(ctx, p, 1);++ ggml_pipeline_allocate_descriptor_sets(ctx);++ ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);++ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);+ ggml_vk_ctx_begin(ctx->device, subctx);+ const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne };+ ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1});+ ggml_vk_ctx_end(subctx);++ auto begin = std::chrono::high_resolution_clock::now();++ ggml_vk_submit(subctx, ctx->fence);+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);+ ctx->device->device.resetFences({ ctx->fence });+ ggml_vk_queue_command_pools_cleanup(ctx->device);++ auto end = std::chrono::high_resolution_clock::now();++ double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;+ ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16);++ int first_err = -1;++ double avg_err = 0.0;+ for (size_t i = 0; i < ne; i++) {+ double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i]));+ avg_err += error;++ if (first_err < 0 && error > 0.05) {+ first_err = i;+ }+ }++ avg_err /= ne;++ std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl;++ if (avg_err > 0.1) {+ std::cerr << "first_error = " << first_err << std::endl;+ std::cerr << "Actual result: " << std::endl << std::endl;+ for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {+ std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", ";+ }+ std::cerr << std::endl << "Expected result: " << std::endl << std::endl;+ for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {+ std::cerr << x_ref[i] << ", ";+ }+ std::cerr << std::endl;+ }++ ggml_vk_destroy_buffer(x_buf);+ ggml_vk_destroy_buffer(qx_buf);++ free(x);+ free(qx);+ free(x_ref);+ free(x_chk);+}++// This does not work without ggml q8_1 quantization support+//+// typedef uint16_t ggml_half;+// typedef uint32_t ggml_half2;+//+// #define QK8_1 32+// typedef struct {+// union {+// struct {+// ggml_half d; // delta+// ggml_half s; // d * sum(qs[i])+// } GGML_COMMON_AGGR_S;+// ggml_half2 ds;+// } GGML_COMMON_AGGR_U;+// int8_t qs[QK8_1]; // quants+// } block_q8_1;+//+// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {+// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")");+// GGML_ASSERT(quant == GGML_TYPE_Q8_1);+//+// const size_t x_sz = sizeof(float) * ne;+// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);+// float * x = (float *) malloc(x_sz);+// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz);+// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz);+// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+//+// for (size_t i = 0; i < ne; i++) {+// x[i] = rand() / (float)RAND_MAX;+// }+//+// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant);+//+// ggml_pipeline_request_descriptor_sets(ctx, p, 1);+//+// ggml_pipeline_allocate_descriptor_sets(ctx);+//+// ggml_vk_buffer_write(x_buf, 0, x, x_sz);+//+// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);+// ggml_vk_ctx_begin(ctx->device, subctx);+// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne);+// ggml_vk_ctx_end(subctx);+//+// auto begin = std::chrono::high_resolution_clock::now();+//+// ggml_vk_submit(subctx, ctx->fence);+// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences");+// ctx->device->device.resetFences({ ctx->fence });+// ggml_vk_queue_command_pools_cleanup(ctx->device);+//+// auto end = std::chrono::high_resolution_clock::now();+//+// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;+// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz);+//+// ggml_vk_quantize_data(x, qx_res, ne, quant);+//+// int first_err = -1;+//+// for (size_t i = 0; i < ne / 32; i++) {+// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d));+//+// if (first_err < 0 && error > 0.1) {+// first_err = i;+// }+//+// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s));+//+// if (first_err < 0 && error > 0.1) {+// first_err = i;+// }+//+// for (size_t j = 0; j < 32; j++) {+// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]);+//+// if (first_err < 0 && error > 1) {+// first_err = i;+// }+// }+// }+//+// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl;+//+// if (first_err != -1) {+// std::cerr << "first_error = " << first_err << std::endl;+// std::cerr << "Actual result: " << std::endl << std::endl;+// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";+// for (size_t j = 0; j < 32; j++) {+// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " ";+// }+// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl;+// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";+// for (size_t j = 0; j < 32; j++) {+// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " ";+// }+// std::cerr << std::endl;+// }+//+// ggml_vk_destroy_buffer(x_buf);+// ggml_vk_destroy_buffer(qx_buf);+//+// free(x);+// free(qx);+// free(qx_res);+// }++static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) {+ VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")");+ const size_t x_ne = m * k * batch;+ const size_t y_ne = k * n * batch;+ const size_t d_ne = m * n * batch;++ ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32;+ bool f16acc = ctx->device->fp16 && !mmq;+ vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc};+ auto dq_it = ctx->device->pipeline_matmul.find(dq_key);+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {+ if (f16acc) {+ dq_key.f16acc = false;+ dq_it = ctx->device->pipeline_matmul.find(dq_key);+ }+ }+ if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {+ std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;+ return;+ }+ auto& dq_configs = dq_it->second;+ if (shader_size >= (int)dq_configs.size()) {+ std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl;+ return;+ }++ std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size);+ vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned;++ const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align);++ if (mmq || k != kpad) {+ p = dq_configs[shader_size].unaligned;+ shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size);+ }++ if (p == nullptr) {+ std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;+ return;+ }++ const size_t x_sz = sizeof(float) * x_ne;+ const size_t y_sz = sizeof(float) * y_ne;+ const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant);+ const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz;+ const size_t d_sz = sizeof(float) * d_ne;+ float * x = (float *) malloc(x_sz);+ float * y = (float *) malloc(y_sz);+ void * qx = malloc(qx_sz);+ vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ float * d = (float *) malloc(d_sz);+ float * d_chk = (float *) malloc(d_sz);++ for (size_t i = 0; i < x_ne; i++) {+ x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;+ // x[i] = (i % k == i / k) ? 1.0f : 0.0f;+ // x[i] = i % k;+ }++ ggml_vk_quantize_data(x, qx, x_ne, quant);++ for (size_t i = 0; i < y_ne; i++) {+ y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;+ // y[i] = (i % k == i / k) ? 1.0f : 0.0f;+ // y[i] = i % k;+ }++ if (split_k > 1) {+ ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);++ if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {+ // Resize buffer+ if (ctx->prealloc_split_k != nullptr) {+ ggml_vk_destroy_buffer(ctx->prealloc_split_k);+ }+ ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});+ }+ }+ if (mmq) {+ vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);+ ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it);+ }++ ggml_pipeline_allocate_descriptor_sets(ctx);++ ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);+ ggml_vk_buffer_write(y_buf, 0, y, y_sz);++ vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);+ ggml_vk_ctx_begin(ctx->device, subctx);+ if (mmq) {+ for (size_t i = 0; i < num_it; i++) {+ ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne);+ ggml_vk_matmul(+ ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },+ m, n, k,+ k, k, m, k*m, k*n, m*n,+ split_k, batch, batch, batch, 1, 1, n+ );+ }+ } else {+ for (size_t i = 0; i < num_it; i++) {+ ggml_vk_matmul(+ ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },+ m, n, k,+ k, k, m, k*m, k*n, m*n,+ split_k, batch, batch, batch, 1, 1, n+ );+ }+ }+ ggml_vk_ctx_end(subctx);++ auto begin = std::chrono::high_resolution_clock::now();++ ggml_vk_submit(subctx, ctx->fence);+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);+ ctx->device->device.resetFences({ ctx->fence });+ ggml_vk_queue_command_pools_cleanup(ctx->device);++ auto end = std::chrono::high_resolution_clock::now();++ double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;+ ggml_vk_buffer_read(d_buf, 0, d, d_sz);++ ggml_init_params iparams = {+ /*.mem_size =*/ 1024*1024*1024,+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,+ };++ ggml_context * ggml_ctx = ggml_init(iparams);++ ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch);+ ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch);+ ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);++ src0_ggml->data = qx;+ src1_ggml->data = y;+ tensor_ggml->data = d_chk;++ ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);+ ggml_build_forward_expand(cgraph, tensor_ggml);++ ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);++ ggml_free(ggml_ctx);++ double avg_err = 0.0;+ int first_err_n = -1;+ int first_err_m = -1;+ int first_err_b = -1;++ for (size_t i = 0; i < m*n*batch; i++) {+ double err = std::fabs(d[i] - d_chk[i]);+ avg_err += err;++ if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {+ first_err_b = i / (m * n);+ first_err_n = (i % (m * n)) / m;+ first_err_m = (i % (m * n)) % m;+ }+ }++ avg_err /= m * n;++ double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0);++ std::cerr << "TEST dequant matmul " << shname;+ if (mmq) {+ std::cerr << " mmq";+ }+ std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;++ if (avg_err > 0.01 || std::isnan(avg_err)) {+ std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;+ std::cerr << "Actual result: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+ std::cerr << std::endl;+ std::cerr << "Expected result: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "src0: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b);+ std::cerr << std::endl;+ std::cerr << "src1: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b);++ if (split_k > 1) {+ float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);+ ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);++ std::cerr << "d_buf0: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf1: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf2: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ std::cerr << "d_buf3: " << std::endl << std::endl;+ ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);++ free(split_k_buf);+ }+ }++ ggml_vk_destroy_buffer(qx_buf);+ ggml_vk_destroy_buffer(y_buf);+ ggml_vk_destroy_buffer(qy_buf);+ ggml_vk_destroy_buffer(d_buf);++ free(x);+ free(qx);+ free(y);+ free(d);+ free(d_chk);+}+#endif++int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {+ switch (op->op) {+ case GGML_OP_GET_ROWS:+ return 0;+ case GGML_OP_MUL_MAT:+ return op->ne[1];+ case GGML_OP_MUL_MAT_ID:+ case GGML_OP_ROPE:+ case GGML_OP_ROPE_BACK:+ return op->ne[2];+ default:+ return ggml_nrows(op);+ }+}++#ifdef GGML_VULKAN_CHECK_RESULTS+static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) {+ if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) {+ return;+ }+ for (int j = 0; j < level; j++) {+ std::cerr << " ";+ }+ std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl;++ done.push_back(tensor);++ for (int i = 0; i < GGML_MAX_SRC; i++) {+ if (tensor->src[i] != nullptr) {+ ggml_vk_print_graph_origin(tensor->src[i], done, level + 1);+ }+ }+}++static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) {+ if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) {+ return;+ }+ i0 = std::max(i0, 5);+ i1 = std::max(i1, 5);+ i2 = std::max(i2, 0);+ i3 = std::max(i3, 0);+ fprintf(stderr, " ");+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ fprintf(stderr, "%7d ", idx1);+ }+ fprintf(stderr, "\n");+ for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {+ fprintf(stderr, "%7d: ", idx0);+ for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {+ if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {+ float val;+ if (tensor->type == GGML_TYPE_F32) {+ val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);+ } else if (tensor->type == GGML_TYPE_F16) {+ val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));+ } else if (tensor->type == GGML_TYPE_I32) {+ val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);+ } else {+ GGML_ABORT("fatal error");+ }+ fprintf(stderr, "% 7.2f ", val);+ } else {+ fprintf(stderr, " ");+ }+ }+ fprintf(stderr, "\n");+ }+}++static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) {+ void * tensor_data = tensor->data;++ const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer);++ if (is_gpu) {+ const size_t tensor_size = ggml_nbytes(tensor);+ tensor_data = malloc(tensor_size);++ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;++ vk_buffer buffer_gpu = buf_ctx->dev_buffer;+ ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size);+ }++ std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl;+ std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl;+ if (tensor->src[0] != nullptr) {+ std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl;+ }+ if (tensor->src[1] != nullptr) {+ std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl;+ }+ std::cerr << std::endl << "Result:" << std::endl;+ ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);+ std::cerr << std::endl;+ std::vector<const ggml_tensor *> done;+ ggml_vk_print_graph_origin(tensor, done);++ if (is_gpu) {+ free(tensor_data);+ }+}++void * comp_result;+size_t comp_size;+size_t comp_nb[GGML_MAX_DIMS];+size_t check_counter = 0;+static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {+ ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];+ if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {+ return;+ }++ check_counter++;+ if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {+ return;+ }++ VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")");++ struct ggml_init_params iparams = {+ /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul,+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ false,+ };++ struct ggml_context * ggml_ctx = ggml_init(iparams);++ std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr};+ const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"};++ std::map<ggml_tensor *, ggml_tensor *> cloned_tensors;+ std::vector<void *> cloned_mallocs;++ struct ggml_tensor * tensor_clone = nullptr;++ for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) {+ tensor = cgraph->nodes[tensor_idx + f];+ for (int i = 0; i < GGML_MAX_SRC; i++) {+ ggml_tensor * srci = tensor->src[i];+ if (srci == nullptr) {+ continue;+ }+ // If a src tensor has been cloned, use that one+ auto it = cloned_tensors.find(srci);+ if (it != cloned_tensors.end()) {+ src_clone[i] = it->second;+ continue;+ }+ ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci);+ size_t srci_size = ggml_nbytes(srci);++ src_clone[i] = srci_clone;+ void *src_buffer = malloc(srci_size);+ cloned_mallocs.push_back(src_buffer);++ srci_clone->data = src_buffer;+ if (ggml_backend_buffer_is_host(srci->buffer)) {+ memcpy(srci_clone->data, srci->data, srci_size);+ memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);+ } else if (ggml_backend_buffer_is_vk(srci->buffer)) {+ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context;+ vk_buffer& buffer_gpu = buf_ctx->dev_buffer;+ uint64_t offset = vk_tensor_offset(srci) + srci->view_offs;+ if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) {+ for (int i3 = 0; i3 < srci->ne[3]; i3++) {+ for (int i2 = 0; i2 < srci->ne[2]; i2++) {+ const int idx = i3*srci->ne[2] + i2;+ ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]);+ }+ }++ srci_clone->nb[0] = srci->nb[0];+ srci_clone->nb[1] = srci->nb[1];+ for (int i = 2; i < GGML_MAX_DIMS; i++) {+ srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1];+ }+ } else {+ if (offset + srci_size >= buffer_gpu->size) {+ srci_size = buffer_gpu->size - offset;+ }+ ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size);+ memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);+ }+ } else {+ GGML_ABORT("fatal error");+ }++ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {+ ggml_vk_print_tensor(srci, srci_name[i]);+ }+ }++ if (tensor->op == GGML_OP_FLASH_ATTN_EXT) {+ const float * params = (const float *)tensor->op_params;+ tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]);+ if (src_clone[4]) {+ ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]);+ }+ } else if (tensor->op == GGML_OP_MUL_MAT) {+ tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_MUL_MAT_ID) {+ tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);+ } else if (tensor->op == GGML_OP_SUB) {+ tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_MUL) {+ tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_DIV) {+ tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_CONCAT) {+ tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params);+ } else if (tensor->op == GGML_OP_UPSCALE) {+ tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]);+ } else if (tensor->op == GGML_OP_SCALE) {+ const float * params = (const float *)tensor->op_params;+ tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]);+ } else if (tensor->op == GGML_OP_ADD1) {+ tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_ARANGE) {+ const float start = ggml_get_op_params_f32(tensor, 0);+ const float stop = ggml_get_op_params_f32(tensor, 1);+ const float step = ggml_get_op_params_f32(tensor, 2);+ tensor_clone = ggml_arange(ggml_ctx, start, stop, step);+ } else if (tensor->op == GGML_OP_FILL) {+ const float value = ggml_get_op_params_f32(tensor, 0);+ tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);+ } else if (tensor->op == GGML_OP_SQR) {+ tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_SQRT) {+ tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_SIN) {+ tensor_clone = ggml_sin(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_COS) {+ tensor_clone = ggml_cos(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_LOG) {+ tensor_clone = ggml_log(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_TRI) {+ tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0));+ } else if (tensor->op == GGML_OP_DIAG) {+ tensor_clone = ggml_diag(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_CLAMP) {+ const float * params = (const float *)tensor->op_params;+ tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]);+ } else if (tensor->op == GGML_OP_PAD) {+ tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3],+ tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]);+ } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) {+ tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]);+ } else if (tensor->op == GGML_OP_REPEAT) {+ tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor);+ } else if (tensor->op == GGML_OP_REPEAT_BACK) {+ tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor);+ } else if (tensor->op == GGML_OP_ADD) {+ tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_ACC) {+ tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);+ } else if (tensor->op == GGML_OP_SET) {+ tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);+ } else if (tensor->op == GGML_OP_NORM) {+ tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);+ } else if (tensor->op == GGML_OP_GROUP_NORM) {+ const float * float_params = (const float *)tensor->op_params;+ tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]);+ } else if (tensor->op == GGML_OP_RMS_NORM) {+ tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);+ } else if (tensor->op == GGML_OP_RMS_NORM_BACK) {+ const float eps = ((float *) tensor->op_params)[0];+ tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps);+ } else if (tensor->op == GGML_OP_SILU_BACK) {+ tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_L2_NORM) {+ const float eps = ((float *) tensor->op_params)[0];+ tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps);+ } else if (tensor->op == GGML_OP_SOFT_MAX) {+ if (tensor->src[1] != nullptr) {+ const float * params = (const float *)tensor->op_params;+ tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]);+ } else {+ tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]);+ }+ } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) {+ tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]);+ } else if (tensor->op == GGML_OP_DIAG_MASK_INF) {+ tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]);+ } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) {+ const int n_dims = ((int32_t *) tensor->op_params)[1];+ const int mode = ((int32_t *) tensor->op_params)[2];+ //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3];+ const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4];+ const float freq_base = ((float *) tensor->op_params)[5];+ const float freq_scale = ((float *) tensor->op_params)[6];+ const float ext_factor = ((float *) tensor->op_params)[7];+ const float attn_factor = ((float *) tensor->op_params)[8];+ const float beta_fast = ((float *) tensor->op_params)[9];+ const float beta_slow = ((float *) tensor->op_params)[10];+ if (mode & GGML_ROPE_TYPE_MROPE) {+ int32_t *sections = ((int32_t *) tensor->op_params) + 11;+ if (tensor->op == GGML_OP_ROPE) {+ tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ } else {+ tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ }+ } else {+ if (tensor->op == GGML_OP_ROPE) {+ tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ } else {+ tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);+ }+ }+ const int n_offs = ((int32_t *) tensor->op_params)[15];+ if (n_offs != 0) {+ tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs);+ }+ } else if (tensor->op == GGML_OP_UNARY) {+ switch (ggml_get_unary_op(tensor)) {+ case GGML_UNARY_OP_EXP:+ tensor_clone = ggml_exp(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_EXPM1:+ tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_ELU:+ tensor_clone = ggml_elu(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_SILU:+ tensor_clone = ggml_silu(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_GELU:+ tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_GELU_ERF:+ tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_GELU_QUICK:+ tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_RELU:+ tensor_clone = ggml_relu(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_XIELU:+ tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0);+ ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1));+ ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2));+ ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3));+ ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4));+ break;+ case GGML_UNARY_OP_NEG:+ tensor_clone = ggml_neg(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_TANH:+ tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_SIGMOID:+ tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_HARDSIGMOID:+ tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_HARDSWISH:+ tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_ABS:+ tensor_clone = ggml_abs(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_SOFTPLUS:+ tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_STEP:+ tensor_clone = ggml_step(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_ROUND:+ tensor_clone = ggml_round(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_CEIL:+ tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_FLOOR:+ tensor_clone = ggml_floor(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_TRUNC:+ tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]);+ break;+ case GGML_UNARY_OP_SGN:+ tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]);+ break;+ default:+ std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;+ GGML_ABORT("fatal error");+ }+ } else if (tensor->op == GGML_OP_GLU) {+ if (src_clone[1] == nullptr) {+ tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]);+ } else {+ tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]);+ }+ ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2));+ ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3));+ } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) {+ if (tensor->src[1] == nullptr) {+ tensor_clone = ggml_dup(ggml_ctx, src_clone[0]);+ tensor_clone->type = tensor->type;+ } else {+ tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]);+ }+ } else if (tensor->op == GGML_OP_CONT) {+ tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);+ } else if (tensor->op == GGML_OP_RESHAPE) {+ tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);+ } else if (tensor->op == GGML_OP_VIEW) {+ tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]);+ } else if (tensor->op == GGML_OP_PERMUTE) {+ int32_t * params = (int32_t *)tensor->op_params;+ tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]);+ } else if (tensor->op == GGML_OP_TRANSPOSE) {+ tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_GET_ROWS) {+ tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_ARGSORT) {+ tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params);+ } else if (tensor->op == GGML_OP_TOP_K) {+ tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]);+ } else if (tensor->op == GGML_OP_SUM) {+ tensor_clone = ggml_sum(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_SUM_ROWS) {+ tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_CUMSUM) {+ tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_DSV4_HC_COMB) {+ tensor_clone = ggml_dsv4_hc_comb(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],+ ggml_get_op_params_f32(tensor, 0), ggml_get_op_params_i32(tensor, 1));+ } else if (tensor->op == GGML_OP_DSV4_HC_PRE) {+ if (ggml_get_op_params_i32(tensor, 1) != 0) {+ tensor_clone = ggml_dsv4_hc_pre_gated(ggml_ctx, src_clone[0], src_clone[1], ggml_get_op_params_f32(tensor, 0));+ } else {+ tensor_clone = ggml_dsv4_hc_pre(ggml_ctx, src_clone[0], src_clone[1]);+ }+ } else if (tensor->op == GGML_OP_DSV4_HC_POST) {+ tensor_clone = ggml_dsv4_hc_post(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);+ } else if (tensor->op == GGML_OP_MEAN) {+ tensor_clone = ggml_mean(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_ARGMAX) {+ tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]);+ } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) {+ tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) {+ tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);+ } else if (tensor->op == GGML_OP_COUNT_EQUAL) {+ tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_SOLVE_TRI) {+ tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false);+ } else if (tensor->op == GGML_OP_IM2COL) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t p0 = tensor->op_params[2];+ const int32_t p1 = tensor->op_params[3];+ const int32_t d0 = tensor->op_params[4];+ const int32_t d1 = tensor->op_params[5];++ const bool is_2D = tensor->op_params[6] == 1;+ tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type);+ } else if (tensor->op == GGML_OP_IM2COL_3D) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t s2 = tensor->op_params[2];+ const int32_t p0 = tensor->op_params[3];+ const int32_t p1 = tensor->op_params[4];+ const int32_t p2 = tensor->op_params[5];+ const int32_t d0 = tensor->op_params[6];+ const int32_t d1 = tensor->op_params[7];+ const int32_t d2 = tensor->op_params[8];+ const int32_t IC = tensor->op_params[9];++ tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type);+ } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) {+ const int32_t dim = tensor->op_params[0];+ const int32_t max_period = tensor->op_params[1];+ tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period);+ } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){+ const int32_t s0 = tensor->op_params[0];+ const int32_t p0 = tensor->op_params[1];+ const int32_t d0 = tensor->op_params[2];+ tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0);+ } else if (tensor->op == GGML_OP_COL2IM_1D) {+ const int32_t stride = tensor->op_params[0];+ const int32_t oc = tensor->op_params[1];+ const int32_t p0 = tensor->op_params[2];+ tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0);+ } else if (tensor->op == GGML_OP_POOL_1D) {+ enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);+ const int32_t k0 = tensor->op_params[1];+ const int32_t s0 = tensor->op_params[2];+ const int32_t p0 = tensor->op_params[3];++ tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0);+ } else if (tensor->op == GGML_OP_POOL_2D) {+ enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);+ const int32_t k0 = tensor->op_params[1];+ const int32_t k1 = tensor->op_params[2];+ const int32_t s0 = tensor->op_params[3];+ const int32_t s1 = tensor->op_params[4];+ const int32_t p0 = tensor->op_params[5];+ const int32_t p1 = tensor->op_params[6];++ tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1);+ } else if (tensor->op == GGML_OP_CONV_2D) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t p0 = tensor->op_params[2];+ const int32_t p1 = tensor->op_params[3];+ const int32_t d0 = tensor->op_params[4];+ const int32_t d1 = tensor->op_params[5];+ tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);+ } else if (tensor->op == GGML_OP_CONV_3D) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t s2 = tensor->op_params[2];+ const int32_t p0 = tensor->op_params[3];+ const int32_t p1 = tensor->op_params[4];+ const int32_t p2 = tensor->op_params[5];+ const int32_t d0 = tensor->op_params[6];+ const int32_t d1 = tensor->op_params[7];+ const int32_t d2 = tensor->op_params[8];+ const int32_t IC = tensor->op_params[9];+ const int32_t N = tensor->op_params[10];+ const int32_t OC = tensor->op_params[11];+ tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC);+ } else if (tensor->op == GGML_OP_CONV_2D_DW) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t p0 = tensor->op_params[2];+ const int32_t p1 = tensor->op_params[3];+ const int32_t d0 = tensor->op_params[4];+ const int32_t d1 = tensor->op_params[5];+ tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);+ } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) {+ const int32_t s = tensor->op_params[0];+ tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s);+ } else if (tensor->op == GGML_OP_LEAKY_RELU) {+ const float * op_params = (const float *)tensor->op_params;+ tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false);+ } else if (tensor->op == GGML_OP_RWKV_WKV6) {+ tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1],+ src_clone[2], src_clone[3], src_clone[4], src_clone[5]);+ } else if (tensor->op == GGML_OP_RWKV_WKV7) {+ tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3],+ src_clone[4], src_clone[5], src_clone[6]);+ } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) {+ const float * op_params = (const float *)tensor->op_params;+ tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],+ src_clone[2], src_clone[3], src_clone[4], op_params[0]);+ } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) {+ tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);+ } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {+ tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],+ src_clone[2], src_clone[3], src_clone[4], src_clone[5],+ ggml_get_op_params_i32(tensor, 0));+ } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) {+ src_clone[0]->flags = tensor->src[0]->flags;+ tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1],+ src_clone[2], src_clone[3], src_clone[4]);+ } else if (tensor->op == GGML_OP_OPT_STEP_SGD) {+ src_clone[0]->flags = tensor->src[0]->flags;+ tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1],+ src_clone[2]);+ } else if (tensor->op == GGML_OP_ADD_ID) {+ tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);+ } else if (tensor->op == GGML_OP_SSM_SCAN) {+ const int32_t K = ggml_get_op_params_i32(tensor, 0);+ tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],+ src_clone[3], src_clone[4], src_clone[5], src_clone[6], K);+ } else if (tensor->op == GGML_OP_SSM_CONV) {+ tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]);+ } else if (tensor->op == GGML_OP_ROLL) {+ const int32_t s0 = tensor->op_params[0];+ const int32_t s1 = tensor->op_params[1];+ const int32_t s2 = tensor->op_params[2];+ const int32_t s3 = tensor->op_params[3];+ tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3);+ }+ else {+ std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;+ GGML_ABORT("fatal error");+ }+ cloned_tensors[tensor] = tensor_clone;+ }++ ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx);+ ggml_build_forward_expand(cgraph_cpu, tensor_clone);++ ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8);++ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {+ ggml_vk_print_tensor(tensor_clone, "tensor_clone");+ }++ comp_size = ggml_nbytes(tensor_clone);++ comp_result = malloc(comp_size);+ memcpy(comp_result, tensor_clone->data, comp_size);+ memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS);++ for (auto m : cloned_mallocs) {+ free(m);+ }++ ggml_free(ggml_ctx);++ VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")");+}++static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {+ ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];+ if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {+ return;+ }++ if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {+ return;+ }++ VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")");++ ggml_tensor * src0 = tensor->src[0];+ ggml_tensor * src1 = tensor->src[1];+ ggml_tensor * src2 = tensor->src[2];+ ggml_tensor * src3 = tensor->src[3];++ void * tensor_data = tensor->data;++ if (ggml_backend_buffer_is_vk(tensor->buffer)) {+ size_t tensor_size = ggml_nbytes(tensor);+ tensor_data = malloc(tensor_size);++ ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;++ vk_buffer& buffer_gpu = buf_ctx->dev_buffer;+ uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs;+ if (offset + tensor_size >= buffer_gpu->size) {+ tensor_size = buffer_gpu->size - offset;+ }++ ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size);+ }++ float first_error_result = -1.0f;+ float first_error_correct = -1.0f;+ std::array<int, 4> first_error = { -1, -1, -1, -1 };+ double avg_err = 0.0;+ size_t counter = 0;++ for (int i3 = 0; i3 < tensor->ne[3]; i3++) {+ for (int i2 = 0; i2 < tensor->ne[2]; i2++) {+ for (int i1 = 0; i1 < tensor->ne[1]; i1++) {+ for (int i0 = 0; i0 < tensor->ne[0]; i0++) {+ const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size;+ float correct = 0.0f;+ float result = 0.0f;++ if (buffer_size_fit) {+ if (tensor->type == GGML_TYPE_F32) {+ correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);+ result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);+ } else if (tensor->type == GGML_TYPE_F16) {+ correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));+ result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));+ } else if (tensor->type == GGML_TYPE_BF16) {+ correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));+ result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));+ } else if (tensor->type == GGML_TYPE_I32) {+ correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);+ result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);+ } else if (tensor->type == GGML_TYPE_I64) {+ correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);+ result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);+ } else {+ std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl;+ }+ } else {+ std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl;+ GGML_ABORT("fatal error");+ }++ if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) {+ std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl;+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;+ if (src0 != nullptr) {+ std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;+ }+ if (src1 != nullptr) {+ std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;+ }+ if (src2 != nullptr) {+ std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;+ }+ if (src3 != nullptr) {+ std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;+ }+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;+ std::cerr << std::endl << "Result:" << std::endl;+ ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3);+ std::cerr << std::endl << "Correct:" << std::endl;+ ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3);+ std::cerr << std::endl;+ std::vector<const ggml_tensor *> done;+ ggml_vk_print_graph_origin(tensor, done);+ GGML_ABORT("fatal error");+ }+ const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f;+ if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) {+ first_error[0] = i0;+ first_error[1] = i1;+ first_error[2] = i2;+ first_error[3] = i3;+ first_error_result = result;+ first_error_correct = correct;+ }++ // Special case, value is infinite, avoid NaN result in avg_err+ // NaN also appears in results, if both are nan error is 0+ if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) {+ avg_err += std::fabs(correct - result) / denom;+ }+ counter++;+ }+ }+ }+ }++ avg_err /= counter;++ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {+ std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;+ if (src0 != nullptr) {+ std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;+ }+ if (src1 != nullptr) {+ std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;+ }+ if (src2 != nullptr) {+ std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;+ }+ if (src3 != nullptr) {+ std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;+ }+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;+ std::cerr << std::endl << "Result:" << std::endl;+ ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);+ std::cerr << std::endl << "Correct:" << std::endl;+ ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0);+ std::cerr << std::endl;+ std::vector<const ggml_tensor *> done;+ ggml_vk_print_graph_origin(tensor, done);+ }++ if (avg_err > 0.01 || std::isnan(avg_err)) {+ std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;+ std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;+ if (src0 != nullptr) {+ std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;+ }+ if (src1 != nullptr) {+ std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;+ }+ if (src2 != nullptr) {+ std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;+ }+ if (src3 != nullptr) {+ std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;+ }+ std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;+ std::cerr << std::endl << "Result:" << std::endl;+ ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]);+ std::cerr << std::endl << "Correct:" << std::endl;+ ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]);+ std::cerr << std::endl;+ std::vector<const ggml_tensor *> done;+ ggml_vk_print_graph_origin(tensor, done);+ GGML_ABORT("fatal error");+ } else {+ std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl;+ }++ free(comp_result);+ comp_result = nullptr;+ comp_size = 0;++ if (ggml_backend_buffer_is_vk(tensor->buffer)) {+ free(tensor_data);+ }++ VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")");+}+#endif+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.hnew file mode 100644index 0000000000..68b3200b3e--- /dev/null+++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h@@ -0,0 +1,1092 @@+#pragma once+#include "ggml-vulkan-types.h"++uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);++uint32_t ggml_vk_concat_unit_size(ggml_type type);++struct vk_mat_mat_push_constants {+ uint32_t M; uint32_t N; uint32_t K;+ uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;+ uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;+ uint32_t base_work_group_z; uint32_t num_batches;+ uint32_t k_split;+ uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;+ uint32_t padded_N;+};++struct vk_mat_vec_push_constants {+ uint32_t ncols;+ uint32_t stride_a;+ uint32_t stride_b;+ uint32_t stride_d;+ uint32_t batch_stride_a;+ uint32_t batch_stride_b;+ uint32_t batch_stride_d;+ uint32_t fusion_flags;+ uint32_t base_work_group_y;+ uint32_t ne02;+ uint32_t ne12;+ uint32_t broadcast2;+ uint32_t broadcast3;+};++struct vk_mat_vec_p021_push_constants {+ uint32_t ncols_x;+ uint32_t nrows_x;+ uint32_t nchannels_x;+ uint32_t nchannels_y;+ uint32_t b_offset;+ uint32_t d_offset;+ uint32_t fusion_flags;+};++struct vk_mat_vec_nc_push_constants {+ uint32_t ncols_x;+ uint32_t nrows_x;+ uint32_t row_stride_x;+ uint32_t channel_stride_x;+ uint32_t channel_stride_y;+ uint32_t channel_x_divisor;+ uint32_t ne12;+ uint32_t b_offset;+ uint32_t d_offset;+ uint32_t nb03;+ uint32_t nb13;+ uint32_t nb23;+ uint32_t fusion_flags;+};++struct vk_mat_mat_id_push_constants {+ uint32_t M; uint32_t N; uint32_t K;+ uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;+ uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;+ uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;+ uint32_t n_experts;+ uint32_t hoist_row_ids;+};++struct vk_mat_vec_id_push_constants {+ uint32_t ncols;+ uint32_t stride_a;+ uint32_t stride_b;+ uint32_t stride_d;+ uint32_t batch_stride_a;+ uint32_t batch_stride_b;+ uint32_t batch_stride_d;+ uint32_t fusion_flags;+ uint32_t nei0;+ uint32_t ne11;+ uint32_t expert_i1;+ uint32_t nbi1;+};++struct vk_flash_attn_push_constants {+ uint32_t N;+ uint32_t KV;++ uint32_t ne1;+ uint32_t ne2;+ uint32_t ne3;++ uint32_t neq2;+ uint32_t neq3;+ uint32_t nek2;+ uint32_t nek3;+ uint32_t nev2;+ uint32_t nev3;+ uint32_t nem1;+ uint32_t nem2;+ uint32_t nem3;++ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb03;+ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;+ uint32_t nb21;+ uint32_t nb22;+ uint32_t nb23;++ float scale;+ float max_bias;+ float logit_softcap;++ uint32_t mask_n_head_log2;+ float m0;+ float m1;++ uint32_t gqa_ratio;+ uint32_t split_kv;+ uint32_t k_num;+};++static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128");++struct vk_op_push_constants {+ uint32_t KX;+ uint32_t KY;+ float param1;+ float param2;+ float param3;+ float param4;+};++struct vk_op_fwht_push_constants {+ uint32_t n_rows;+ uint32_t src_offset;+ uint32_t dst_offset;+ float scale;+};++struct vk_op_dsv4_hc_comb_push_constants {+ uint32_t n_tokens;++ uint32_t nbm0; uint32_t nbm1;+ uint32_t nbs0;+ uint32_t nbb0;+ uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;++ uint32_t m_offset;+ uint32_t s_offset;+ uint32_t b_offset;+ uint32_t d_offset;++ float eps;+ uint32_t n_iter;+};++struct vk_op_dsv4_hc_pre_push_constants {+ uint32_t n_embd;+ uint32_t n_tokens;++ uint32_t nbx0; uint32_t nbx1; uint32_t nbx2;+ uint32_t nbw0; uint32_t nbw1; uint32_t nbw2;+ uint32_t nbd0; uint32_t nbd1;++ uint32_t x_offset;+ uint32_t w_offset;+ uint32_t d_offset;++ float scale;+};++struct vk_op_dsv4_hc_post_push_constants {+ uint32_t n_embd;+ uint32_t n_tokens;++ uint32_t nbx0; uint32_t nbx1;+ uint32_t nbr0; uint32_t nbr1; uint32_t nbr2;+ uint32_t nbp0; uint32_t nbp1;+ uint32_t nbc0; uint32_t nbc1; uint32_t nbc2;+ uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;++ uint32_t x_offset;+ uint32_t r_offset;+ uint32_t p_offset;+ uint32_t c_offset;+ uint32_t d_offset;+};++struct vk_op_count_experts_push_constants {+ uint32_t ne00;+ uint32_t ne01;+ uint32_t nb00;+ uint32_t nb01;+ uint32_t a_offset;+ uint32_t n_experts;+ uint32_t hoist_row_ids;+ uint32_t ne00mp;+ uint32_t ne00L;+};++struct vk_op_glu_push_constants {+ uint32_t N;+ uint32_t ne00;+ uint32_t ne20;+ uint32_t mode; // 0: default, 1: swapped, 2: split+ float alpha; // for swiglu_oai+ float limit;+ uint32_t nb00;+ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb03;+ uint32_t nb10;+ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;+ uint32_t nb20;+ uint32_t nb21;+ uint32_t nb22;+ uint32_t nb23;+ uint32_t ne21;+ uint32_t ne22;+ uint32_t misalign_offsets;+ uint32_t ne2_012mp; uint32_t ne2_012L;+ uint32_t ne2_01mp; uint32_t ne2_01L;+ uint32_t ne2_0mp; uint32_t ne2_0L;+};++static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128");++struct vk_op_unary_push_constants {+ uint32_t ne;+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;+ uint32_t misalign_offsets;+ float param1; float param2; float param3; float param4;+ uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls;+ uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls;+};++static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");++static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) {+ GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst)));+ ne = ne != 0 ? ne : ggml_nelements(dst);+ GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());++ vk_op_unary_push_constants p{};+ p.ne = (uint32_t)ne;++ size_t src0_tsize = ggml_type_size(src0->type);+ p.ne00 = (uint32_t)src0->ne[0];+ p.ne01 = (uint32_t)src0->ne[1];+ p.ne02 = (uint32_t)src0->ne[2];+ p.ne03 = (uint32_t)src0->ne[3];+ p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);+ p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);+ p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);+ p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);++ size_t dst_tsize = ggml_type_size(dst->type);+ p.ne10 = (uint32_t)dst->ne[0];+ p.ne11 = (uint32_t)dst->ne[1];+ p.ne12 = (uint32_t)dst->ne[2];+ p.ne13 = (uint32_t)dst->ne[3];+ p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);+ p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);+ p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);+ p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);++ return p; // offsets are initialized later in ggml_vk_op+}++struct vk_op_pad_push_constants {+ uint32_t ne;+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;+ uint32_t misalign_offsets;+ uint32_t circular;++ uint32_t lp0; uint32_t rp0;+ uint32_t lp1; uint32_t rp1;+ uint32_t lp2; uint32_t rp2;+ uint32_t lp3; uint32_t rp3;+};++static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) {+ int64_t ne = ggml_nelements(dst);+ GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());++ vk_op_pad_push_constants p{};+ p.ne = (uint32_t)ne;++ size_t src0_tsize = ggml_type_size(src0->type);+ p.ne00 = (uint32_t)src0->ne[0];+ p.ne01 = (uint32_t)src0->ne[1];+ p.ne02 = (uint32_t)src0->ne[2];+ p.ne03 = (uint32_t)src0->ne[3];+ p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);+ p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);+ p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);+ p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);++ size_t dst_tsize = ggml_type_size(dst->type);+ p.ne10 = (uint32_t)dst->ne[0];+ p.ne11 = (uint32_t)dst->ne[1];+ p.ne12 = (uint32_t)dst->ne[2];+ p.ne13 = (uint32_t)dst->ne[3];+ p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);+ p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);+ p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);+ p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);++ p.lp0 = dst->op_params[0];+ p.rp0 = dst->op_params[1];+ p.lp1 = dst->op_params[2];+ p.rp1 = dst->op_params[3];+ p.lp2 = dst->op_params[4];+ p.rp2 = dst->op_params[5];+ p.lp3 = dst->op_params[6];+ p.rp3 = dst->op_params[7];+ p.circular = dst->op_params[8];++ return p; // fastdiv values and offsets are initialized later in ggml_vk_op+}++static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)+{+ // compute L = ceil(log2(d));+ L = 0;+ while (L < 32 && (uint32_t{1} << L) < d) {+ L++;+ }++ mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);+}++static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) {+ return L0 | (L1 << 8) | (L2 << 16);+}++template <typename T> void init_pushconst_fastdiv(T &p) {+ GGML_UNUSED(p);+ static_assert(!std::is_const<T>::value, "unexpected type");+}++template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {+ // Compute magic values to divide by these six numbers.+ uint32_t ne0_012L;+ uint32_t ne0_01L;+ uint32_t ne0_0L;+ uint32_t ne1_012L;+ uint32_t ne1_01L;+ uint32_t ne1_0L;++ init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L);+ init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L);+ init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L);+ init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L);+ init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L);+ init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L);++ p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L);+ p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L);+}++template <> inline void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {+ // GLU linearizes over dst, then uses dst coordinates for src0/src1.+ init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L);+ init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L);+ init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);+}++template <> inline void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) {+ init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L);+}++struct vk_op_binary_push_constants {+ uint32_t ne;+ uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;+ uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23;+ uint32_t misalign_offsets;+ float param1; float param2; int32_t param3;+};++struct vk_op_concat_push_constants : vk_op_binary_push_constants {};++static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));++static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);++struct vk_op_multi_add_push_constants {+ // shape for dst+ uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;++ // strides for srcs+dst+ uint32_t nb[MAX_PARAMETER_COUNT][4];++ uint32_t rms_partials;+};++static_assert(MAX_PARAMETER_COUNT == 12);++static_assert(sizeof(vk_op_multi_add_push_constants) <= 256);++struct vk_op_topk_moe_push_constants {+ uint32_t n_rows;+ uint32_t n_experts_push;+ uint32_t n_expert_used;+ float clamp_min;+ float clamp_max;+ uint32_t gating_func;+ uint32_t has_bias;+ uint32_t with_norm;+ float output_scale;+ float output_bias;+};++struct vk_op_add_id_push_constants {+ uint32_t ne0;+ uint32_t ne1;+ uint32_t s01;+ uint32_t s02;+ uint32_t s11;+ uint32_t s21;+};++struct vk_op_diag_mask_push_constants {+ uint32_t ncols;+ uint32_t rows_per_channel;+ int32_t n_past;+};++struct vk_op_rope_push_constants {+ uint32_t rope_mode;+ uint32_t nrows;+ uint32_t n_dims;+ uint32_t n_offs;+ float freq_scale;+ float freq_base;+ float ext_factor;+ float attn_factor;+ float corr_dims[2];+ float theta_scale;+ uint32_t has_ff;+ int32_t sections[4];+ uint32_t is_imrope;+ uint32_t is_back;+ uint32_t set_rows_stride;+ uint32_t ne00;+ uint32_t ne01;+ uint32_t ne02;+ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb03;+ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;+ uint32_t a_offset;+ uint32_t d_offset;+};++static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128");++struct vk_op_rms_norm_mul_rope_push_constants {+ vk_op_binary_push_constants bin;+ vk_op_rope_push_constants rope;+};++struct vk_op_soft_max_push_constants {+ uint32_t KX;+ uint32_t KY;+ uint32_t ne00;+ uint32_t ne01;+ uint32_t ne02;+ uint32_t ne12;+ uint32_t ne13;+ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;+ float scale;+ float max_bias;+ float m0;+ float m1;+ uint32_t n_head_log2;+ uint32_t nrows_x;+ uint32_t has_sinks;+};++struct vk_op_argsort_push_constants {+ uint32_t ncols;+ uint32_t ncols_padded;+ uint32_t ncols_padded_log2;+ uint32_t nrows;+ uint32_t order;+ uint32_t outer_start;+ uint32_t outer_end;+ uint32_t inner_start;+ uint32_t inner_end;+};++struct vk_op_topk_push_constants {+ uint32_t orig_ncols;+ uint32_t ncols_input;+ uint32_t ncols_output;+ uint32_t k;+ uint32_t nrows;+ uint32_t first_pass;+ uint32_t last_pass;+};++struct vk_op_topk_radix_push_constants {+ uint32_t ncols;+ uint32_t k;+ uint32_t nrows;+ uint32_t n_tps; // QSA only+ uint32_t n_blocks; // QSA only+ uint32_t n_stream; // QSA only+};++struct vk_op_im2col_push_constants {+ uint64_t dst_addr;+ uint32_t batch_offset; uint32_t offset_delta;+ uint32_t IC;+ uint32_t IW; uint32_t IH;+ uint32_t OW; uint32_t OH;+ uint32_t KW; uint32_t KH;+ uint32_t OH_batch;+ uint32_t CHW;+ int32_t s0; int32_t s1;+ int32_t p0; int32_t p1;+ int32_t d0; int32_t d1;+ uint32_t batch_IC;+};++struct vk_op_im2col_3d_push_constants {+ uint64_t dst_addr;+ uint32_t nb10;+ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;+ uint32_t s0;+ uint32_t s1;+ uint32_t s2;+ uint32_t p0;+ uint32_t p1;+ uint32_t p2;+ uint32_t d0;+ uint32_t d1;+ uint32_t d2;+ uint32_t IW;+ uint32_t IH;+ uint32_t ID;+ uint32_t IC;+ uint32_t KW;+ uint32_t OH;+ uint32_t KD_KH_KW;+ uint32_t KH_KW;+ uint32_t IC_KD_KH_KW;+ uint32_t N_OD_OH;+ uint32_t OD_OH;+ uint32_t OD_OH_OW_IC_KD_KH_KW;+ uint32_t OH_OW_IC_KD_KH_KW;+ uint32_t OW_IC_KD_KH_KW;+ uint32_t misalign_offsets;+};++struct vk_op_timestep_embedding_push_constants {+ uint32_t nb1;+ uint32_t dim;+ uint32_t max_period;+};++struct vk_op_col2im_1d_push_constants {+ uint32_t T_out;+ uint32_t OC;+ uint32_t K_OC;+ uint32_t T_in;+ uint32_t K;+ int32_t stride;+ int32_t p0;+};++struct vk_op_conv_transpose_1d_push_constants {+ uint32_t Cout;+ uint32_t Cin;+ uint32_t K;+ uint32_t L;+ uint32_t KL;++ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb11;+ uint32_t nb1;++ int32_t s0;+};++struct vk_op_snake_push_constants {+ uint32_t ne0;+ uint32_t ne1;+};++struct vk_op_pool1d_push_constants {+ uint32_t IL;+ uint32_t OL;+ uint32_t OC;+ uint32_t pelements;+ uint32_t op;+ int32_t k0;+ int32_t s0;+ int32_t p0;+};++struct vk_op_pool2d_push_constants {+ uint32_t IW; uint32_t IH;+ uint32_t OW; uint32_t OH;+ uint32_t OC;+ uint32_t pelements;+ uint32_t op;+ int32_t k0; int32_t k1;+ int32_t s0; int32_t s1;+ int32_t p0; int32_t p1;+};++struct vk_op_rwkv_wkv6_push_constants {+ uint32_t B;+ uint32_t T;+ uint32_t C;+ uint32_t H;+};++struct vk_op_rwkv_wkv7_push_constants {+ uint32_t B;+ uint32_t T;+ uint32_t C;+ uint32_t H;+};++struct vk_op_gated_linear_attn_push_constants {+ uint32_t B;+ uint32_t T;+ uint32_t C;+ uint32_t H;+ float scale;+};++struct vk_op_lightning_indexer_push_constants {+ uint32_t n_kv;+ uint32_t n_heads;+ uint32_t n_tokens;+ uint32_t n_streams;+ uint32_t n_masks;+ uint32_t dispatch_x;+ uint32_t q_nb1;+ uint32_t q_nb2;+ uint32_t q_nb3;+ uint32_t k_nb2;+ uint32_t k_nb3;+ uint32_t w_nb1;+ uint32_t w_nb3;+ uint32_t m_nb1;+ uint32_t m_nb3;+ uint32_t d_nb1;+ uint32_t d_nb3;+};++static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128);++struct vk_op_gated_delta_net_push_constants {+ uint32_t H;+ uint32_t n_tokens;+ uint32_t n_seqs;+ uint32_t s_off;+ uint32_t sq1, sq2, sq3;+ uint32_t sv1, sv2, sv3;+ uint32_t sb1, sb2, sb3;+ uint32_t neq1, rq3;+ float scale;+ uint32_t K;+};++struct vk_op_ssm_scan_push_constants {+ uint32_t nb02, nb03, nb12, nb13;+ uint32_t nb21, nb22, nb31;+ uint32_t nb42, nb43, nb52, nb53;+ uint32_t s_off;+ uint32_t n_head, d_head, n_group, n_tok;+ uint32_t n_seq, K;+};++struct vk_op_ssm_conv_push_constants {+ uint32_t nb01, nb02;+ uint32_t nb11;+ uint32_t dst_nb0, dst_nb1, dst_nb2;+ uint32_t nc, ncs, nr, n_t, n_s;+};++struct vk_op_conv2d_push_constants {+ uint32_t Cout;+ uint32_t Cin;+ uint32_t N;++ uint32_t W;+ uint32_t H;+ uint32_t OW;+ uint32_t OH;++ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb03;++ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;++ uint32_t nb1;+ uint32_t nb2;+ uint32_t nb3;++ // init_fastdiv_values constants for dividing by OW, OW*OH+ uint32_t OWmp; uint32_t OWL;+ uint32_t OWOHmp; uint32_t OWOHL;+};++template <> inline void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) {+ // Compute magic values to divide by OW, OW*OH+ init_fastdiv_values(p.OW, p.OWmp, p.OWL);+ init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);+}++struct vk_op_conv3d_push_constants {+ uint32_t OC;+ uint32_t IC;+ uint32_t N;++ uint32_t IW;+ uint32_t IH;+ uint32_t ID;+ uint32_t OW;+ uint32_t OH;+ uint32_t OD;++ uint32_t nb01;+ uint32_t nb02;+ uint32_t nb03;++ uint32_t nb11;+ uint32_t nb12;+ uint32_t nb13;++ uint32_t nb1;+ uint32_t nb2;+ uint32_t nb3;++ uint32_t OWmp; uint32_t OWL;+ uint32_t OWOHmp; uint32_t OWOHL;+ uint32_t OWOHODmp; uint32_t OWOHODL;+};++template <> inline void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) {+ init_fastdiv_values(p.OW, p.OWmp, p.OWL);+ init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);+ init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL);+}++struct vk_op_conv2d_dw_push_constants {+ uint32_t ne;+ uint32_t batches;+ uint32_t channels;+ uint32_t dst_w;+ uint32_t dst_h;+ uint32_t src_w;+ uint32_t src_h;+ uint32_t knl_w;+ uint32_t knl_h;+ int32_t stride_x;+ int32_t stride_y;+ int32_t pad_x;+ int32_t pad_y;+ int32_t dilation_x;+ int32_t dilation_y;+};++struct vk_op_upscale_push_constants {+ uint32_t ne; uint32_t a_offset; uint32_t d_offset;+ uint32_t ne00; uint32_t ne01;+ uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;+ uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13;+ float sf0; float sf1; float sf2; float sf3;+ float pixel_offset;+};++struct vk_op_sum_rows_push_constants+{+ uint32_t n_cols;+ uint32_t ne01, ne02;+ uint32_t nb01, nb02, nb03;+ uint32_t nb11, nb12, nb13;+ float weight;+ uint32_t misalign_offsets;+ uint32_t ne0_12mp, ne0_12L;+ uint32_t ne0_1mp, ne0_1L;+};++static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) {+ uint32_t type_size = (uint32_t)ggml_type_size(src->type);+ vk_op_sum_rows_push_constants p = {};+ p.n_cols = (uint32_t)n_cols;+ p.ne01 = (uint32_t)src->ne[1];+ p.ne02 = (uint32_t)src->ne[2];+ p.nb01 = (uint32_t)src->nb[1] / type_size;+ p.nb02 = (uint32_t)src->nb[2] / type_size;+ p.nb03 = (uint32_t)src->nb[3] / type_size;+ p.nb11 = (uint32_t)dst->nb[1] / type_size;+ p.nb12 = (uint32_t)dst->nb[2] / type_size;+ p.nb13 = (uint32_t)dst->nb[3] / type_size;+ p.weight = 1.0f;+ return p;+}++template <> inline void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) {+ init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L);+ init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L);+}++struct vk_quantize_q8_1_push_constants {+ uint32_t ne;+ uint32_t num_blocks;+};++struct vk_op_flash_attn_split_k_reduce_push_constants {+ uint32_t D;+ uint32_t ne1;+ uint32_t ne2;+ uint32_t ne3;+ uint32_t k_num;+ uint32_t sinks;+};++struct vk_op_flash_attn_mask_opt_push_constants {+ uint32_t nem0;+ uint32_t nem1;+ uint32_t nem2;+ uint32_t nbm1;+ uint32_t nbm2;+ uint32_t nbm3;+ uint32_t nbd1;+ uint32_t nbd2;+ uint32_t nbd3;+};++struct vk_op_flash_attn_sparse_compact_push_constants {+ uint32_t KV;+ uint32_t nem1;+ uint32_t nem2;+ uint32_t nbm1;+ uint32_t nbm2;+ uint32_t nbm3;+ uint32_t n_kv_max;+};++template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ GGML_UNUSED(p);+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+ GGML_UNUSED(dst);+ static_assert(!std::is_const<T>::value, "unexpected type");+ GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0);+ GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0);+ GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0);+ GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0);+ GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.b_offset = b_offset;+ p.d_offset = d_offset;++ GGML_UNUSED(src0);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.b_offset = b_offset;+ p.d_offset = d_offset;++ GGML_UNUSED(src0);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_comb_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ p.m_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ p.s_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ p.b_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_pre_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ p.w_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_post_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ p.r_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ p.p_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);+ p.c_offset = src3 ? get_misalign_bytes(ctx, src3) / ggml_type_size(src3->type) : 0;+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);+}++template <typename T> size_t push_constant_size(const T &t) {+ static_assert(std::is_class<T>::value, "T must be a struct/class");+ GGML_UNUSED(t);+ return sizeof(T);+}++template <typename T> size_t push_constant_size(const std::vector<T> &t) {+ GGML_UNUSED(t);+ return sizeof(T) * t.size();+}++template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) {+ GGML_UNUSED(t);+ return sizeof(T) * N;+}++template <typename T> const T *push_constant_data(const T &t) {+ static_assert(std::is_class<T>::value, "T must be a struct/class");+ return &t;+}++template <typename T> const T *push_constant_data(const std::vector<T> &t) {+ return t.data();+}++template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) {+ return t.data();+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.misalign_offsets = (a_offset << 16) | d_offset;++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset;+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_ASSERT(a_offset < (1u << 8));+ GGML_ASSERT(b_offset < (1u << 8));+ GGML_ASSERT(d_offset < (1u << 8));++ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;++ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.misalign_offsets = (a_offset << 16) | d_offset;++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.misalign_offsets = (a_offset << 16) | d_offset;++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.misalign_offsets = (a_offset << 16) | d_offset;++ GGML_UNUSED(src0);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_ASSERT(a_offset <= 0xFFFF);+ GGML_ASSERT(b_offset <= 0xFF);+ GGML_ASSERT(d_offset <= 0xFF);++ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;++ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;+ const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;++ p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;++ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ p.a_offset = a_offset;+ p.d_offset = d_offset;++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++template <> inline void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {+ p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);+ p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);++ GGML_UNUSED(src1);+ GGML_UNUSED(src2);+ GGML_UNUSED(src3);+}++static vk_op_binary_push_constants ggml_vk_rms_norm_push_constants(+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst,+ float eps, uint32_t num_partials) {+ const uint32_t src0_type_size = ggml_type_size(src0->type);+ const uint32_t src1_type_size = ggml_type_size(src1->type);+ const uint32_t dst_type_size = ggml_type_size(dst->type);++ return {+ (uint32_t)ggml_nelements(src0),+ (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,+ (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,+ (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,+ 0,+ eps, 0.0f, (int32_t)num_partials,+ };+}+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.hnew file mode 100644index 0000000000..67e3361ed3--- /dev/null+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h@@ -0,0 +1,1432 @@+#pragma once++#include "ggml-vulkan.h"++#include <vulkan/vulkan_core.h>++#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS)+#include <chrono>+#include "ggml-cpu.h"+#endif++#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1++#if VK_HEADER_VERSION >= 301+namespace vk::detail { class DispatchLoaderDynamic; }+using vk::detail::DispatchLoaderDynamic;+#else+namespace vk { class DispatchLoaderDynamic; }+using vk::DispatchLoaderDynamic;+#endif++DispatchLoaderDynamic & ggml_vk_default_dispatcher();++#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher()++#include <vulkan/vulkan.hpp>++#ifndef VK_NV_cooperative_matrix_decode_vector+#define VK_NV_cooperative_matrix_decode_vector 1+#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000)+typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {+ VkStructureType sType;+ void* pNext;+ VkBool32 cooperativeMatrixDecodeVector;+} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;+#endif++#if __has_include(<spirv/unified1/spirv.hpp>)+# include <spirv/unified1/spirv.hpp>+#elif __has_include(<spirv-headers/spirv.hpp>)+# include <spirv-headers/spirv.hpp>+#elif __has_include(<spirv.hpp>)+# include <spirv.hpp>+#else+ // Fallback to let the compiler throw a standard "file not found" error+# include <spirv/unified1/spirv.hpp>+#endif++#include <algorithm>++#include <cmath>++#include <iomanip>++#include <iostream>++#include <tuple>++#include <vector>++#include <deque>++#include <sstream>++#include <utility>++#include <memory>++#include <limits>++#include <map>++#include <set>++#include <unordered_map>++#include <shared_mutex>++#include <mutex>++#include <future>++#include <condition_variable>++#include <thread>++#if defined(_MSC_VER)+# define NOMINMAX 1+# include <windows.h>+# define YIELD() YieldProcessor()+#elif defined(__clang__) || defined(__GNUC__)+# if defined(__x86_64__) ||defined(__i386__)+# include <immintrin.h>+# define YIELD() _mm_pause()+# elif defined(__arm__) || defined(__aarch64__)+# if defined(__clang__)+# include <arm_acle.h>+# define YIELD() __yield()+# else+# define YIELD() asm volatile("yield")+# endif+# endif+#endif++#if !defined(YIELD)+#define YIELD()+#endif++#include "ggml-impl.h"++#include "ggml-backend-impl.h"++#include "ggml-vulkan-shaders.hpp"++#if !defined(VK_KHR_shader_bfloat16)++#define VK_KHR_shader_bfloat16 1+#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1+#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)+#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)++typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {+ VkStructureType sType;+ void* pNext;+ VkBool32 shaderBFloat16Type;+ VkBool32 shaderBFloat16DotProduct;+ VkBool32 shaderBFloat16CooperativeMatrix;+} VkPhysicalDeviceShaderBfloat16FeaturesKHR;+#endif++#if !defined(VK_VALVE_shader_mixed_float_dot_product)+#define VK_VALVE_shader_mixed_float_dot_product 1+#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1+#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000)+typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE {+ VkStructureType sType;+ void* pNext;+ VkBool32 shaderMixedFloatDotProductFloat16AccFloat32;+ VkBool32 shaderMixedFloatDotProductFloat16AccFloat16;+ VkBool32 shaderMixedFloatDotProductBFloat16Acc;+ VkBool32 shaderMixedFloatDotProductFloat8AccFloat32;+} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE;+#endif++#if !defined(VK_EXT_shader_ocp_microscaling_types)+#define VK_EXT_shader_ocp_microscaling_types 1+#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1+#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000)+typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT {+ VkStructureType sType;+ void* pNext;+ VkBool32 shaderFloat4;+ VkBool32 shaderFloat6;+ VkBool32 shaderFloat8UnsignedE8M0;+ VkBool32 shaderMXInt8;+} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT;+#endif++#if !defined(VK_EXT_shader_float8)+#define VK_EXT_shader_float8 1+#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1+#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000)+typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT {+ VkStructureType sType;+ void* pNext;+ VkBool32 shaderFloat8;+ VkBool32 shaderFloat8CooperativeMatrix;+} VkPhysicalDeviceShaderFloat8FeaturesEXT;+#endif++#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME+#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues"+#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000)+#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004)++// Compile-time constant guaranteed; no runtime initialization overhead+static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR =+ static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004);++typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR {+ VkStructureType sType;+ void* pNext;+ VkBool32 internallySynchronizedQueues;+} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR;+#else+static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR;+#endif++#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1))++#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0))++static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }++#define VK_VENDOR_ID_AMD 0x1002++#define VK_VENDOR_ID_APPLE 0x106b++#define VK_VENDOR_ID_INTEL 0x8086++#define VK_VENDOR_ID_NVIDIA 0x10de++#define VK_VENDOR_ID_QUALCOMM 0x5143++#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256++#define VK_CHECK(err, msg, dev) \+ do { \+ vk::Result err_; \+ try { \+ err_ = (err); \+ } catch (vk::DeviceLostError &) { \+ ggml_vk_print_device_lost_info(dev); \+ GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \+ #err, __FILE__, __LINE__); \+ throw; \+ } \+ if (err_ != vk::Result::eSuccess) { \+ GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \+ #err, to_string(err_).c_str(), __FILE__, __LINE__); \+ throw vk::SystemError(vk::make_error_code(err_), \+ "ggml_vulkan: " msg); \+ } \+ } while (0)++#ifdef GGML_VULKAN_DEBUG+#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl+#else+#define VK_LOG_DEBUG(msg) ((void) 0)+#endif // GGML_VULKAN_DEBUG++#define MAX_PARAMETER_COUNT 12++#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)++struct vk_pipeline_struct;++typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;++struct vk_pipeline_struct {+ std::string name;+ vk::ShaderModule shader_module;+ vk::PipelineLayout layout;+ vk::Pipeline pipeline;+ uint32_t push_constant_size;+ uint32_t parameter_count;+ std::array<uint32_t, 3> wg_denoms;+ uint32_t align;+ // true if fields have been set by ggml_vk_create_pipeline+ bool initialized {};+ // true while a compile is in flight, used to dedupe concurrent claims.+ // Protected by device->compile_mutex.+ bool compile_pending {};+ // set to true when the shader has been compiled+ std::atomic<bool> compiled {};+ // number of registers used, extracted from pipeline executable properties+ uint32_t register_count {};++#if defined(VK_EXT_shader_64bit_indexing)+ bool is_64b_indexing {};+#endif+ // linked list of pipelines for multiple compilation variants.+ // currently only used to compile a 64-bit indexing variant.+ vk_pipeline next;+};++typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;++struct vk_matmul_pipeline_key {+ ggml_type type_a;+ ggml_type type_b;+ bool mul_mat_id;+ bool f16acc;++ bool operator<(const vk_matmul_pipeline_key & o) const {+ return std::tie(type_a, type_b, mul_mat_id, f16acc)+ < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);+ }+};++struct vk_matmul_pipeline_pair {+ vk_pipeline unaligned;+ vk_pipeline aligned;+ uint32_t align;+};++struct vk_tile_config {+ std::vector<uint32_t> warptile;+ std::array<uint32_t, 3> wg_denoms;+ uint32_t align;+};++using matmul_tile_selector_t = std::function<uint32_t(+ uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count,+ const std::vector<vk_matmul_pipeline_pair>& configs)>;++struct vk_device_struct;++typedef std::shared_ptr<vk_device_struct> vk_device;++typedef std::weak_ptr<vk_device_struct> vk_device_ref;++struct vk_buffer_struct;++typedef std::shared_ptr<vk_buffer_struct> vk_buffer;++typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref;++struct ggml_backend_vk_buffer_type_context {+ std::string name;+ vk_device device;+};++struct vk_command_buffer {+ vk::CommandBuffer buf;+ uint64_t use_counter = 0;+ bool in_use = false;+};++struct vk_queue;++struct vk_command_pool {+ void init(vk_device& device, vk_queue *q_);+ void destroy(vk::Device& device);++ vk::CommandPool pool;+ // Using deque so the pointers to command buffers+ // remain valid even if we add more+ std::deque<vk_command_buffer> cmd_buffers;++ vk_queue *q;++ size_t buffers_in_use() const {+ return std::count_if(cmd_buffers.begin(), cmd_buffers.end(),+ [](const auto& cb) { return cb.in_use; });+ }+};++struct vk_queue_handle {+ vk::Queue queue;+ vk_device_ref device;+ std::mutex * device_submit_mutex = nullptr;+ virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;+ virtual void lock() {} // no-op by default (internally synchronized case)+ virtual void unlock() {}+ virtual ~vk_queue_handle() = default;+};++struct vk_queue_handle_synchronized : vk_queue_handle {+ std::mutex mutex;+ void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override;++ void lock() override { mutex.lock(); }+ void unlock() override { mutex.unlock(); }+};++struct vk_queue_handle_unsynchronized : vk_queue_handle {+ void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override;++ // lock()/unlock() inherited no-ops+};++struct vk_queue {+ uint32_t queue_family_index;+ std::shared_ptr<vk_queue_handle> handle;++ vk_command_pool cmd_pool;++ vk::PipelineStageFlags stage_flags;++ bool transfer_only;+};++static constexpr uint32_t mul_mat_vec_max_cols = 8;++static constexpr uint32_t p021_max_gqa_ratio = 8;++enum vk_device_architecture {+ OTHER,+ AMD_GCN,+ AMD_RDNA1,+ AMD_RDNA2,+ AMD_RDNA3,+ INTEL_XE1,+ INTEL_XE2,+ NVIDIA_PRE_TURING,+ NVIDIA_TURING,+};++enum vk_conv_shapes {+ CONV_SHAPE_128x128,+ CONV_SHAPE_64x32,+ CONV_SHAPE_32x256,+ CONV_SHAPE_64x128,+ CONV_SHAPE_COUNT,+};++struct vk_conv_block_size {+ uint32_t K;+ uint32_t NPQ;+ uint32_t CRS;+};++inline vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = {+ // K NPQ CRS+ { 128, 128, 16 }, // CONV_SHAPE_128x128+ { 64, 32, 32 }, // CONV_SHAPE_64x32+ { 32, 256, 16 }, // CONV_SHAPE_32x256+ { 64, 128, 16 }, // CONV_SHAPE_64x128+};++enum dmmv_wg_sizes {+ DMMV_WG_SIZE_SUBGROUP,+ DMMV_WG_SIZE_LARGE,+ DMMV_WG_SIZE_COUNT,+};++enum FaCodePath {+ FA_SCALAR,+ FA_COOPMAT1,+ FA_COOPMAT2,+};++struct vk_fa_pipeline_state {+ uint32_t HSK, HSV;+ uint32_t Br, Bc;+ uint32_t D_split, row_split;+ bool shmem_staging;+ FaCodePath path;+ uint32_t workgroup_size, subgroup_size;+ bool aligned;+ bool f32acc;+ uint32_t flags;+ uint32_t limit_occupancy_shmem;+ ggml_type k_type;+ ggml_type v_type;++ bool operator<(const vk_fa_pipeline_state &b) const {+ return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) <+ std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type);+ }+};++struct vk_conv2d_pipeline_state {+ vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned)+ : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {}++ uint32_t s0, s1, p0, p1, d0, d1, KW, KH;+ // when set, shader can skip K/CRS/NPQ bounds checks and address clamps+ uint32_t aligned;++ bool operator<(const vk_conv2d_pipeline_state &b) const {+ return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) <+ std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned);+ }+};++struct vk_conv3d_pipeline_state {+ vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,+ uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)+ : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}++ uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;+ uint32_t aligned;++ bool operator<(const vk_conv3d_pipeline_state &b) const {+ return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <+ std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);+ }+};++struct vk_solve_tri_pipeline_state {+ vk_solve_tri_pipeline_state(uint32_t N, uint32_t K)+ : N(N), K(K) {}++ uint32_t N, K;++ bool operator<(const vk_solve_tri_pipeline_state &b) const {+ return std::tie(N, K) <+ std::tie(b.N, b.K);+ }+};++enum shader_reduction_mode {+ SHADER_REDUCTION_MODE_SHMEM,+ SHADER_REDUCTION_MODE_HYBRID,+ SHADER_REDUCTION_MODE_SUBGROUP,+ SHADER_REDUCTION_MODE_COUNT,+};++static constexpr uint32_t num_argsort_pipelines = 11;++static constexpr uint32_t num_topk_moe_pipelines = 10;++static constexpr uint32_t num_topk_pipelines = 11;++static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,+ GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE,+ GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV,+ GGML_OP_RESHAPE };++static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD,+ GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS,+ GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP,+ GGML_OP_DIV, GGML_OP_RESHAPE };++static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT,+ GGML_OP_RESHAPE, GGML_OP_ADD,+ GGML_OP_ARGSORT, GGML_OP_VIEW,+ GGML_OP_GET_ROWS, GGML_OP_RESHAPE,+ GGML_OP_SUM_ROWS, GGML_OP_CLAMP,+ GGML_OP_DIV, GGML_OP_RESHAPE };++static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,+ GGML_OP_VIEW, GGML_OP_GET_ROWS };++static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW,+ GGML_OP_GET_ROWS, GGML_OP_RESHAPE,+ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE };++static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN,+ GGML_OP_SQR, GGML_OP_MUL,+ GGML_OP_ADD };++static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE,+ GGML_OP_CONT, GGML_OP_CPY,+ GGML_OP_RESHAPE, GGML_OP_ADD,+ GGML_OP_TOP_K };++static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges {+ { 1, 0, 0 }, // permute->src[0] == get_rows+ { 2, 0, 1 }, // cont->src[0] == permute+ { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast)+ { 5, 0, 2 }, // add->src[0] == cont+ { 5, 1, 4 }, // add->src[1] == reshape+ { 6, 0, 5 }, // top_k->src[0] == add+};++static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_mul_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL };++static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };++static constexpr std::initializer_list<ggml_op> rms_norm_mul_rope_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };++static constexpr std::initializer_list<ggml_op> rms_norm_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_VIEW, GGML_OP_SET_ROWS };++static constexpr std::initializer_list<ggml_op> rope_view_set_rows_pattern { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };++static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges {+ { 1, 0, 0 }, // reshape->src[0] == softmax+ { 2, 0, 0 }, // argsort->src[0] == softmax+ { 3, 0, 2 }, // view->src[0] == argsort+ { 4, 0, 1 }, // get_rows->src[0] == reshape+ { 4, 1, 3 }, // get_rows->src[1] == view+ { 5, 0, 4 }, // reshape->src[0] == get_rows+ { 6, 0, 5 }, // sum_rows->src[0] == reshape+ { 7, 0, 6 }, // clamp->src[0] == sum_rows+ { 8, 0, 5 }, // div->src[0] == reshape+ { 8, 1, 7 }, // div->src[1] == clamp+ { 9, 0, 8 }, // reshape->src[0] == div+};++static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges {+ { 1, 0, 0 }, // reshape->src[0] == sigmoid+ { 2, 0, 0 }, // add->src[0] == sigmoid+ { 3, 0, 2 }, // argsort->src[0] == add+ { 4, 0, 3 }, // view->src[0] == argsort+ { 5, 0, 1 }, // get_rows->src[0] == reshape+ { 5, 1, 4 }, // get_rows->src[1] == view+ { 6, 0, 5 }, // reshape->src[0] == get_rows+ { 7, 0, 6 }, // sum_rows->src[0] == reshape+ { 8, 0, 7 }, // clamp->src[0] == sum_rows+ { 9, 0, 6 }, // div->src[0] == reshape+ { 9, 1, 8 }, // div->src[1] == clamp+ {10, 0, 9 }, // reshape->src[0] == div+};++static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges {+ { 1, 0, 0 }, // sqrt->src[0] == softplus+ { 2, 0, 1 }, // reshape->src[0] == sqrt+ { 3, 0, 1 }, // add->src[0] == sqrt+ { 4, 0, 3 }, // argsort->src[0] == add+ { 5, 0, 4 }, // view->src[0] == argsort+ { 6, 0, 2 }, // get_rows->src[0] == reshape+ { 6, 1, 5 }, // get_rows->src[1] == view+ { 7, 0, 6 }, // reshape->src[0] == get_rows+ { 8, 0, 7 }, // sum_rows->src[0] == reshape+ { 9, 0, 8 }, // clamp->src[0] == sum_rows+ {10, 0, 7 }, // div->src[0] == reshape+ {10, 1, 9 }, // div->src[1] == clamp+ {11, 0,10 }, // reshape->src[0] == div+};++static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges {+ { 1, 0, 0 }, // reshape->src[0] == softmax+ { 2, 0, 0 }, // argsort->src[0] == softmax+ { 3, 0, 2 }, // view->src[0] == argsort+ { 4, 0, 1 }, // get_rows->src[0] == reshape+ { 4, 1, 3 }, // get_rows->src[1] == view+};++static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges {+ { 1, 0, 0 }, // view->src[0] == argsort+ { 2, 1, 1 }, // get_rows->src[1] == view+ { 3, 0, 2 }, // reshape->src[0] == get_rows+ { 4, 0, 3 }, // soft_max->src[0] == reshape+ { 5, 0, 4 }, // reshape->src[0] == soft_max+};++enum topk_moe_mode {+ TOPK_MOE_EARLY_SOFTMAX,+ TOPK_MOE_EARLY_SOFTMAX_NORM,+ TOPK_MOE_LATE_SOFTMAX,+ TOPK_MOE_SIGMOID_NORM_BIAS,+ TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS,+ TOPK_MOE_COUNT,+};++enum rms_norm_mode {+ RMS_NORM_MUL,+ RMS_NORM_MUL_ADD,+ RMS_NORM_MUL_ADD_MUL,+ RMS_NORM_MUL_ROPE,+ RMS_NORM_MUL_ROPE_VIEW_SET_ROWS,+ RMS_NORM_VIEW_SET_ROWS,+ RMS_NORM_COUNT,+};++static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges {+ { 1, 0, 0 }, // view->src[0] == rope+ { 2, 0, 1 }, // set_rows->src[0] == view+};++static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges {+ { 1, 0, 0 }, // mul->src[0] == rms+ { 2, 0, 1 }, // rope->src[0] == mul+ { 3, 0, 2 }, // view->src[0] == rope+ { 4, 0, 3 }, // set_rows->src[0] == view+};++static constexpr std::initializer_list<std::array<int, 3>> rms_norm_view_set_rows_edges {+ { 1, 0, 0 }, // view->src[0] == rms_norm+ { 2, 0, 1 }, // set_rows->src[0] == view+};++static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = {+ GGML_TYPE_F32,+ GGML_TYPE_F16,+ GGML_TYPE_BF16,+ GGML_TYPE_Q8_0,+ GGML_TYPE_Q5_1,+ GGML_TYPE_Q5_0,+ GGML_TYPE_Q4_1,+ GGML_TYPE_Q4_0,+ GGML_TYPE_IQ4_NL,+};++class vk_memory_logger;++struct vk_device_struct {+ std::recursive_mutex mutex;+ std::mutex queue_submit_mutex;+ mutable std::shared_mutex pinned_memory_mutex;++ // Guards compile_pending, all_pipelines, and the dynamic pipeline maps+ // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile+ // runs with no lock held, so different pipelines can compile in parallel.+ // Lock order is device->mutex -> compile_mutex, never the reverse.+ std::mutex compile_mutex;+ std::condition_variable compile_cv;++ uint32_t debug_cmdbuf_idx {};++ vk::PhysicalDevice physical_device;+ vk::PhysicalDeviceProperties properties;+ std::string name;+ uint64_t max_memory_allocation_size;+ uint64_t max_buffer_size;+ uint64_t suballocation_block_size;+ uint64_t min_imported_host_pointer_alignment;+ bool external_memory_host {};+ bool fp16;+ bool bf16;+ bool pipeline_robustness;+ bool memory_priority;+ vk::Device device;+ uint32_t vendor_id;+ vk::DriverId driver_id;+ vk_device_architecture architecture;+ std::unique_ptr<vk_queue> compute_queue;+ std::unique_ptr<vk_queue> transfer_queue;+ bool single_queue;+ bool support_async;+ bool async_use_transfer_queue;+ bool has_internally_synchronized_queues = false;+ uint32_t subgroup_size;+ uint32_t subgroup_size_log2;+ uint32_t shader_core_count;+ bool uma;+ bool prefer_host_memory;+ bool float_controls_rte_fp16;+ bool float_controls_denorm_preserve_fp16;+ bool subgroup_basic;+ bool subgroup_arithmetic;+ bool subgroup_shuffle;+ bool subgroup_ballot;+ bool subgroup_clustered;+ bool subgroup_vote;+ bool multi_add;+ bool shader_int64;+ bool buffer_device_address;+ bool vulkan_memory_model;++ bool add_rms_fusion;+ uint32_t partials_binding_alignment;+ uint32_t max_nodes_per_submit;++ bool shader_64b_indexing;++ bool integer_dot_product;+ // 0: default, 1: force mmvq, -1: disable mmvq+ int32_t mmvq_mode;++ bool subgroup_size_control;+ uint32_t subgroup_min_size;+ uint32_t subgroup_max_size;+ bool subgroup_require_full_support;++ // floor(log2(maxComputeWorkGroupInvocations))+ uint32_t max_workgroup_size_log2 {};++ bool coopmat_support;+ bool coopmat_acc_f32_support {};+ bool coopmat_acc_f16_support {};+ bool coopmat_bf16_support {};+ bool coopmat_support_16x16x16_f16acc {};+ bool coopmat_support_16x16x16_f32acc {};+ bool coopmat1_fa_support {};+ uint32_t coopmat_m;+ uint32_t coopmat_n;+ uint32_t coopmat_k;++ bool coopmat_int_support;+ uint32_t coopmat_int_m;+ uint32_t coopmat_int_n;+ uint32_t coopmat_int_k;++ bool coopmat2;+ bool coopmat2_bf16_support {};+ bool coopmat2_decode_vector;++ bool dot2_f16 {};+ bool ocp_fp4 {};++ bool pipeline_executable_properties_support {};++ bool device_fault {};+ PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};++ bool serialize_submissions {};++ const ggml_cgraph * diag_cgraph {};+ int diag_prev_start = -1;+ int diag_prev_end = -1;++ size_t idx;++ bool mul_mat_l[GGML_TYPE_COUNT];+ bool mul_mat_m[GGML_TYPE_COUNT];+ bool mul_mat_s[GGML_TYPE_COUNT];+ bool mul_mat_id_l[GGML_TYPE_COUNT];+ bool mul_mat_id_m[GGML_TYPE_COUNT];+ bool mul_mat_id_s[GGML_TYPE_COUNT];++ // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses+ // a different shared-memory layout than the float matmul shaders.+ bool mul_mat_l_int[GGML_TYPE_COUNT];+ bool mul_mat_m_int[GGML_TYPE_COUNT];+ bool mul_mat_s_int[GGML_TYPE_COUNT];+ bool mul_mat_id_l_int[GGML_TYPE_COUNT];+ bool mul_mat_id_m_int[GGML_TYPE_COUNT];+ bool mul_mat_id_s_int[GGML_TYPE_COUNT];++ vk::DescriptorSetLayout dsl;++ std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul;+ matmul_tile_selector_t matmul_tile_selector;+ matmul_tile_selector_t matmul_id_tile_selector;++ vk_pipeline pipeline_matmul_split_k_reduce;+ vk_pipeline pipeline_quantize_q8_1_x4;++ vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];+ vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV+ vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];+ vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];+ vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];++ vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];+ vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];++ vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio];+ vk_pipeline pipeline_mul_mat_vec_nc_f16_f32;+ vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT];+ vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT];+ vk_pipeline pipeline_get_rows_back_f32;+ vk_pipeline pipeline_acc_f32;+ vk_pipeline pipeline_set_f32;++ // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]+ vk_pipeline pipeline_add[2][2][2];+ vk_pipeline pipeline_add_norepeat[2][2][2];+ vk_pipeline pipeline_sub[2][2][2];+ vk_pipeline pipeline_sub_norepeat[2][2][2];+ vk_pipeline pipeline_mul[2][2][2];+ vk_pipeline pipeline_mul_norepeat[2][2][2];+ vk_pipeline pipeline_div[2][2][2];+ vk_pipeline pipeline_div_norepeat[2][2][2];+ vk_pipeline pipeline_add_rms[2][2][2];+ vk_pipeline pipeline_add_rms_norepeat[2][2][2];++ // indexed by num_additional_fused_ops == num_adds - 1+ vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS];+ vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS];++ vk_pipeline pipeline_add_id_f32;++ vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;+ vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;+ vk_pipeline pipeline_scale_f32;+ vk_pipeline pipeline_log[2];+ vk_pipeline pipeline_tri[2];+ vk_pipeline pipeline_diag[2];+ vk_pipeline pipeline_clamp[2];+ vk_pipeline pipeline_pad_f32;+ vk_pipeline pipeline_pad_reflect_1d_f32;+ vk_pipeline pipeline_roll_f32;+ vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;+ vk_pipeline pipeline_repeat_i16;+ vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;+ vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;+ vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];+ vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];+ vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32;+ vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32;+ // [src0 0=fp32,1=fp16][dst]+ vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT];+ vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];+ vk_pipeline pipeline_norm_f32;+ vk_pipeline pipeline_group_norm_f32;+ vk_pipeline pipeline_rms_norm_f32;+ vk_pipeline pipeline_rms_norm_mul_f32;+ vk_pipeline pipeline_rms_norm_mul_add_f32;+ vk_pipeline pipeline_rms_norm_mul_add_mul_f32;+ vk_pipeline pipeline_rms_norm_mul_add_partials_f32;+ vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;+ vk_pipeline pipeline_rms_norm_set_rows_f32_f32;+ vk_pipeline pipeline_rms_norm_set_rows_f32_f16;+ vk_pipeline pipeline_rms_norm_partials_f32;+ vk_pipeline pipeline_rms_norm_mul_partials_f32;+ vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;+ vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;+ vk_pipeline pipeline_rms_norm_back_f32;+ vk_pipeline pipeline_l2_norm_f32;++ // [src/dst 0=fp32,1=fp16]+ vk_pipeline pipeline_exp[2];+ vk_pipeline pipeline_expm1[2];+ vk_pipeline pipeline_elu[2];+ vk_pipeline pipeline_gelu[2];+ vk_pipeline pipeline_gelu_erf[2];+ vk_pipeline pipeline_gelu_quick[2];+ vk_pipeline pipeline_silu[2];+ vk_pipeline pipeline_relu[2];+ vk_pipeline pipeline_sqr[2];+ vk_pipeline pipeline_sqrt[2];+ vk_pipeline pipeline_sin[2];+ vk_pipeline pipeline_cos[2];+ vk_pipeline pipeline_xielu[2];+ vk_pipeline pipeline_neg[2];+ vk_pipeline pipeline_tanh[2];+ vk_pipeline pipeline_sigmoid[2];+ vk_pipeline pipeline_hardsigmoid[2];+ vk_pipeline pipeline_hardswish[2];+ vk_pipeline pipeline_abs[2];+ vk_pipeline pipeline_softplus[2];+ vk_pipeline pipeline_step[2];+ vk_pipeline pipeline_round[2];+ vk_pipeline pipeline_ceil[2];+ vk_pipeline pipeline_floor[2];+ vk_pipeline pipeline_trunc[2];+ vk_pipeline pipeline_sgn[2];++ // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]+ vk_pipeline pipeline_unary_mul[4][2][2][2];++ vk_pipeline pipeline_add1_f16_f16;+ vk_pipeline pipeline_add1_f16_f32;+ vk_pipeline pipeline_add1_f32_f32;++ vk_pipeline pipeline_arange_f32;++ vk_pipeline pipeline_fill_f32;+ vk_pipeline pipeline_fill_f16;++ vk_pipeline pipeline_geglu[2];+ vk_pipeline pipeline_reglu[2];+ vk_pipeline pipeline_swiglu[2];+ vk_pipeline pipeline_swiglu_oai[2];+ vk_pipeline pipeline_swiglu_clamp[2];+ vk_pipeline pipeline_geglu_erf[2];+ vk_pipeline pipeline_geglu_quick[2];++ vk_pipeline pipeline_leaky_relu[2];+ vk_pipeline pipeline_silu_back_f32;+ vk_pipeline pipeline_diag_mask_inf_f32;+ vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;+ vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;+ vk_pipeline pipeline_soft_max_back_f32;++ vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16;+ vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16;+ vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16;++ vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16;+ vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16;+ vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16;+ vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16;+ vk_pipeline pipeline_argsort_f32[num_argsort_pipelines];+ vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines];+ vk_pipeline pipeline_topk_f32[num_topk_pipelines];+ vk_pipeline pipeline_topk_radix_f32;+ vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask)+ vk_pipeline pipeline_sum_rows_f32;+ vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512;+ vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512;+ vk_pipeline pipeline_fwht_f32[4];+ vk_pipeline pipeline_cumsum_f32;+ vk_pipeline pipeline_cumsum_small_f32;+ vk_pipeline pipeline_cumsum_multipass1_f32;+ vk_pipeline pipeline_cumsum_multipass2_f32;+ vk_pipeline pipeline_argmax_f32;+ vk_pipeline pipeline_count_equal_i32;+ vk_pipeline pipeline_dsv4_hc_comb_f32;+ vk_pipeline pipeline_dsv4_hc_pre_f32;+ vk_pipeline pipeline_dsv4_hc_pre_gated_f32;+ vk_pipeline pipeline_dsv4_hc_post_f32;+ vk_pipeline pipeline_dsv4_hc_post_nocomb_f32;+ std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32;+ vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16;+ vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16;+ vk_pipeline pipeline_timestep_embedding_f32;+ vk_pipeline pipeline_conv_transpose_1d_f32;+ vk_pipeline pipeline_col2im_1d_f32;+ vk_pipeline pipeline_col2im_1d_f16;+ vk_pipeline pipeline_col2im_1d_bf16;+ vk_pipeline pipeline_out_prod_f32;+ vk_pipeline pipeline_snake_f32;+ vk_pipeline pipeline_snake_f16;+ vk_pipeline pipeline_snake_bf16;+ vk_pipeline pipeline_pool1d_f32;+ vk_pipeline pipeline_pool2d_f32;+ vk_pipeline pipeline_rwkv_wkv6_f32;+ vk_pipeline pipeline_rwkv_wkv7_f32;+ vk_pipeline pipeline_gated_linear_attn_f32;+ vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];+ // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128+ vk_pipeline pipeline_gated_delta_net[4][2];+ vk_pipeline pipeline_ssm_scan_f32_d128;+ vk_pipeline pipeline_ssm_scan_f32_d256;+ vk_pipeline pipeline_ssm_conv_f32;+ vk_pipeline pipeline_ssm_conv_silu_f32;+ vk_pipeline pipeline_ssm_conv_bias_silu_f32;+ vk_pipeline pipeline_opt_step_adamw_f32;+ vk_pipeline pipeline_opt_step_sgd_f32;+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT];+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT];+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT];+ std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT];+ std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT];+ std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT];+ vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32;+ vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32;++ std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16;++ std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt;++ vk_pipeline pipeline_fa_sparse_compact;+ vk_pipeline pipeline_fa_sparse_compact_subgroup;+ bool fa_sparse_compact_use_subgroups;++ vk_pipeline pipeline_flash_attn_split_k_reduce;+ vk_pipeline pipeline_count_experts;++ // [2] is for whether to take n_experts from spec constant (0) or push constant (1)+ vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2];++ std::vector<vk_pipeline_ref> all_pipelines;++ std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory;++ vk::Fence fence;+ vk_buffer sync_staging;++ ggml_backend_buffer_type buffer_type;++ bool disable_fusion;+ bool disable_host_visible_vidmem;+ bool allow_sysmem_fallback;+ bool disable_graph_optimize;++ std::unique_ptr<vk_memory_logger> memory_logger;++ ~vk_device_struct();++};++inline void vk_command_pool::init(vk_device& device, vk_queue *q_) {+ cmd_buffers.clear();+ q = q_;++ vk::CommandPoolCreateInfo command_pool_create_info(+ vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT),+ q->queue_family_index);+ pool = device->device.createCommandPool(command_pool_create_info);+}++inline void vk_command_pool::destroy(vk::Device& device) {+ device.destroyCommandPool(pool);+ pool = nullptr;+ cmd_buffers.clear();+}++struct vk_buffer_struct {+ vk::Buffer buffer = VK_NULL_HANDLE;+ vk::DeviceMemory device_memory = VK_NULL_HANDLE;+ vk::MemoryPropertyFlags memory_property_flags;+ void * ptr;+ size_t size = 0;+ vk::DeviceAddress bda_addr {};++ vk_device device;++ ~vk_buffer_struct() {+ if (size == 0) {+ return;+ }+ VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");++ device->device.freeMemory(device_memory);+ device->device.destroyBuffer(buffer);+ }+};++struct vk_subbuffer {+ vk_buffer buffer;+ uint64_t offset;+ uint64_t size;++ operator vk::DescriptorBufferInfo() const {+ return { buffer->buffer, offset, size };+ }+};++struct vk_semaphore {+ vk::Semaphore s;+ uint64_t value;+};++struct vk_event {+ std::vector<vk::Event> events_free; // Events available for reuse+ std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize+ vk::Event event;+ bool has_event;++ vk_semaphore tl_semaphore;+ vk_command_buffer* cmd_buffer = nullptr;+ uint64_t cmd_buffer_use_counter = 0;+};++struct vk_submission {+ vk_command_buffer* buffer = nullptr;+ std::vector<vk_semaphore> wait_semaphores;+ std::vector<vk_semaphore> signal_semaphores;+};++typedef std::vector<vk_submission> vk_sequence;++#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1++#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2++#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4++#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8++struct vk_staging_memcpy {+ vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {}++ void * dst;+ const void * src;+ size_t n;+};++struct vk_staging_memset {+ vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {}++ void * dst;+ uint32_t val;+ size_t n;+};++struct vk_context_struct {+ vk_submission * s;+ std::vector<vk_sequence> seqs;++ int exit_tensor_idx;++ std::vector<vk_staging_memcpy> in_memcpys;+ std::vector<vk_staging_memcpy> out_memcpys;+ std::vector<vk_staging_memset> memsets;++ std::vector<std::string> debug_labels;++ vk_command_pool * p {};+};++typedef std::shared_ptr<vk_context_struct> vk_context;++typedef std::weak_ptr<vk_context_struct> vk_context_ref;++struct ggml_vk_garbage_collector {+ std::vector<vk_semaphore> tl_semaphores;+ std::vector<vk_semaphore> semaphores;+ std::vector<vk::Event> events;+ std::vector<vk_context> contexts;+};++#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; }++static std::string format_size(size_t size) {+ const size_t kib = 1024;+ const size_t mib = kib * 1024;+ const size_t gib = mib * 1024;++ std::ostringstream oss;+ oss << std::fixed << std::setprecision(2);++ if (size >= gib) {+ oss << static_cast<double>(size) / gib << " GiB";+ } else if (size >= mib) {+ oss << static_cast<double>(size) / mib << " MiB";+ } else if (size >= kib) {+ oss << static_cast<double>(size) / kib << " KiB";+ } else {+ oss << size << " B";+ }++ return oss.str();+}++class vk_memory_logger {+public:+ vk_memory_logger(): total_device(0), total_host(0) {}+ void log_allocation(vk_buffer_ref buf_ref, size_t size);+ void log_deallocation(vk_buffer_ref buf_ref);++private:+ std::map<vk::Buffer, size_t> allocations; // Track allocations+ size_t total_device;+ size_t total_host;+ static std::mutex log_mutex;+};++inline std::mutex vk_memory_logger::log_mutex;++class vk_perf_logger {+ public:+ void print_timings(bool force = false);+++ std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops);+++ void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) {+ uint64_t n_flops;+ std::string name = get_node_fusion_name(node, fusion_name, &n_flops);+ if (n_flops) {+ flops[name].push_back(n_flops);+ }+ timings[name].push_back(time);+ }++ void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) {+ uint64_t total_flops = 0;+ std::string name;+ for (size_t n = 0; n < nodes.size(); ++n) {+ uint64_t n_flops = 0;+ name += get_node_fusion_name(nodes[n], names[n], &n_flops);+ total_flops += n_flops;++ if (n != nodes.size() - 1) {+ name += ", ";+ }+ }+ if (total_flops) {+ flops[name].push_back(total_flops);+ }+ timings[name].push_back(time);+ }++ private:+ std::map<std::string, std::vector<uint64_t>> timings;+ std::map<std::string, std::vector<uint64_t>> flops;+ uint32_t print_count {};+};++struct ggml_backend_vk_context {+ std::string name;++ vk_device device;++ size_t semaphore_idx, event_idx;+ ggml_vk_garbage_collector gc;+ size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset;+ vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging;+ vk::Fence fence, almost_ready_fence;+ bool submit_pending {};+ bool almost_ready_fence_pending {};+ // Set before op_add and unset after op_rms_norm to indicate that the add should+ // write partial sums to accumulate the square of the vector components+ bool do_add_rms_partials_offset_calculation;+ bool do_add_rms_partials;++ uint64_t last_total_flops {UINT64_MAX};++ // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert.+ vk_pipeline_struct * prealloc_y_last_pipeline_used {};+ const ggml_tensor * prealloc_y_last_tensor_used {};+ // True when the K dimension in prealloc_y is padded.+ bool prealloc_y_last_k_padded {};++ // Track which nodes have been used since the last sync, and whether they were written to+ std::vector<const ggml_tensor *> unsynced_nodes_written;+ std::vector<const ggml_tensor *> unsynced_nodes_read;+ // Track which prealloc buffers have pending reads that need to be synchronized.+ // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set),+ // and set to true after the buffer contents are consumed.+ bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync;++ vk_context_ref compute_ctx;++ vk_context_ref transfer_ctx;+ vk_semaphore transfer_semaphore;+ uint64_t transfer_semaphore_last_submitted {};++ std::vector<vk_context_ref> tensor_ctxs;++ std::vector<vk::DescriptorPool> descriptor_pools;+ std::vector<vk::DescriptorSet> descriptor_sets;+ uint32_t descriptor_set_idx {};+ uint32_t pipeline_descriptor_set_requirements {};++ vk_command_pool compute_cmd_pool;+ vk_command_pool transfer_cmd_pool;++ // number of additional consecutive nodes that are being fused with the+ // node currently being processed+ int num_additional_fused_ops {};+ // Bitmask of which fused ops need to write an intermediate value to memory.+ // Bit 'i' means nodes[start_of_fusion + i] writes to memory.+ // If there's no fusion, bit 0 is still set.+ int fused_ops_write_mask {};+ topk_moe_mode fused_topk_moe_mode {};+ bool fused_topk_moe_scale {};+ // QSA indexer gather+add+top_k fused into one radix-select+ bool fused_topk_qsa {};+ rms_norm_mode fused_rms_norm_mode {RMS_NORM_COUNT};++ // for GGML_VK_PERF_LOGGER+ std::unique_ptr<vk_perf_logger> perf_logger;+ vk::QueryPool query_pool;+ std::vector<const char *> query_fusion_names;+ std::vector<int> query_fusion_node_count;+ std::vector<ggml_tensor *> query_nodes;+ std::vector<int> query_node_idx;+ int32_t num_queries {};+ int32_t query_idx {};+};++struct ggml_backend_vk_buffer_context {+ vk_device_ref device;+ vk_buffer dev_buffer;+ std::string name;++ ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) :+ device(device),+ dev_buffer(dev_buffer),+ name(name) {+ }++ ~ggml_backend_vk_buffer_context();++};++struct vk_instance_t {+ vk::Instance instance;++ bool debug_utils_support = false; // VK_EXT_debug_utils enabled+ PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {};+ PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {};+ PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {};+ PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {};+ PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {};+ PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};++ std::vector<size_t> device_indices;+ std::vector<bool> device_supports_membudget;+ vk_device devices[GGML_VK_MAX_DEVICES];+};++typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);++static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;++static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;++static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;++struct vk_fa_tuning_params {+ FaCodePath path;+ uint32_t workgroup_size;+ uint32_t subgroup_size;+ uint32_t block_rows;+ uint32_t block_cols;+ uint32_t d_split;+ uint32_t row_split;+ bool shmem_staging;+ bool disable_subgroups;+ uint32_t limit_occupancy_shmem;++ void print() const {+ std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size <<+ " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split <<+ " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups <<+ " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl;+ }+};++struct GpuPipelineConfig {+ // GPU architecture identifier.+ // Example: vk_device_architecture::AMD_GCN+ vk_device_architecture arch;++ // Mapping of pipeline names to their specific subgroup sizes.+ // Example: {"soft_max_f32", 64}+ std::unordered_map<std::string, uint32_t> pipelines;++ // Default subgroup size for this GPU.+ // Defaults to 0 if not explicitly provided.+ uint32_t default_subgroup_size = 0;+};++static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32;++struct CompileTask {+ vk_pipeline pipeline;+ size_t spv_size;+ const void * spv_data;+ std::string entrypoint;+ uint32_t parameter_count;+ std::array<uint32_t, 3> wg_denoms;+ std::vector<uint32_t> specialization_constants;+ bool disable_robustness;+ bool require_full_subgroups;+ uint32_t required_subgroup_size;+};++struct ggml_vk_debug_label {+ // at most one of these is set, depending on the scope the label was opened in+ vk_context_struct * subctx {};+ vk_queue_handle * qhandle {};++ // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)".+ // RGP cannot recover the pipeline name on its own, it only has the hash+ ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2);+++ // one region per graph node+ // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19"+ ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused);+++ // one region per graph evaluation, opened on the queue instead of a command buffer+ // so it spans every submit the evaluation makes+ ggml_vk_debug_label(vk_queue_handle * handle, const char * name);+++ // call before the command buffer can end, the destructor covers the rest+ void close();+++ ~ggml_vk_debug_label() {+ close();+ }++ ggml_vk_debug_label(const ggml_vk_debug_label &) = delete;+ ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete;++private:+ // the constructors check this too, so the name is not built when markers are off+ void begin(vk_context & ctx, const std::string & name);++};++#define UNUSED GGML_UNUSED++struct ggml_backend_vk_device_context {+ size_t device;+ std::string name;+ std::string description;+ bool is_integrated_gpu;+ std::string pci_bus_id;+ int op_offload_min_batch_size;+};+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 917b03deab..7b53d1b5c3 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -1,432 +1,10 @@-#include "ggml-vulkan.h"-#include <vulkan/vulkan_core.h>-#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_CHECK_RESULTS)-#include <chrono>-#include "ggml-cpu.h"-#endif--// See https://github.com/KhronosGroup/Vulkan-Hpp?tab=readme-ov-file#extensions--per-device-function-pointers--#define VULKAN_HPP_DISPATCH_LOADER_DYNAMIC 1-// We use VULKAN_HPP_DEFAULT_DISPATCHER, but not VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE-// to avoid conflicts with applications or other libraries who might use it.-#if VK_HEADER_VERSION >= 301-namespace vk::detail { class DispatchLoaderDynamic; }-using vk::detail::DispatchLoaderDynamic;-#else-namespace vk { class DispatchLoaderDynamic; }-using vk::DispatchLoaderDynamic;-#endif-DispatchLoaderDynamic & ggml_vk_default_dispatcher();-#define VULKAN_HPP_DEFAULT_DISPATCHER ggml_vk_default_dispatcher()--#include <vulkan/vulkan.hpp>--// Fallback definitions for VK_NV_cooperative_matrix_decode_vector in case the-// installed Vulkan headers predate the extension.-#ifndef VK_NV_cooperative_matrix_decode_vector-#define VK_NV_cooperative_matrix_decode_vector 1-#define VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME "VK_NV_cooperative_matrix_decode_vector"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV ((VkStructureType)1000689000)-typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {- VkStructureType sType;- void* pNext;- VkBool32 cooperativeMatrixDecodeVector;-} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;-#endif--// SPIR-V Headers: different SDK installations expose different include paths.-// LunarG Vulkan SDK on Windows typically provides <spirv-headers/spirv.hpp>.-// Linux packages, MSYS2 and MinGW often use the Khronos layout <spirv/unified1/spirv.hpp>.-#if __has_include(<spirv/unified1/spirv.hpp>)-# include <spirv/unified1/spirv.hpp>-#elif __has_include(<spirv-headers/spirv.hpp>)-# include <spirv-headers/spirv.hpp>-#elif __has_include(<spirv.hpp>)-# include <spirv.hpp>-#else- // Fallback to let the compiler throw a standard "file not found" error-# include <spirv/unified1/spirv.hpp>-#endif--#include <algorithm>-#include <cmath>-#include <iomanip>-#include <iostream>-#include <tuple>-#include <vector>-#include <deque>-#include <sstream>-#include <utility>-#include <memory>-#include <limits>-#include <map>-#include <set>-#include <unordered_map>-#include <shared_mutex>-#include <mutex>-#include <future>-#include <condition_variable>-#include <thread>--#if defined(_MSC_VER)-# define NOMINMAX 1-# include <windows.h>-# define YIELD() YieldProcessor()-#elif defined(__clang__) || defined(__GNUC__)-# if defined(__x86_64__) ||defined(__i386__)-# include <immintrin.h>-# define YIELD() _mm_pause()-# elif defined(__arm__) || defined(__aarch64__)-# if defined(__clang__)-# include <arm_acle.h>-# define YIELD() __yield()-# else-# define YIELD() asm volatile("yield")-# endif-# endif-#endif--#if !defined(YIELD)-#define YIELD()-#endif+#include "ggml-vulkan-common.h"-#include "ggml-impl.h"-#include "ggml-backend-impl.h"--#include "ggml-vulkan-shaders.hpp"--// On 32-bit platforms, Vulkan non-dispatchable handles such as VkBuffer are represented as uint64_t,-// and Vulkan-Hpp disables implicit conversions for type safety.namespace {inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) {return os << static_cast<VkBuffer>(buffer);}}--// remove this once it's more widely available in the SDK-#if !defined(VK_KHR_shader_bfloat16)--#define VK_KHR_shader_bfloat16 1-#define VK_KHR_SHADER_BFLOAT16_SPEC_VERSION 1-#define VK_KHR_SHADER_BFLOAT16_EXTENSION_NAME "VK_KHR_shader_bfloat16"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_BFLOAT16_FEATURES_KHR ((VkStructureType)1000141000)-#define VK_COMPONENT_TYPE_BFLOAT16_KHR ((VkComponentTypeKHR)1000141000)--typedef struct VkPhysicalDeviceShaderBfloat16FeaturesKHR {- VkStructureType sType;- void* pNext;- VkBool32 shaderBFloat16Type;- VkBool32 shaderBFloat16DotProduct;- VkBool32 shaderBFloat16CooperativeMatrix;-} VkPhysicalDeviceShaderBfloat16FeaturesKHR;-#endif--#if !defined(VK_VALVE_shader_mixed_float_dot_product)-#define VK_VALVE_shader_mixed_float_dot_product 1-#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_SPEC_VERSION 1-#define VK_VALVE_SHADER_MIXED_FLOAT_DOT_PRODUCT_EXTENSION_NAME "VK_VALVE_shader_mixed_float_dot_product"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE ((VkStructureType)1000673000)-typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE {- VkStructureType sType;- void* pNext;- VkBool32 shaderMixedFloatDotProductFloat16AccFloat32;- VkBool32 shaderMixedFloatDotProductFloat16AccFloat16;- VkBool32 shaderMixedFloatDotProductBFloat16Acc;- VkBool32 shaderMixedFloatDotProductFloat8AccFloat32;-} VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE;-#endif--#if !defined(VK_EXT_shader_ocp_microscaling_types)-#define VK_EXT_shader_ocp_microscaling_types 1-#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_SPEC_VERSION 1-#define VK_EXT_SHADER_OCP_MICROSCALING_TYPES_EXTENSION_NAME "VK_EXT_shader_ocp_microscaling_types"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OCP_MICROSCALING_TYPES_FEATURES_EXT ((VkStructureType)1000672000)-typedef struct VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT {- VkStructureType sType;- void* pNext;- VkBool32 shaderFloat4;- VkBool32 shaderFloat6;- VkBool32 shaderFloat8UnsignedE8M0;- VkBool32 shaderMXInt8;-} VkPhysicalDeviceShaderOCPMicroscalingTypesFeaturesEXT;-#endif--#if !defined(VK_EXT_shader_float8)-#define VK_EXT_shader_float8 1-#define VK_EXT_SHADER_FLOAT8_SPEC_VERSION 1-#define VK_EXT_SHADER_FLOAT8_EXTENSION_NAME "VK_EXT_shader_float8"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT ((VkStructureType)1000567000)-typedef struct VkPhysicalDeviceShaderFloat8FeaturesEXT {- VkStructureType sType;- void* pNext;- VkBool32 shaderFloat8;- VkBool32 shaderFloat8CooperativeMatrix;-} VkPhysicalDeviceShaderFloat8FeaturesEXT;-#endif--#ifndef VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME-#define VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME "VK_KHR_internally_synchronized_queues"-#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INTERNALLY_SYNCHRONIZED_QUEUES_FEATURES_KHR ((VkStructureType)1000504000)-#define VK_DEVICE_QUEUE_CREATE_INTERNALLY_SYNCHRONIZED_BIT_KHR ((VkDeviceQueueCreateFlagBits)0x00000004)--// Compile-time constant guaranteed; no runtime initialization overhead-static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR =- static_cast<vk::DeviceQueueCreateFlagBits>(0x00000004);--typedef struct VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR {- VkStructureType sType;- void* pNext;- VkBool32 internallySynchronizedQueues;-} VkPhysicalDeviceInternallySynchronizedQueuesFeaturesKHR;-#else-static constexpr vk::DeviceQueueCreateFlagBits eInternallySynchronizedKHR = vk::DeviceQueueCreateFlagBits::eInternallySynchronizedKHR;-#endif--#define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1))-#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0))-static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }--#define VK_VENDOR_ID_AMD 0x1002-#define VK_VENDOR_ID_APPLE 0x106b-#define VK_VENDOR_ID_INTEL 0x8086-#define VK_VENDOR_ID_NVIDIA 0x10de-#define VK_VENDOR_ID_QUALCOMM 0x5143--#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256--#define VK_CHECK(err, msg, dev) \- do { \- vk::Result err_; \- try { \- err_ = (err); \- } catch (vk::DeviceLostError &) { \- ggml_vk_print_device_lost_info(dev); \- GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \- #err, __FILE__, __LINE__); \- throw; \- } \- if (err_ != vk::Result::eSuccess) { \- GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \- #err, to_string(err_).c_str(), __FILE__, __LINE__); \- throw vk::SystemError(vk::make_error_code(err_), \- "ggml_vulkan: " msg); \- } \- } while (0)--#ifdef GGML_VULKAN_DEBUG-#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl-#else-#define VK_LOG_DEBUG(msg) ((void) 0)-#endif // GGML_VULKAN_DEBUG--struct ggml_backend_vk_context;--#define MAX_PARAMETER_COUNT 12-// Max number of adds that can be fused without exceeding MAX_PARAMETER_COUNT.-#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)--typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;--struct vk_pipeline_struct {- std::string name;- vk::ShaderModule shader_module;- vk::PipelineLayout layout;- vk::Pipeline pipeline;- uint32_t push_constant_size;- uint32_t parameter_count;- std::array<uint32_t, 3> wg_denoms;- uint32_t align;- // true if fields have been set by ggml_vk_create_pipeline- bool initialized {};- // true while a compile is in flight, used to dedupe concurrent claims.- // Protected by device->compile_mutex.- bool compile_pending {};- // set to true when the shader has been compiled- std::atomic<bool> compiled {};- // number of registers used, extracted from pipeline executable properties- uint32_t register_count {};--#if defined(VK_EXT_shader_64bit_indexing)- bool is_64b_indexing {};-#endif- // linked list of pipelines for multiple compilation variants.- // currently only used to compile a 64-bit indexing variant.- vk_pipeline next;-};--typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;--static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);--struct vk_matmul_pipeline_key {- ggml_type type_a;- ggml_type type_b;- bool mul_mat_id;- bool f16acc;-- bool operator<(const vk_matmul_pipeline_key & o) const {- return std::tie(type_a, type_b, mul_mat_id, f16acc)- < std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);- }-};--struct vk_matmul_pipeline_pair {- vk_pipeline unaligned;- vk_pipeline aligned;- uint32_t align;-};--struct vk_tile_config {- std::vector<uint32_t> warptile;- std::array<uint32_t, 3> wg_denoms;- uint32_t align;-};--using matmul_tile_selector_t = std::function<uint32_t(- uint32_t m, uint32_t n, uint32_t k, uint32_t shader_core_count,- const std::vector<vk_matmul_pipeline_pair>& configs)>;--struct vk_device_struct;-typedef std::shared_ptr<vk_device_struct> vk_device;-typedef std::weak_ptr<vk_device_struct> vk_device_ref;--struct vk_buffer_struct;-typedef std::shared_ptr<vk_buffer_struct> vk_buffer;-typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref;--struct ggml_backend_vk_buffer_type_context {- std::string name;- vk_device device;-};--struct vk_queue;--struct vk_command_buffer {- vk::CommandBuffer buf;- uint64_t use_counter = 0;- bool in_use = false;-};--// Stores command pool/buffers. There's an instance of this-// for each (context,queue) pair and for each (device,queue) pair.-struct vk_command_pool {- void init(vk_device& device, vk_queue *q_);- void destroy(vk::Device& device);-- vk::CommandPool pool;- // Using deque so the pointers to command buffers- // remain valid even if we add more- std::deque<vk_command_buffer> cmd_buffers;-- vk_queue *q;-- size_t buffers_in_use() const {- return std::count_if(cmd_buffers.begin(), cmd_buffers.end(),- [](const auto& cb) { return cb.in_use; });- }-};--static void ggml_vk_print_device_fault_info(const vk_device& device);-static void ggml_vk_print_device_lost_info(const vk_device& device);--// Prevent simultaneous submissions to the same queue.-struct vk_queue_handle {- vk::Queue queue;- vk_device_ref device;- std::mutex * device_submit_mutex = nullptr;- virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;- virtual void lock() {} // no-op by default (internally synchronized case)- virtual void unlock() {}- virtual ~vk_queue_handle() = default;-};--struct vk_queue_handle_synchronized : vk_queue_handle {- std::mutex mutex;- void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {- // Workaround for NVIDIA driver bug- std::unique_lock<std::mutex> device_guard;- if (device_submit_mutex) {- device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);- }- std::lock_guard<std::mutex> guard(mutex);- try {- queue.submit(submits, fence);- } catch (vk::DeviceLostError &) {- if (auto dev = device.lock()) {- ggml_vk_print_device_lost_info(dev);- }- throw;- }- }- void lock() override { mutex.lock(); }- void unlock() override { mutex.unlock(); }-};--// Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues-struct vk_queue_handle_unsynchronized : vk_queue_handle {- void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {- // Workaround for NVIDIA driver bug- std::unique_lock<std::mutex> device_guard;- if (device_submit_mutex) {- device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);- }- try {- queue.submit(submits, fence);- } catch (vk::DeviceLostError &) {- if (auto dev = device.lock()) {- ggml_vk_print_device_lost_info(dev);- }- throw;- }- }- // lock()/unlock() inherited no-ops-};--struct vk_queue {- uint32_t queue_family_index;- std::shared_ptr<vk_queue_handle> handle;-- vk_command_pool cmd_pool;-- vk::PipelineStageFlags stage_flags;-- bool transfer_only;-};--static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);-static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);-static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);-static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);-static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);-static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {- /* .get_name = */ ggml_backend_vk_buffer_type_name,- /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,- /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,- /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,- /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,- /* .is_host = */ NULL,-};--class vk_memory_logger;-class vk_perf_logger;-static void ggml_vk_destroy_buffer(vk_buffer& buf);-static void ggml_vk_synchronize(ggml_backend_vk_context * ctx);--static constexpr uint32_t mul_mat_vec_max_cols = 8;-static constexpr uint32_t p021_max_gqa_ratio = 8;--enum vk_device_architecture {- OTHER,- AMD_GCN,- AMD_RDNA1,- AMD_RDNA2,- AMD_RDNA3,- INTEL_XE1,- INTEL_XE2,- NVIDIA_PRE_TURING,- NVIDIA_TURING,-};-static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) {vk::PhysicalDeviceProperties props = device.getProperties();@@ -547,706 +125,10 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice&return vk_device_architecture::OTHER;}-enum vk_conv_shapes {- CONV_SHAPE_128x128,- CONV_SHAPE_64x32,- CONV_SHAPE_32x256,- CONV_SHAPE_64x128,- CONV_SHAPE_COUNT,-};--struct vk_conv_block_size {- uint32_t K;- uint32_t NPQ;- uint32_t CRS;-};--vk_conv_block_size vk_conv_block_sizes[CONV_SHAPE_COUNT] = {- // K NPQ CRS- { 128, 128, 16 }, // CONV_SHAPE_128x128- { 64, 32, 32 }, // CONV_SHAPE_64x32- { 32, 256, 16 }, // CONV_SHAPE_32x256- { 64, 128, 16 }, // CONV_SHAPE_64x128-};--enum dmmv_wg_sizes {- DMMV_WG_SIZE_SUBGROUP,- DMMV_WG_SIZE_LARGE,- DMMV_WG_SIZE_COUNT,-};--enum FaCodePath {- FA_SCALAR,- FA_COOPMAT1,- FA_COOPMAT2,-};--struct vk_fa_pipeline_state {- uint32_t HSK, HSV;- uint32_t Br, Bc;- uint32_t D_split, row_split;- bool shmem_staging;- FaCodePath path;- uint32_t workgroup_size, subgroup_size;- bool aligned;- bool f32acc;- uint32_t flags;- uint32_t limit_occupancy_shmem;- ggml_type k_type;- ggml_type v_type;-- bool operator<(const vk_fa_pipeline_state &b) const {- return std::tie(HSK, HSV, Br, Bc, D_split, row_split, shmem_staging, path, workgroup_size, subgroup_size, aligned, f32acc, flags, limit_occupancy_shmem, k_type, v_type) <- std::tie(b.HSK, b.HSV, b.Br, b.Bc, b.D_split, b.row_split, b.shmem_staging, b.path, b.workgroup_size, b.subgroup_size, b.aligned, b.f32acc, b.flags, b.limit_occupancy_shmem, b.k_type, b.v_type);- }-};--struct vk_conv2d_pipeline_state {- vk_conv2d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t p0, uint32_t p1, uint32_t d0, uint32_t d1, uint32_t KW, uint32_t KH, uint32_t aligned)- : s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), KW(KW), KH(KH), aligned(aligned) {}-- uint32_t s0, s1, p0, p1, d0, d1, KW, KH;- // when set, shader can skip K/CRS/NPQ bounds checks and address clamps- uint32_t aligned;-- bool operator<(const vk_conv2d_pipeline_state &b) const {- return std::tie(s0, s1, p0, p1, d0, d1, KW, KH, aligned) <- std::tie(b.s0, b.s1, b.p0, b.p1, b.d0, b.d1, b.KW, b.KH, b.aligned);- }-};--struct vk_conv3d_pipeline_state {- vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,- uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)- : s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}-- uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;- uint32_t aligned;-- bool operator<(const vk_conv3d_pipeline_state &b) const {- return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <- std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);- }-};--struct vk_solve_tri_pipeline_state {- vk_solve_tri_pipeline_state(uint32_t N, uint32_t K)- : N(N), K(K) {}-- uint32_t N, K;-- bool operator<(const vk_solve_tri_pipeline_state &b) const {- return std::tie(N, K) <- std::tie(b.N, b.K);- }-};--enum shader_reduction_mode {- SHADER_REDUCTION_MODE_SHMEM,- SHADER_REDUCTION_MODE_HYBRID,- SHADER_REDUCTION_MODE_SUBGROUP,- SHADER_REDUCTION_MODE_COUNT,-};--// argsort pipelines for up to 1<<10 invocations per workgroup-static constexpr uint32_t num_argsort_pipelines = 11;-static constexpr uint32_t num_topk_moe_pipelines = 10;-static constexpr uint32_t num_topk_pipelines = 11;--static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,- GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE,- GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV,- GGML_OP_RESHAPE };--static constexpr std::initializer_list<ggml_op> topk_moe_sigmoid_norm_bias{ GGML_OP_UNARY, GGML_OP_RESHAPE, GGML_OP_ADD,- GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS,- GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP,- GGML_OP_DIV, GGML_OP_RESHAPE };--static constexpr std::initializer_list<ggml_op> topk_moe_sqrt_softplus_norm_bias{ GGML_OP_UNARY, GGML_OP_SQRT,- GGML_OP_RESHAPE, GGML_OP_ADD,- GGML_OP_ARGSORT, GGML_OP_VIEW,- GGML_OP_GET_ROWS, GGML_OP_RESHAPE,- GGML_OP_SUM_ROWS, GGML_OP_CLAMP,- GGML_OP_DIV, GGML_OP_RESHAPE };--static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax { GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,- GGML_OP_VIEW, GGML_OP_GET_ROWS };--static constexpr std::initializer_list<ggml_op> topk_moe_late_softmax { GGML_OP_ARGSORT, GGML_OP_VIEW,- GGML_OP_GET_ROWS, GGML_OP_RESHAPE,- GGML_OP_SOFT_MAX, GGML_OP_RESHAPE };--// Snake activation: y = x + sin(a*x)^2 * inv_b. Used by the optimize_graph reorder-// pass so it keeps the chain contiguous and by the dispatcher to detect the fusion.-static constexpr std::initializer_list<ggml_op> snake_pattern { GGML_OP_MUL, GGML_OP_SIN,- GGML_OP_SQR, GGML_OP_MUL,- GGML_OP_ADD };--// qwen4 QSA indexer: gather per-block scores to cells + add f16 mask (cast+reshape) + top-k,-// fused into one radix-select. The cast/reshape are elided; the raw f16 mask is read in-shader.-static constexpr std::initializer_list<ggml_op> topk_qsa_pattern { GGML_OP_GET_ROWS, GGML_OP_PERMUTE,- GGML_OP_CONT, GGML_OP_CPY,- GGML_OP_RESHAPE, GGML_OP_ADD,- GGML_OP_TOP_K };-static constexpr std::initializer_list<std::array<int, 3>> topk_qsa_edges {- { 1, 0, 0 }, // permute->src[0] == get_rows- { 2, 0, 1 }, // cont->src[0] == permute- { 4, 0, 3 }, // reshape->src[0] == cpy (mask cast)- { 5, 0, 2 }, // add->src[0] == cont- { 5, 1, 4 }, // add->src[1] == reshape- { 6, 0, 5 }, // top_k->src[0] == add-};-static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_mul_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_MUL };-static constexpr std::initializer_list<ggml_op> rms_norm_mul_add_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD };-static constexpr std::initializer_list<ggml_op> rms_norm_mul_rope_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };-static constexpr std::initializer_list<ggml_op> rms_norm_view_set_rows_pattern { GGML_OP_RMS_NORM, GGML_OP_VIEW, GGML_OP_SET_ROWS };-static constexpr std::initializer_list<ggml_op> rope_view_set_rows_pattern { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };--//node #978 ( SOFT_MAX): ffn_moe_probs-15 ( 0K) [Vulka ] use=2: ffn_moe_logits-15 ( 0K) [Vulka ]-//node #979 ( RESHAPE): ffn_moe_probs-15 (re ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ]-//node #980 ( ARGSORT): ffn_moe_argsort-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 ( 0K) [Vulka ]-//node #981 ( VIEW): ffn_moe_topk-15 ( 0K) [Vulka ] use=4: ffn_moe_argsort-15 ( 0K) [Vulka ]-//node #982 ( GET_ROWS): ffn_moe_weights-15 ( 0K) [Vulka ] use=1: ffn_moe_probs-15 (re ( 0K) [Vulka ] ffn_moe_topk-15 ( 0K) [Vulka ]-//node #983 ( RESHAPE): ffn_moe_weights-15 ( ( 0K) [Vulka ] use=2: ffn_moe_weights-15 ( 0K) [Vulka ]-//node #984 ( SUM_ROWS): ffn_moe_weights_sum- ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ]-//node #985 ( CLAMP): ffn_moe_weights_sum_ ( 0K) [Vulka ] use=1: ffn_moe_weights_sum- ( 0K) [Vulka ]-//node #986 ( DIV): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights-15 ( ( 0K) [Vulka ] ffn_moe_weights_sum_ ( 0K) [Vulka ]-//node #987 ( RESHAPE): ffn_moe_weights_norm ( 0K) [Vulka ] use=1: ffn_moe_weights_norm ( 0K) [Vulka ]-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_norm_edges {- { 1, 0, 0 }, // reshape->src[0] == softmax- { 2, 0, 0 }, // argsort->src[0] == softmax- { 3, 0, 2 }, // view->src[0] == argsort- { 4, 0, 1 }, // get_rows->src[0] == reshape- { 4, 1, 3 }, // get_rows->src[1] == view- { 5, 0, 4 }, // reshape->src[0] == get_rows- { 6, 0, 5 }, // sum_rows->src[0] == reshape- { 7, 0, 6 }, // clamp->src[0] == sum_rows- { 8, 0, 5 }, // div->src[0] == reshape- { 8, 1, 7 }, // div->src[1] == clamp- { 9, 0, 8 }, // reshape->src[0] == div-};--//node #436 ( UNARY): ffn_moe_probs-10 ( 256K) [Vulka ] use=2: ffn_moe_logits-10 ( 256K) [Vulka ]-//node #437 ( RESHAPE): ffn_moe_probs-10 (re ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ]-//node #438 ( ADD): ffn_moe_probs_biased ( 256K) [Vulka ] use=1: ffn_moe_probs-10 ( 256K) [Vulka ] blk.10.exp_probs_b.b ( 0K) [Vulka ]-//node #439 ( ARGSORT): ffn_moe_argsort-10 ( 256K) [Vulka ] use=1: ffn_moe_probs_biased ( 256K) [Vulka ]-//node #440 ( VIEW): ffn_moe_topk-10 ( 255K) [Vulka ] use=3: ffn_moe_argsort-10 ( 256K) [Vulka ]-//node #441 ( GET_ROWS): ffn_moe_weights-10 ( 12K) [Vulka ] use=1: ffn_moe_probs-10 (re ( 256K) [Vulka ] ffn_moe_topk-10 ( 255K) [Vulka ]-//node #442 ( RESHAPE): ffn_moe_weights-10 ( ( 12K) [Vulka ] use=2: ffn_moe_weights-10 ( 12K) [Vulka ]-//node #443 ( SUM_ROWS): ffn_moe_weights_sum- ( 2K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ]-//node #444 ( CLAMP): ffn_moe_weights_sum_ ( 2K) [Vulka ] use=1: ffn_moe_weights_sum- ( 2K) [Vulka ]-//node #445 ( DIV): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights-10 ( ( 12K) [Vulka ] ffn_moe_weights_sum_ ( 2K) [Vulka ]-//node #446 ( RESHAPE): ffn_moe_weights_norm ( 12K) [Vulka ] use=1: ffn_moe_weights_norm ( 12K) [Vulka ]-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sigmoid_norm_bias_edges {- { 1, 0, 0 }, // reshape->src[0] == sigmoid- { 2, 0, 0 }, // add->src[0] == sigmoid- { 3, 0, 2 }, // argsort->src[0] == add- { 4, 0, 3 }, // view->src[0] == argsort- { 5, 0, 1 }, // get_rows->src[0] == reshape- { 5, 1, 4 }, // get_rows->src[1] == view- { 6, 0, 5 }, // reshape->src[0] == get_rows- { 7, 0, 6 }, // sum_rows->src[0] == reshape- { 8, 0, 7 }, // clamp->src[0] == sum_rows- { 9, 0, 6 }, // div->src[0] == reshape- { 9, 1, 8 }, // div->src[1] == clamp- {10, 0, 9 }, // reshape->src[0] == div-};--static constexpr std::initializer_list<std::array<int, 3>> topk_moe_sqrt_softplus_norm_bias_edges {- { 1, 0, 0 }, // sqrt->src[0] == softplus- { 2, 0, 1 }, // reshape->src[0] == sqrt- { 3, 0, 1 }, // add->src[0] == sqrt- { 4, 0, 3 }, // argsort->src[0] == add- { 5, 0, 4 }, // view->src[0] == argsort- { 6, 0, 2 }, // get_rows->src[0] == reshape- { 6, 1, 5 }, // get_rows->src[1] == view- { 7, 0, 6 }, // reshape->src[0] == get_rows- { 8, 0, 7 }, // sum_rows->src[0] == reshape- { 9, 0, 8 }, // clamp->src[0] == sum_rows- {10, 0, 7 }, // div->src[0] == reshape- {10, 1, 9 }, // div->src[1] == clamp- {11, 0,10 }, // reshape->src[0] == div-};--// same as early_softmax_norm but ending after the get_rows-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_early_softmax_edges {- { 1, 0, 0 }, // reshape->src[0] == softmax- { 2, 0, 0 }, // argsort->src[0] == softmax- { 3, 0, 2 }, // view->src[0] == argsort- { 4, 0, 1 }, // get_rows->src[0] == reshape- { 4, 1, 3 }, // get_rows->src[1] == view-};--//node #652 ( ARGSORT): ffn_moe_argsort-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 ( 0K) [Vulka ]-//node #653 ( VIEW): ffn_moe_topk-11 ( 0K) [Vulka ] use=7: ffn_moe_argsort-11 ( 0K) [Vulka ]-//node #654 ( GET_ROWS): ffn_moe_weights-11 ( 0K) [Vulka ] use=1: ffn_moe_probs-11 (re ( 0K) [Vulka ] ffn_moe_topk-11 ( 0K) [Vulka ]-//node #655 ( RESHAPE): ffn_moe_weights-11 ( ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( 0K) [Vulka ]-//node #656 ( SOFT_MAX): node_656 ( 0K) [Vulka ] use=1: ffn_moe_weights-11 ( ( 0K) [Vulka ]-//node #657 ( RESHAPE): ffn_moe_weights_soft ( 0K) [Vulka ] use=1: node_656 ( 0K) [Vulka ]-static constexpr std::initializer_list<std::array<int, 3>> topk_moe_late_softmax_edges {- { 1, 0, 0 }, // view->src[0] == argsort- { 2, 1, 1 }, // get_rows->src[1] == view- { 3, 0, 2 }, // reshape->src[0] == get_rows- { 4, 0, 3 }, // soft_max->src[0] == reshape- { 5, 0, 4 }, // reshape->src[0] == soft_max-};--enum topk_moe_mode {- TOPK_MOE_EARLY_SOFTMAX,- TOPK_MOE_EARLY_SOFTMAX_NORM,- TOPK_MOE_LATE_SOFTMAX,- TOPK_MOE_SIGMOID_NORM_BIAS,- TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS,- TOPK_MOE_COUNT,-};--enum rms_norm_mode {- RMS_NORM_MUL,- RMS_NORM_MUL_ADD,- RMS_NORM_MUL_ADD_MUL,- RMS_NORM_MUL_ROPE,- RMS_NORM_MUL_ROPE_VIEW_SET_ROWS,- RMS_NORM_VIEW_SET_ROWS,- RMS_NORM_COUNT,-};--static constexpr std::initializer_list<std::array<int, 3>> rope_view_set_rows_edges {- { 1, 0, 0 }, // view->src[0] == rope- { 2, 0, 1 }, // set_rows->src[0] == view-};--static constexpr std::initializer_list<std::array<int, 3>> rms_norm_mul_rope_view_set_rows_edges {- { 1, 0, 0 }, // mul->src[0] == rms- { 2, 0, 1 }, // rope->src[0] == mul- { 3, 0, 2 }, // view->src[0] == rope- { 4, 0, 3 }, // set_rows->src[0] == view-};--static constexpr std::initializer_list<std::array<int, 3>> rms_norm_view_set_rows_edges {- { 1, 0, 0 }, // view->src[0] == rms_norm- { 2, 0, 1 }, // set_rows->src[0] == view-};--static constexpr std::array<ggml_type, 9> lightning_indexer_k_types = {- GGML_TYPE_F32,- GGML_TYPE_F16,- GGML_TYPE_BF16,- GGML_TYPE_Q8_0,- GGML_TYPE_Q5_1,- GGML_TYPE_Q5_0,- GGML_TYPE_Q4_1,- GGML_TYPE_Q4_0,- GGML_TYPE_IQ4_NL,-};--static bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) {+bool ggml_vk_lightning_indexer_k_type_supported(ggml_type type) {return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end();}--struct vk_device_struct {- std::recursive_mutex mutex;- std::mutex queue_submit_mutex;- mutable std::shared_mutex pinned_memory_mutex;-- // Guards compile_pending, all_pipelines, and the dynamic pipeline maps- // (flash_attn, fa_mask_opt, solve_tri, conv2d, etc). The actual compile- // runs with no lock held, so different pipelines can compile in parallel.- // Lock order is device->mutex -> compile_mutex, never the reverse.- std::mutex compile_mutex;- std::condition_variable compile_cv;-- uint32_t debug_cmdbuf_idx {};-- vk::PhysicalDevice physical_device;- vk::PhysicalDeviceProperties properties;- std::string name;- uint64_t max_memory_allocation_size;- uint64_t max_buffer_size;- uint64_t suballocation_block_size;- uint64_t min_imported_host_pointer_alignment;- bool external_memory_host {};- bool fp16;- bool bf16;- bool pipeline_robustness;- bool memory_priority;- vk::Device device;- uint32_t vendor_id;- vk::DriverId driver_id;- vk_device_architecture architecture;- std::unique_ptr<vk_queue> compute_queue;- std::unique_ptr<vk_queue> transfer_queue;- bool single_queue;- bool support_async;- bool async_use_transfer_queue;- bool has_internally_synchronized_queues = false;- uint32_t subgroup_size;- uint32_t subgroup_size_log2;- uint32_t shader_core_count;- bool uma;- bool prefer_host_memory;- bool float_controls_rte_fp16;- bool float_controls_denorm_preserve_fp16;- bool subgroup_basic;- bool subgroup_arithmetic;- bool subgroup_shuffle;- bool subgroup_ballot;- bool subgroup_clustered;- bool subgroup_vote;- bool multi_add;- bool shader_int64;- bool buffer_device_address;- bool vulkan_memory_model;-- bool add_rms_fusion;- uint32_t partials_binding_alignment;- uint32_t max_nodes_per_submit;-- bool shader_64b_indexing;-- bool integer_dot_product;- // 0: default, 1: force mmvq, -1: disable mmvq- int32_t mmvq_mode;-- bool subgroup_size_control;- uint32_t subgroup_min_size;- uint32_t subgroup_max_size;- bool subgroup_require_full_support;-- // floor(log2(maxComputeWorkGroupInvocations))- uint32_t max_workgroup_size_log2 {};-- bool coopmat_support;- bool coopmat_acc_f32_support {};- bool coopmat_acc_f16_support {};- bool coopmat_bf16_support {};- bool coopmat_support_16x16x16_f16acc {};- bool coopmat_support_16x16x16_f32acc {};- bool coopmat1_fa_support {};- uint32_t coopmat_m;- uint32_t coopmat_n;- uint32_t coopmat_k;-- bool coopmat_int_support;- uint32_t coopmat_int_m;- uint32_t coopmat_int_n;- uint32_t coopmat_int_k;-- bool coopmat2;- bool coopmat2_bf16_support {};- bool coopmat2_decode_vector;-- bool dot2_f16 {};- bool ocp_fp4 {};-- bool pipeline_executable_properties_support {};-- bool device_fault {};- PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};-- bool serialize_submissions {};-- const ggml_cgraph * diag_cgraph {};- int diag_prev_start = -1;- int diag_prev_end = -1;-- size_t idx;-- bool mul_mat_l[GGML_TYPE_COUNT];- bool mul_mat_m[GGML_TYPE_COUNT];- bool mul_mat_s[GGML_TYPE_COUNT];- bool mul_mat_id_l[GGML_TYPE_COUNT];- bool mul_mat_id_m[GGML_TYPE_COUNT];- bool mul_mat_id_s[GGML_TYPE_COUNT];-- // Separate flags for the q8_1 (integer dot) mmq path, whose shader uses- // a different shared-memory layout than the float matmul shaders.- bool mul_mat_l_int[GGML_TYPE_COUNT];- bool mul_mat_m_int[GGML_TYPE_COUNT];- bool mul_mat_s_int[GGML_TYPE_COUNT];- bool mul_mat_id_l_int[GGML_TYPE_COUNT];- bool mul_mat_id_m_int[GGML_TYPE_COUNT];- bool mul_mat_id_s_int[GGML_TYPE_COUNT];-- vk::DescriptorSetLayout dsl;-- std::map<vk_matmul_pipeline_key, std::vector<vk_matmul_pipeline_pair>> pipeline_matmul;- matmul_tile_selector_t matmul_tile_selector;- matmul_tile_selector_t matmul_id_tile_selector;-- vk_pipeline pipeline_matmul_split_k_reduce;- vk_pipeline pipeline_quantize_q8_1_x4;-- vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];- vk_pipeline pipeline_dequant_transpose[GGML_TYPE_COUNT]; // fused dequant+transpose for FA quant-KV- vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];- vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];- vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];-- vk_pipeline pipeline_dequant_mul_mat_vec_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT][mul_mat_vec_max_cols];- vk_pipeline pipeline_dequant_mul_mat_vec_id_q8_1_f32[DMMV_WG_SIZE_COUNT][GGML_TYPE_COUNT];-- vk_pipeline pipeline_mul_mat_vec_p021_f16_f32[p021_max_gqa_ratio];- vk_pipeline pipeline_mul_mat_vec_nc_f16_f32;- vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT];- vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT];- vk_pipeline pipeline_get_rows_back_f32;- vk_pipeline pipeline_acc_f32;- vk_pipeline pipeline_set_f32;-- // [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]- vk_pipeline pipeline_add[2][2][2];- vk_pipeline pipeline_add_norepeat[2][2][2];- vk_pipeline pipeline_sub[2][2][2];- vk_pipeline pipeline_sub_norepeat[2][2][2];- vk_pipeline pipeline_mul[2][2][2];- vk_pipeline pipeline_mul_norepeat[2][2][2];- vk_pipeline pipeline_div[2][2][2];- vk_pipeline pipeline_div_norepeat[2][2][2];- vk_pipeline pipeline_add_rms[2][2][2];- vk_pipeline pipeline_add_rms_norepeat[2][2][2];-- // indexed by num_additional_fused_ops == num_adds - 1- vk_pipeline pipeline_multi_add[MAX_FUSED_ADDS];- vk_pipeline pipeline_multi_add_rms[MAX_FUSED_ADDS];-- vk_pipeline pipeline_add_id_f32;-- vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;- vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;- vk_pipeline pipeline_scale_f32;- vk_pipeline pipeline_log[2];- vk_pipeline pipeline_tri[2];- vk_pipeline pipeline_diag[2];- vk_pipeline pipeline_clamp[2];- vk_pipeline pipeline_pad_f32;- vk_pipeline pipeline_pad_reflect_1d_f32;- vk_pipeline pipeline_roll_f32;- vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;- vk_pipeline pipeline_repeat_i16;- vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;- vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;- vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];- vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];- vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32;- vk_pipeline pipeline_cpy_transpose_02_16, pipeline_cpy_transpose_02_32;- // [src0 0=fp32,1=fp16][dst]- vk_pipeline pipeline_set_rows_i32[2][GGML_TYPE_COUNT];- vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];- vk_pipeline pipeline_norm_f32;- vk_pipeline pipeline_group_norm_f32;- vk_pipeline pipeline_rms_norm_f32;- vk_pipeline pipeline_rms_norm_mul_f32;- vk_pipeline pipeline_rms_norm_mul_add_f32;- vk_pipeline pipeline_rms_norm_mul_add_mul_f32;- vk_pipeline pipeline_rms_norm_mul_add_partials_f32;- vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;- vk_pipeline pipeline_rms_norm_set_rows_f32_f32;- vk_pipeline pipeline_rms_norm_set_rows_f32_f16;- vk_pipeline pipeline_rms_norm_partials_f32;- vk_pipeline pipeline_rms_norm_mul_partials_f32;- vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;- vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;- vk_pipeline pipeline_rms_norm_back_f32;- vk_pipeline pipeline_l2_norm_f32;-- // [src/dst 0=fp32,1=fp16]- vk_pipeline pipeline_exp[2];- vk_pipeline pipeline_expm1[2];- vk_pipeline pipeline_elu[2];- vk_pipeline pipeline_gelu[2];- vk_pipeline pipeline_gelu_erf[2];- vk_pipeline pipeline_gelu_quick[2];- vk_pipeline pipeline_silu[2];- vk_pipeline pipeline_relu[2];- vk_pipeline pipeline_sqr[2];- vk_pipeline pipeline_sqrt[2];- vk_pipeline pipeline_sin[2];- vk_pipeline pipeline_cos[2];- vk_pipeline pipeline_xielu[2];- vk_pipeline pipeline_neg[2];- vk_pipeline pipeline_tanh[2];- vk_pipeline pipeline_sigmoid[2];- vk_pipeline pipeline_hardsigmoid[2];- vk_pipeline pipeline_hardswish[2];- vk_pipeline pipeline_abs[2];- vk_pipeline pipeline_softplus[2];- vk_pipeline pipeline_step[2];- vk_pipeline pipeline_round[2];- vk_pipeline pipeline_ceil[2];- vk_pipeline pipeline_floor[2];- vk_pipeline pipeline_trunc[2];- vk_pipeline pipeline_sgn[2];-- // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]- vk_pipeline pipeline_unary_mul[4][2][2][2];-- vk_pipeline pipeline_add1_f16_f16;- vk_pipeline pipeline_add1_f16_f32;- vk_pipeline pipeline_add1_f32_f32;-- vk_pipeline pipeline_arange_f32;-- vk_pipeline pipeline_fill_f32;- vk_pipeline pipeline_fill_f16;-- vk_pipeline pipeline_geglu[2];- vk_pipeline pipeline_reglu[2];- vk_pipeline pipeline_swiglu[2];- vk_pipeline pipeline_swiglu_oai[2];- vk_pipeline pipeline_swiglu_clamp[2];- vk_pipeline pipeline_geglu_erf[2];- vk_pipeline pipeline_geglu_quick[2];-- vk_pipeline pipeline_leaky_relu[2];- vk_pipeline pipeline_silu_back_f32;- vk_pipeline pipeline_diag_mask_inf_f32;- vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;- vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;- vk_pipeline pipeline_soft_max_back_f32;-- vk_pipeline pipeline_soft_max_large1_f32, pipeline_soft_max_large1_f32_f16;- vk_pipeline pipeline_soft_max_large2_f32, pipeline_soft_max_large2_f32_f16;- vk_pipeline pipeline_soft_max_large3_f32, pipeline_soft_max_large3_f32_f16;-- vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16, pipeline_rope_norm_f32_f16;- vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16, pipeline_rope_neox_f32_f16;- vk_pipeline pipeline_rope_multi_f32, pipeline_rope_multi_f16, pipeline_rope_multi_f32_f16;- vk_pipeline pipeline_rope_vision_f32, pipeline_rope_vision_f16;- vk_pipeline pipeline_argsort_f32[num_argsort_pipelines];- vk_pipeline pipeline_argsort_large_f32[num_argsort_pipelines];- vk_pipeline pipeline_topk_f32[num_topk_pipelines];- vk_pipeline pipeline_topk_radix_f32;- vk_pipeline pipeline_topk_radix_qsa; // qwen4 QSA indexer fusion (f16 mask)- vk_pipeline pipeline_sum_rows_f32;- vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512;- vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512;- vk_pipeline pipeline_fwht_f32[4];- vk_pipeline pipeline_cumsum_f32;- vk_pipeline pipeline_cumsum_small_f32;- vk_pipeline pipeline_cumsum_multipass1_f32;- vk_pipeline pipeline_cumsum_multipass2_f32;- vk_pipeline pipeline_argmax_f32;- vk_pipeline pipeline_count_equal_i32;- vk_pipeline pipeline_dsv4_hc_comb_f32;- vk_pipeline pipeline_dsv4_hc_pre_f32;- vk_pipeline pipeline_dsv4_hc_pre_gated_f32;- vk_pipeline pipeline_dsv4_hc_post_f32;- vk_pipeline pipeline_dsv4_hc_post_nocomb_f32;- std::map<vk_solve_tri_pipeline_state, vk_pipeline> pipeline_solve_tri_f32;- vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16;- vk_pipeline pipeline_im2col_3d_f32, pipeline_im2col_3d_f32_f16;- vk_pipeline pipeline_timestep_embedding_f32;- vk_pipeline pipeline_conv_transpose_1d_f32;- vk_pipeline pipeline_col2im_1d_f32;- vk_pipeline pipeline_col2im_1d_f16;- vk_pipeline pipeline_col2im_1d_bf16;- vk_pipeline pipeline_out_prod_f32;- vk_pipeline pipeline_snake_f32;- vk_pipeline pipeline_snake_f16;- vk_pipeline pipeline_snake_bf16;- vk_pipeline pipeline_pool1d_f32;- vk_pipeline pipeline_pool2d_f32;- vk_pipeline pipeline_rwkv_wkv6_f32;- vk_pipeline pipeline_rwkv_wkv7_f32;- vk_pipeline pipeline_gated_linear_attn_f32;- vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];- // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128- vk_pipeline pipeline_gated_delta_net[4][2];- vk_pipeline pipeline_ssm_scan_f32_d128;- vk_pipeline pipeline_ssm_scan_f32_d256;- vk_pipeline pipeline_ssm_conv_f32;- vk_pipeline pipeline_ssm_conv_silu_f32;- vk_pipeline pipeline_ssm_conv_bias_silu_f32;- vk_pipeline pipeline_opt_step_adamw_f32;- vk_pipeline pipeline_opt_step_sgd_f32;- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f32[CONV_SHAPE_COUNT];- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv2d_f16_f32[CONV_SHAPE_COUNT];- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f32[CONV_SHAPE_COUNT];- std::map<vk_conv2d_pipeline_state, vk_pipeline> pipeline_conv_transpose_2d_f16_f32[CONV_SHAPE_COUNT];- std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f32[CONV_SHAPE_COUNT];- std::map<vk_conv3d_pipeline_state, vk_pipeline> pipeline_conv3d_f16_f32[CONV_SHAPE_COUNT];- vk_pipeline pipeline_conv2d_dw_whcn_f32, pipeline_conv2d_dw_whcn_f16_f32;- vk_pipeline pipeline_conv2d_dw_cwhn_f32, pipeline_conv2d_dw_cwhn_f16_f32;-- std::map<vk_fa_pipeline_state, vk_pipeline> pipeline_flash_attn_f32_f16;-- std::map<std::pair<uint32_t, uint32_t>, vk_pipeline> pipeline_fa_mask_opt;-- vk_pipeline pipeline_fa_sparse_compact;- vk_pipeline pipeline_fa_sparse_compact_subgroup;- bool fa_sparse_compact_use_subgroups;-- vk_pipeline pipeline_flash_attn_split_k_reduce;- vk_pipeline pipeline_count_experts;-- // [2] is for whether to take n_experts from spec constant (0) or push constant (1)- vk_pipeline pipeline_topk_moe[num_topk_moe_pipelines][2];-- std::vector<vk_pipeline_ref> all_pipelines;-- std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory;-- vk::Fence fence;- vk_buffer sync_staging;-- ggml_backend_buffer_type buffer_type;-- bool disable_fusion;- bool disable_host_visible_vidmem;- bool allow_sysmem_fallback;- bool disable_graph_optimize;-- std::unique_ptr<vk_memory_logger> memory_logger;-- ~vk_device_struct() {- VK_LOG_DEBUG("destroy device " << name);-- device.destroyFence(fence);-- ggml_vk_destroy_buffer(sync_staging);-- if (compute_queue) compute_queue->cmd_pool.destroy(device);- if (transfer_queue) transfer_queue->cmd_pool.destroy(device);-- // Explicitly clear to ensure queues drop their shared_ptrs to handles- // before the Vulkan logical device instance is destroyed- compute_queue.reset();- transfer_queue.reset();-- for (auto& pipeline : all_pipelines) {- if (pipeline.expired()) {- continue;- }-- vk_pipeline pl = pipeline.lock();- ggml_vk_destroy_pipeline(device, pl);- }- all_pipelines.clear();-- device.destroyDescriptorSetLayout(dsl);-- device.destroy();- }-};--void vk_command_pool::init(vk_device& device, vk_queue *q_) {- cmd_buffers.clear();- q = q_;-- vk::CommandPoolCreateInfo command_pool_create_info(- vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT),- q->queue_family_index);- pool = device->device.createCommandPool(command_pool_create_info);-}--void vk_command_pool::destroy(vk::Device& device) {- device.destroyCommandPool(pool);- pool = nullptr;- cmd_buffers.clear();-}--static void ggml_vk_print_device_fault_info(const vk_device& device) {+void ggml_vk_print_device_fault_info(const vk_device& device) {if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {return;}@@ -1271,1043 +153,32 @@ static void ggml_vk_print_device_fault_info(const vk_device& device) {if (res != VK_SUCCESS) {GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);return;- }-- if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {- return;- }-- if (fault_info.description[0] != '\0') {- GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);- }-- for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {- const auto& info = address_infos[i];- GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",- i, (int)info.addressType,- (unsigned long long)info.reportedAddress,- (unsigned long long)info.addressPrecision);- }- for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {- const auto& info = vendor_infos[i];- GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",- i, info.description,- (unsigned long long)info.vendorFaultCode,- (unsigned long long)info.vendorFaultData);- }-}--struct vk_buffer_struct {- vk::Buffer buffer = VK_NULL_HANDLE;- vk::DeviceMemory device_memory = VK_NULL_HANDLE;- vk::MemoryPropertyFlags memory_property_flags;- void * ptr;- size_t size = 0;- vk::DeviceAddress bda_addr {};-- vk_device device;-- ~vk_buffer_struct() {- if (size == 0) {- return;- }- VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");-- device->device.freeMemory(device_memory);- device->device.destroyBuffer(buffer);- }-};--struct vk_subbuffer {- vk_buffer buffer;- uint64_t offset;- uint64_t size;-- operator vk::DescriptorBufferInfo() const {- return { buffer->buffer, offset, size };- }-};--struct vk_semaphore {- vk::Semaphore s;- uint64_t value;-};--// vk_event is used for the event-related backend interfaces. It uses vk::Events for-// event_wait and a timeline semaphore for event_synchronize. Polling on an event for-// event_synchronize wouldn't be sufficient to wait for command buffers to complete,-// and would lead to validation errors.-struct vk_event {- std::vector<vk::Event> events_free; // Events available for reuse- std::vector<vk::Event> events_submitted; // Events that are fully submitted and can be reused on next synchronize- vk::Event event;- bool has_event;-- vk_semaphore tl_semaphore;- vk_command_buffer* cmd_buffer = nullptr;- uint64_t cmd_buffer_use_counter = 0;-};--struct vk_submission {- vk_command_buffer* buffer = nullptr;- std::vector<vk_semaphore> wait_semaphores;- std::vector<vk_semaphore> signal_semaphores;-};--typedef std::vector<vk_submission> vk_sequence;--struct vk_mat_mat_push_constants {- uint32_t M; uint32_t N; uint32_t K;- uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;- uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;- uint32_t base_work_group_z; uint32_t num_batches;- uint32_t k_split;- uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;- uint32_t padded_N;-};--#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1-#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2-#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4-#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8--struct vk_mat_vec_push_constants {- uint32_t ncols;- uint32_t stride_a;- uint32_t stride_b;- uint32_t stride_d;- uint32_t batch_stride_a;- uint32_t batch_stride_b;- uint32_t batch_stride_d;- uint32_t fusion_flags;- uint32_t base_work_group_y;- uint32_t ne02;- uint32_t ne12;- uint32_t broadcast2;- uint32_t broadcast3;-};--struct vk_mat_vec_p021_push_constants {- uint32_t ncols_x;- uint32_t nrows_x;- uint32_t nchannels_x;- uint32_t nchannels_y;- uint32_t b_offset;- uint32_t d_offset;- uint32_t fusion_flags;-};--struct vk_mat_vec_nc_push_constants {- uint32_t ncols_x;- uint32_t nrows_x;- uint32_t row_stride_x;- uint32_t channel_stride_x;- uint32_t channel_stride_y;- uint32_t channel_x_divisor;- uint32_t ne12;- uint32_t b_offset;- uint32_t d_offset;- uint32_t nb03;- uint32_t nb13;- uint32_t nb23;- uint32_t fusion_flags;-};--struct vk_mat_mat_id_push_constants {- uint32_t M; uint32_t N; uint32_t K;- uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;- uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;- uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;- uint32_t n_experts;- uint32_t hoist_row_ids;-};-struct vk_mat_vec_id_push_constants {- uint32_t ncols;- uint32_t stride_a;- uint32_t stride_b;- uint32_t stride_d;- uint32_t batch_stride_a;- uint32_t batch_stride_b;- uint32_t batch_stride_d;- uint32_t fusion_flags;- uint32_t nei0;- uint32_t ne11;- uint32_t expert_i1;- uint32_t nbi1;-};--struct vk_flash_attn_push_constants {- uint32_t N;- uint32_t KV;-- uint32_t ne1;- uint32_t ne2;- uint32_t ne3;-- uint32_t neq2;- uint32_t neq3;- uint32_t nek2;- uint32_t nek3;- uint32_t nev2;- uint32_t nev3;- uint32_t nem1;- uint32_t nem2;- uint32_t nem3;-- uint32_t nb01;- uint32_t nb02;- uint32_t nb03;- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;- uint32_t nb21;- uint32_t nb22;- uint32_t nb23;-- float scale;- float max_bias;- float logit_softcap;-- uint32_t mask_n_head_log2;- float m0;- float m1;-- uint32_t gqa_ratio;- uint32_t split_kv;- uint32_t k_num;-};-static_assert(sizeof(vk_flash_attn_push_constants) <= 128, "sizeof(vk_flash_attn_push_constants) must be <= 128");--struct vk_op_push_constants {- uint32_t KX;- uint32_t KY;- float param1;- float param2;- float param3;- float param4;-};--struct vk_op_fwht_push_constants {- uint32_t n_rows;- uint32_t src_offset;- uint32_t dst_offset;- float scale;-};--struct vk_op_dsv4_hc_comb_push_constants {- uint32_t n_tokens;-- uint32_t nbm0; uint32_t nbm1;- uint32_t nbs0;- uint32_t nbb0;- uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;-- uint32_t m_offset;- uint32_t s_offset;- uint32_t b_offset;- uint32_t d_offset;-- float eps;- uint32_t n_iter;-};--struct vk_op_dsv4_hc_pre_push_constants {- uint32_t n_embd;- uint32_t n_tokens;-- uint32_t nbx0; uint32_t nbx1; uint32_t nbx2;- uint32_t nbw0; uint32_t nbw1; uint32_t nbw2;- uint32_t nbd0; uint32_t nbd1;-- uint32_t x_offset;- uint32_t w_offset;- uint32_t d_offset;-- float scale;-};--struct vk_op_dsv4_hc_post_push_constants {- uint32_t n_embd;- uint32_t n_tokens;-- uint32_t nbx0; uint32_t nbx1;- uint32_t nbr0; uint32_t nbr1; uint32_t nbr2;- uint32_t nbp0; uint32_t nbp1;- uint32_t nbc0; uint32_t nbc1; uint32_t nbc2;- uint32_t nbd0; uint32_t nbd1; uint32_t nbd2;-- uint32_t x_offset;- uint32_t r_offset;- uint32_t p_offset;- uint32_t c_offset;- uint32_t d_offset;-};--struct vk_op_count_experts_push_constants {- uint32_t ne00;- uint32_t ne01;- uint32_t nb00;- uint32_t nb01;- uint32_t a_offset;- uint32_t n_experts;- uint32_t hoist_row_ids;- uint32_t ne00mp;- uint32_t ne00L;-};--struct vk_op_glu_push_constants {- uint32_t N;- uint32_t ne00;- uint32_t ne20;- uint32_t mode; // 0: default, 1: swapped, 2: split- float alpha; // for swiglu_oai- float limit;- uint32_t nb00;- uint32_t nb01;- uint32_t nb02;- uint32_t nb03;- uint32_t nb10;- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;- uint32_t nb20;- uint32_t nb21;- uint32_t nb22;- uint32_t nb23;- uint32_t ne21;- uint32_t ne22;- uint32_t misalign_offsets;- uint32_t ne2_012mp; uint32_t ne2_012L;- uint32_t ne2_01mp; uint32_t ne2_01L;- uint32_t ne2_0mp; uint32_t ne2_0L;-};-static_assert(sizeof(vk_op_glu_push_constants) <= 128, "sizeof(vk_op_glu_push_constants) must be <= 128");--struct vk_op_unary_push_constants {- uint32_t ne;- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;- uint32_t misalign_offsets;- float param1; float param2; float param3; float param4;- uint32_t ne0_012mp; uint32_t ne0_01mp; uint32_t ne0_0mp; uint32_t ne0_Ls;- uint32_t ne1_012mp; uint32_t ne1_01mp; uint32_t ne1_0mp; uint32_t ne1_Ls;-};-static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");--static vk_op_unary_push_constants vk_op_unary_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst, int64_t ne = 0) {- GGML_ASSERT(ne != 0 || (ggml_nelements(src0) == ggml_nelements(dst)));- ne = ne != 0 ? ne : ggml_nelements(dst);- GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());-- vk_op_unary_push_constants p{};- p.ne = (uint32_t)ne;-- size_t src0_tsize = ggml_type_size(src0->type);- p.ne00 = (uint32_t)src0->ne[0];- p.ne01 = (uint32_t)src0->ne[1];- p.ne02 = (uint32_t)src0->ne[2];- p.ne03 = (uint32_t)src0->ne[3];- p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);- p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);- p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);- p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);-- size_t dst_tsize = ggml_type_size(dst->type);- p.ne10 = (uint32_t)dst->ne[0];- p.ne11 = (uint32_t)dst->ne[1];- p.ne12 = (uint32_t)dst->ne[2];- p.ne13 = (uint32_t)dst->ne[3];- p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);- p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);- p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);- p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);-- return p; // offsets are initialized later in ggml_vk_op-}--struct vk_op_pad_push_constants {- uint32_t ne;- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;- uint32_t misalign_offsets;- uint32_t circular;-- uint32_t lp0; uint32_t rp0;- uint32_t lp1; uint32_t rp1;- uint32_t lp2; uint32_t rp2;- uint32_t lp3; uint32_t rp3;-};--static vk_op_pad_push_constants vk_op_pad_push_constants_init(const ggml_tensor * src0, const ggml_tensor * dst) {- int64_t ne = ggml_nelements(dst);- GGML_ASSERT(ne <= (int64_t)std::numeric_limits<uint32_t>::max());-- vk_op_pad_push_constants p{};- p.ne = (uint32_t)ne;-- size_t src0_tsize = ggml_type_size(src0->type);- p.ne00 = (uint32_t)src0->ne[0];- p.ne01 = (uint32_t)src0->ne[1];- p.ne02 = (uint32_t)src0->ne[2];- p.ne03 = (uint32_t)src0->ne[3];- p.nb00 = (uint32_t)(src0->nb[0] / src0_tsize);- p.nb01 = (uint32_t)(src0->nb[1] / src0_tsize);- p.nb02 = (uint32_t)(src0->nb[2] / src0_tsize);- p.nb03 = (uint32_t)(src0->nb[3] / src0_tsize);-- size_t dst_tsize = ggml_type_size(dst->type);- p.ne10 = (uint32_t)dst->ne[0];- p.ne11 = (uint32_t)dst->ne[1];- p.ne12 = (uint32_t)dst->ne[2];- p.ne13 = (uint32_t)dst->ne[3];- p.nb10 = (uint32_t)(dst->nb[0] / dst_tsize);- p.nb11 = (uint32_t)(dst->nb[1] / dst_tsize);- p.nb12 = (uint32_t)(dst->nb[2] / dst_tsize);- p.nb13 = (uint32_t)(dst->nb[3] / dst_tsize);-- p.lp0 = dst->op_params[0];- p.rp0 = dst->op_params[1];- p.lp1 = dst->op_params[2];- p.rp1 = dst->op_params[3];- p.lp2 = dst->op_params[4];- p.rp2 = dst->op_params[5];- p.lp3 = dst->op_params[6];- p.rp3 = dst->op_params[7];- p.circular = dst->op_params[8];-- return p; // fastdiv values and offsets are initialized later in ggml_vk_op-}--// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.-// Precompute mp (m' in the paper) and L such that division-// can be computed using a multiply (high 32b of 64b result)-// and a shift:-//-// n/d = (mulhi(n, mp) + n) >> L;-static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)-{- // compute L = ceil(log2(d));- L = 0;- while (L < 32 && (uint32_t{1} << L) < d) {- L++;- }-- mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);-}--static uint32_t pack_fastdiv_L(uint32_t L0, uint32_t L1, uint32_t L2) {- return L0 | (L1 << 8) | (L2 << 16);-}--template <typename T> void init_pushconst_fastdiv(T &p) {- GGML_UNUSED(p);- static_assert(!std::is_const<T>::value, "unexpected type");-}--template <> void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {- // Compute magic values to divide by these six numbers.- uint32_t ne0_012L;- uint32_t ne0_01L;- uint32_t ne0_0L;- uint32_t ne1_012L;- uint32_t ne1_01L;- uint32_t ne1_0L;-- init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, ne0_012L);- init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, ne0_01L);- init_fastdiv_values(p.ne00, p.ne0_0mp, ne0_0L);- init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, ne1_012L);- init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, ne1_01L);- init_fastdiv_values(p.ne10, p.ne1_0mp, ne1_0L);-- p.ne0_Ls = pack_fastdiv_L(ne0_012L, ne0_01L, ne0_0L);- p.ne1_Ls = pack_fastdiv_L(ne1_012L, ne1_01L, ne1_0L);-}--template <> void init_pushconst_fastdiv(vk_op_glu_push_constants &p) {- // GLU linearizes over dst, then uses dst coordinates for src0/src1.- init_fastdiv_values(p.ne22*p.ne21*p.ne20, p.ne2_012mp, p.ne2_012L);- init_fastdiv_values(p.ne21*p.ne20, p.ne2_01mp, p.ne2_01L);- init_fastdiv_values(p.ne20, p.ne2_0mp, p.ne2_0L);-}--template <> void init_pushconst_fastdiv(vk_op_count_experts_push_constants &p) {- init_fastdiv_values(p.ne00, p.ne00mp, p.ne00L);-}--struct vk_op_binary_push_constants {- uint32_t ne;- uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;- uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23;- uint32_t misalign_offsets;- float param1; float param2; int32_t param3;-};--// Distinct type with the same layout so concat can overload tensor offset initialization.-struct vk_op_concat_push_constants : vk_op_binary_push_constants {};-static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants));-static_assert(std::is_standard_layout_v<vk_op_concat_push_constants>);--struct vk_op_multi_add_push_constants {- // shape for dst- uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23;-- // strides for srcs+dst- uint32_t nb[MAX_PARAMETER_COUNT][4];-- uint32_t rms_partials;-};-// update multi_add.comp if this changes-static_assert(MAX_PARAMETER_COUNT == 12);-static_assert(sizeof(vk_op_multi_add_push_constants) <= 256);--struct vk_op_topk_moe_push_constants {- uint32_t n_rows;- uint32_t n_experts_push;- uint32_t n_expert_used;- float clamp_min;- float clamp_max;- uint32_t gating_func;- uint32_t has_bias;- uint32_t with_norm;- float output_scale;- float output_bias;-};--struct vk_op_add_id_push_constants {- uint32_t ne0;- uint32_t ne1;- uint32_t s01;- uint32_t s02;- uint32_t s11;- uint32_t s21;-};--struct vk_op_diag_mask_push_constants {- uint32_t ncols;- uint32_t rows_per_channel;- int32_t n_past;-};--struct vk_op_rope_push_constants {- uint32_t rope_mode;- uint32_t nrows;- uint32_t n_dims;- uint32_t n_offs;- float freq_scale;- float freq_base;- float ext_factor;- float attn_factor;- float corr_dims[2];- float theta_scale;- uint32_t has_ff;- int32_t sections[4];- uint32_t is_imrope;- uint32_t is_back;- uint32_t set_rows_stride;- uint32_t ne00;- uint32_t ne01;- uint32_t ne02;- uint32_t nb01;- uint32_t nb02;- uint32_t nb03;- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;- uint32_t a_offset;- uint32_t d_offset;-};-static_assert(sizeof(vk_op_rope_push_constants) <= 128, "sizeof(vk_op_rope_push_constants) must be <= 128");--// For fused rms_norm+mul+rope(+view+set_rows)-struct vk_op_rms_norm_mul_rope_push_constants {- vk_op_binary_push_constants bin;- vk_op_rope_push_constants rope;-};--struct vk_op_soft_max_push_constants {- uint32_t KX;- uint32_t KY;- uint32_t ne00;- uint32_t ne01;- uint32_t ne02;- uint32_t ne12;- uint32_t ne13;- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;- float scale;- float max_bias;- float m0;- float m1;- uint32_t n_head_log2;- uint32_t nrows_x;- uint32_t has_sinks;-};--struct vk_op_argsort_push_constants {- uint32_t ncols;- uint32_t ncols_padded;- uint32_t ncols_padded_log2;- uint32_t nrows;- uint32_t order;- uint32_t outer_start;- uint32_t outer_end;- uint32_t inner_start;- uint32_t inner_end;-};--struct vk_op_topk_push_constants {- uint32_t orig_ncols;- uint32_t ncols_input;- uint32_t ncols_output;- uint32_t k;- uint32_t nrows;- uint32_t first_pass;- uint32_t last_pass;-};--struct vk_op_topk_radix_push_constants {- uint32_t ncols;- uint32_t k;- uint32_t nrows;- uint32_t n_tps; // QSA only- uint32_t n_blocks; // QSA only- uint32_t n_stream; // QSA only-};--struct vk_op_im2col_push_constants {- uint64_t dst_addr;- uint32_t batch_offset; uint32_t offset_delta;- uint32_t IC;- uint32_t IW; uint32_t IH;- uint32_t OW; uint32_t OH;- uint32_t KW; uint32_t KH;- uint32_t OH_batch;- uint32_t CHW;- int32_t s0; int32_t s1;- int32_t p0; int32_t p1;- int32_t d0; int32_t d1;- uint32_t batch_IC;-};--struct vk_op_im2col_3d_push_constants {- uint64_t dst_addr;- uint32_t nb10;- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;- uint32_t s0;- uint32_t s1;- uint32_t s2;- uint32_t p0;- uint32_t p1;- uint32_t p2;- uint32_t d0;- uint32_t d1;- uint32_t d2;- uint32_t IW;- uint32_t IH;- uint32_t ID;- uint32_t IC;- uint32_t KW;- uint32_t OH;- uint32_t KD_KH_KW;- uint32_t KH_KW;- uint32_t IC_KD_KH_KW;- uint32_t N_OD_OH;- uint32_t OD_OH;- uint32_t OD_OH_OW_IC_KD_KH_KW;- uint32_t OH_OW_IC_KD_KH_KW;- uint32_t OW_IC_KD_KH_KW;- uint32_t misalign_offsets;-};--struct vk_op_timestep_embedding_push_constants {- uint32_t nb1;- uint32_t dim;- uint32_t max_period;-};--struct vk_op_col2im_1d_push_constants {- uint32_t T_out;- uint32_t OC;- uint32_t K_OC;- uint32_t T_in;- uint32_t K;- int32_t stride;- int32_t p0;-};--struct vk_op_conv_transpose_1d_push_constants {- uint32_t Cout;- uint32_t Cin;- uint32_t K;- uint32_t L;- uint32_t KL;-- uint32_t nb01;- uint32_t nb02;- uint32_t nb11;- uint32_t nb1;-- int32_t s0;-};--struct vk_op_snake_push_constants {- uint32_t ne0;- uint32_t ne1;-};--struct vk_op_pool1d_push_constants {- uint32_t IL;- uint32_t OL;- uint32_t OC;- uint32_t pelements;- uint32_t op;- int32_t k0;- int32_t s0;- int32_t p0;-};--struct vk_op_pool2d_push_constants {- uint32_t IW; uint32_t IH;- uint32_t OW; uint32_t OH;- uint32_t OC;- uint32_t pelements;- uint32_t op;- int32_t k0; int32_t k1;- int32_t s0; int32_t s1;- int32_t p0; int32_t p1;-};--struct vk_op_rwkv_wkv6_push_constants {- uint32_t B;- uint32_t T;- uint32_t C;- uint32_t H;-};--struct vk_op_rwkv_wkv7_push_constants {- uint32_t B;- uint32_t T;- uint32_t C;- uint32_t H;-};-struct vk_op_gated_linear_attn_push_constants {- uint32_t B;- uint32_t T;- uint32_t C;- uint32_t H;- float scale;-};-struct vk_op_lightning_indexer_push_constants {- uint32_t n_kv;- uint32_t n_heads;- uint32_t n_tokens;- uint32_t n_streams;- uint32_t n_masks;- uint32_t dispatch_x;- uint32_t q_nb1;- uint32_t q_nb2;- uint32_t q_nb3;- uint32_t k_nb2;- uint32_t k_nb3;- uint32_t w_nb1;- uint32_t w_nb3;- uint32_t m_nb1;- uint32_t m_nb3;- uint32_t d_nb1;- uint32_t d_nb3;-};-static_assert(sizeof(vk_op_lightning_indexer_push_constants) <= 128);-struct vk_op_gated_delta_net_push_constants {- uint32_t H;- uint32_t n_tokens;- uint32_t n_seqs;- uint32_t s_off;- uint32_t sq1, sq2, sq3;- uint32_t sv1, sv2, sv3;- uint32_t sb1, sb2, sb3;- uint32_t neq1, rq3;- float scale;- uint32_t K;-};--struct vk_op_ssm_scan_push_constants {- uint32_t nb02, nb03, nb12, nb13;- uint32_t nb21, nb22, nb31;- uint32_t nb42, nb43, nb52, nb53;- uint32_t s_off;- uint32_t n_head, d_head, n_group, n_tok;- uint32_t n_seq, K;-};-struct vk_op_ssm_conv_push_constants {- uint32_t nb01, nb02;- uint32_t nb11;- uint32_t dst_nb0, dst_nb1, dst_nb2;- uint32_t nc, ncs, nr, n_t, n_s;-};--struct vk_op_conv2d_push_constants {- uint32_t Cout;- uint32_t Cin;- uint32_t N;-- uint32_t W;- uint32_t H;- uint32_t OW;- uint32_t OH;-- uint32_t nb01;- uint32_t nb02;- uint32_t nb03;-- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;-- uint32_t nb1;- uint32_t nb2;- uint32_t nb3;-- // init_fastdiv_values constants for dividing by OW, OW*OH- uint32_t OWmp; uint32_t OWL;- uint32_t OWOHmp; uint32_t OWOHL;-};--template <> void init_pushconst_fastdiv(vk_op_conv2d_push_constants &p) {- // Compute magic values to divide by OW, OW*OH- init_fastdiv_values(p.OW, p.OWmp, p.OWL);- init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);-}--struct vk_op_conv3d_push_constants {- uint32_t OC;- uint32_t IC;- uint32_t N;-- uint32_t IW;- uint32_t IH;- uint32_t ID;- uint32_t OW;- uint32_t OH;- uint32_t OD;-- uint32_t nb01;- uint32_t nb02;- uint32_t nb03;-- uint32_t nb11;- uint32_t nb12;- uint32_t nb13;-- uint32_t nb1;- uint32_t nb2;- uint32_t nb3;-- uint32_t OWmp; uint32_t OWL;- uint32_t OWOHmp; uint32_t OWOHL;- uint32_t OWOHODmp; uint32_t OWOHODL;-};--template <> void init_pushconst_fastdiv(vk_op_conv3d_push_constants &p) {- init_fastdiv_values(p.OW, p.OWmp, p.OWL);- init_fastdiv_values(p.OW*p.OH, p.OWOHmp, p.OWOHL);- init_fastdiv_values(p.OW*p.OH*p.OD, p.OWOHODmp, p.OWOHODL);-}--struct vk_op_conv2d_dw_push_constants {- uint32_t ne;- uint32_t batches;- uint32_t channels;- uint32_t dst_w;- uint32_t dst_h;- uint32_t src_w;- uint32_t src_h;- uint32_t knl_w;- uint32_t knl_h;- int32_t stride_x;- int32_t stride_y;- int32_t pad_x;- int32_t pad_y;- int32_t dilation_x;- int32_t dilation_y;-};--struct vk_op_upscale_push_constants {- uint32_t ne; uint32_t a_offset; uint32_t d_offset;- uint32_t ne00; uint32_t ne01;- uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;- uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13;- float sf0; float sf1; float sf2; float sf3;- float pixel_offset;-};--struct vk_op_sum_rows_push_constants-{- uint32_t n_cols;- uint32_t ne01, ne02;- uint32_t nb01, nb02, nb03;- uint32_t nb11, nb12, nb13;- float weight;- uint32_t misalign_offsets;- uint32_t ne0_12mp, ne0_12L;- uint32_t ne0_1mp, ne0_1L;-};--static vk_op_sum_rows_push_constants vk_op_sum_rows_push_constants_init(const ggml_tensor * src, const ggml_tensor * dst, int64_t n_cols) {- uint32_t type_size = (uint32_t)ggml_type_size(src->type);- vk_op_sum_rows_push_constants p = {};- p.n_cols = (uint32_t)n_cols;- p.ne01 = (uint32_t)src->ne[1];- p.ne02 = (uint32_t)src->ne[2];- p.nb01 = (uint32_t)src->nb[1] / type_size;- p.nb02 = (uint32_t)src->nb[2] / type_size;- p.nb03 = (uint32_t)src->nb[3] / type_size;- p.nb11 = (uint32_t)dst->nb[1] / type_size;- p.nb12 = (uint32_t)dst->nb[2] / type_size;- p.nb13 = (uint32_t)dst->nb[3] / type_size;- p.weight = 1.0f;- return p;-}--template <> void init_pushconst_fastdiv(vk_op_sum_rows_push_constants &p) {- init_fastdiv_values(p.ne01*p.ne02, p.ne0_12mp, p.ne0_12L);- init_fastdiv_values(p.ne01, p.ne0_1mp, p.ne0_1L);-}--struct vk_quantize_q8_1_push_constants {- uint32_t ne;- uint32_t num_blocks;-};--struct vk_op_flash_attn_split_k_reduce_push_constants {- uint32_t D;- uint32_t ne1;- uint32_t ne2;- uint32_t ne3;- uint32_t k_num;- uint32_t sinks;-};--struct vk_op_flash_attn_mask_opt_push_constants {- uint32_t nem0;- uint32_t nem1;- uint32_t nem2;- uint32_t nbm1;- uint32_t nbm2;- uint32_t nbm3;- uint32_t nbd1;- uint32_t nbd2;- uint32_t nbd3;-};--struct vk_op_flash_attn_sparse_compact_push_constants {- uint32_t KV;- uint32_t nem1;- uint32_t nem2;- uint32_t nbm1;- uint32_t nbm2;- uint32_t nbm3;- uint32_t n_kv_max;-};--// Allow pre-recording command buffers-struct vk_staging_memcpy {- vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {}-- void * dst;- const void * src;- size_t n;-};--struct vk_staging_memset {- vk_staging_memset(void * _dst, uint32_t _val, size_t _n) : dst(_dst), val(_val), n(_n) {}-- void * dst;- uint32_t val;- size_t n;-};--struct vk_context_struct {- vk_submission * s;- std::vector<vk_sequence> seqs;-- int exit_tensor_idx;-- std::vector<vk_staging_memcpy> in_memcpys;- std::vector<vk_staging_memcpy> out_memcpys;- std::vector<vk_staging_memset> memsets;-- std::vector<std::string> debug_labels;-- vk_command_pool * p {};-};-typedef std::shared_ptr<vk_context_struct> vk_context;-typedef std::weak_ptr<vk_context_struct> vk_context_ref;--struct ggml_vk_garbage_collector {- std::vector<vk_semaphore> tl_semaphores;- std::vector<vk_semaphore> semaphores;- std::vector<vk::Event> events;- std::vector<vk_context> contexts;-};--static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx);-static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr);-static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx);-static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor);--static bool vk_memory_logger_enabled = false;--#define VK_LOG_MEMORY(msg) if (vk_memory_logger_enabled) { std::cerr << "ggml_vulkan memory: " << msg << std::endl; }--static std::string format_size(size_t size) {- const size_t kib = 1024;- const size_t mib = kib * 1024;- const size_t gib = mib * 1024;+ }- std::ostringstream oss;- oss << std::fixed << std::setprecision(2);+ if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {+ return;+ }- if (size >= gib) {- oss << static_cast<double>(size) / gib << " GiB";- } else if (size >= mib) {- oss << static_cast<double>(size) / mib << " MiB";- } else if (size >= kib) {- oss << static_cast<double>(size) / kib << " KiB";- } else {- oss << size << " B";+ if (fault_info.description[0] != '\0') {+ GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);}- return oss.str();+ for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {+ const auto& info = address_infos[i];+ GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",+ i, (int)info.addressType,+ (unsigned long long)info.reportedAddress,+ (unsigned long long)info.addressPrecision);+ }+ for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {+ const auto& info = vendor_infos[i];+ GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",+ i, info.description,+ (unsigned long long)info.vendorFaultCode,+ (unsigned long long)info.vendorFaultData);+ }}--class vk_memory_logger {-public:- vk_memory_logger(): total_device(0), total_host(0) {}- void log_allocation(vk_buffer_ref buf_ref, size_t size);- void log_deallocation(vk_buffer_ref buf_ref);--private:- std::map<vk::Buffer, size_t> allocations; // Track allocations- size_t total_device;- size_t total_host;- static std::mutex log_mutex;-};--std::mutex vk_memory_logger::log_mutex;--static bool vk_perf_logger_enabled = false;-static bool vk_perf_logger_concurrent = false;-static bool vk_enable_sync_logger = false;-// number of calls between perf logger prints-static uint32_t vk_perf_logger_frequency = 1;-static std::string vk_pipeline_stats_filter;--static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {+uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {const uint64_t m = node->ne[0];const uint64_t n = node->ne[1];@@ -2338,8 +209,7 @@ static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) {}return 0;}--static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {+void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {uint64_t total_flops = 0;int n_ops = 0;for (int j = start; j <= end && j < cgraph->n_nodes; j++) {@@ -2357,8 +227,7 @@ static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int e}GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);}--static void ggml_vk_print_device_lost_info(const vk_device& device) {+void ggml_vk_print_device_lost_info(const vk_device& device) {ggml_vk_print_device_fault_info(device);if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",@@ -2368,244 +237,16 @@ static void ggml_vk_print_device_lost_info(const vk_device& device) {GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());}}+void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT-class vk_perf_logger {- public:- void print_timings(bool force = false) {- if (timings.empty()) {- return;- }- print_count++;- if ((print_count % vk_perf_logger_frequency) != 0 && !force) {- return;- }- print_count = 0;- uint64_t total_all_op_times = 0;- std::cerr << "----------------\nVulkan Timings:" << std::endl;- for (const auto & t : timings) {- uint64_t total_op_times = 0;- for (const auto & time : t.second) {- total_op_times += time;- }- std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0)- << " us = " << (total_op_times / 1000.0) << " us";-- // If we have as many flops entries as timing entries for the op, then compute and log the flops/S.- auto it = flops.find(t.first);- if (it != flops.end() && (it->second).size() == t.second.size()) {- uint64_t total_op_flops = 0;- for (const auto & elem : it->second) {- total_op_flops += elem;- }- std::cerr << " ("- << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) /- (double(total_op_times) / (1000.0 * 1000.0 * 1000.0))- << " GFLOPS/s)";- }-- total_all_op_times += total_op_times;-- std::cerr << std::endl;- }-- if (timings.size() > 0) {- std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl;- }-- timings.clear();- flops.clear();- }-- std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) {- *n_flops = ggml_vk_get_node_flops(node);- std::string fusion_str;- if (fusion_name) {- fusion_str = fusion_name + std::string(" ");- }- if (node->op == GGML_OP_UNARY) {- return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node));- }- if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {- const uint64_t m = node->ne[0];- const uint64_t n = node->ne[1];- const uint64_t k = node->src[1]->ne[0];- const uint64_t batch = node->ne[2] * node->ne[3];- std::string name = ggml_op_name(node->op);- if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols) ||- (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) {- name += "_VEC";- }- name += " ";- name += ggml_type_name(node->src[0]->type);- name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k);- if (node->op == GGML_OP_MUL_MAT_ID) {- name += " n_expert=" + std::to_string(node->src[0]->ne[2]);- }- if (batch > 1) {- name += " batch=" + std::to_string(batch);- }- return fusion_str + name;- }- if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) {- std::string name = ggml_op_name(node->op);- const ggml_tensor * knl = node->src[0];- uint64_t Cout = node->ne[2];- uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1];- uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1];- name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) +- ", N=N*OW*OH=" + std::to_string(size_N);- return fusion_str + name;- }- if (node->op == GGML_OP_RMS_NORM) {- std::string name = ggml_op_name(node->op);- name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")";- return fusion_str + name;- }- if (node->op == GGML_OP_FLASH_ATTN_EXT) {- const ggml_tensor * dst = node;- const ggml_tensor * q = node->src[0];- const ggml_tensor * k = node->src[1];- const ggml_tensor * v = node->src[2];- const ggml_tensor * m = node->src[3];- std::stringstream name;- name << fusion_str;- name << ggml_op_name(node->op) <<- " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " <<- " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " <<- " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " <<- " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " <<- " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")";- return name.str();- }- if (node->op == GGML_OP_TOP_K) {- std::stringstream name;- name << fusion_str;- name << ggml_op_name(node->op) <<- " K=" << node->ne[0] <<- " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")";- return name.str();- }- return fusion_str + ggml_op_name(node->op);- }-- void log_timing(const ggml_tensor * node, const char *fusion_name, uint64_t time) {- uint64_t n_flops;- std::string name = get_node_fusion_name(node, fusion_name, &n_flops);- if (n_flops) {- flops[name].push_back(n_flops);- }- timings[name].push_back(time);- }-- void log_timing(const std::vector<ggml_tensor *> &nodes, const std::vector<const char *> &names, uint64_t time) {- uint64_t total_flops = 0;- std::string name;- for (size_t n = 0; n < nodes.size(); ++n) {- uint64_t n_flops = 0;- name += get_node_fusion_name(nodes[n], names[n], &n_flops);- total_flops += n_flops;-- if (n != nodes.size() - 1) {- name += ", ";- }- }- if (total_flops) {- flops[name].push_back(total_flops);- }- timings[name].push_back(time);- }-- private:- std::map<std::string, std::vector<uint64_t>> timings;- std::map<std::string, std::vector<uint64_t>> flops;- uint32_t print_count {};-};--struct ggml_backend_vk_context {- std::string name;-- vk_device device;-- size_t semaphore_idx, event_idx;- ggml_vk_garbage_collector gc;- size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset;- vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging;- vk::Fence fence, almost_ready_fence;- bool submit_pending {};- bool almost_ready_fence_pending {};- // Set before op_add and unset after op_rms_norm to indicate that the add should- // write partial sums to accumulate the square of the vector components- bool do_add_rms_partials_offset_calculation;- bool do_add_rms_partials;-- uint64_t last_total_flops {UINT64_MAX};-- // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert.- vk_pipeline_struct * prealloc_y_last_pipeline_used {};- const ggml_tensor * prealloc_y_last_tensor_used {};- // True when the K dimension in prealloc_y is padded.- bool prealloc_y_last_k_padded {};-- // Track which nodes have been used since the last sync, and whether they were written to- std::vector<const ggml_tensor *> unsynced_nodes_written;- std::vector<const ggml_tensor *> unsynced_nodes_read;- // Track which prealloc buffers have pending reads that need to be synchronized.- // These are checked before writing to the buffer (and call ggml_vk_sync_buffers if set),- // and set to true after the buffer contents are consumed.- bool prealloc_x_need_sync, prealloc_y_need_sync, prealloc_split_k_need_sync;-- vk_context_ref compute_ctx;-- vk_context_ref transfer_ctx;- vk_semaphore transfer_semaphore;- uint64_t transfer_semaphore_last_submitted {};-- std::vector<vk_context_ref> tensor_ctxs;-- std::vector<vk::DescriptorPool> descriptor_pools;- std::vector<vk::DescriptorSet> descriptor_sets;- uint32_t descriptor_set_idx {};- uint32_t pipeline_descriptor_set_requirements {};-- vk_command_pool compute_cmd_pool;- vk_command_pool transfer_cmd_pool;-- // number of additional consecutive nodes that are being fused with the- // node currently being processed- int num_additional_fused_ops {};- // Bitmask of which fused ops need to write an intermediate value to memory.- // Bit 'i' means nodes[start_of_fusion + i] writes to memory.- // If there's no fusion, bit 0 is still set.- int fused_ops_write_mask {};- topk_moe_mode fused_topk_moe_mode {};- bool fused_topk_moe_scale {};- // QSA indexer gather+add+top_k fused into one radix-select- bool fused_topk_qsa {};- rms_norm_mode fused_rms_norm_mode {RMS_NORM_COUNT};-- // for GGML_VK_PERF_LOGGER- std::unique_ptr<vk_perf_logger> perf_logger;- vk::QueryPool query_pool;- std::vector<const char *> query_fusion_names;- std::vector<int> query_fusion_node_count;- std::vector<ggml_tensor *> query_nodes;- std::vector<int> query_node_idx;- int32_t num_queries {};- int32_t query_idx {};-};--static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT--static uint64_t vk_tensor_offset(const ggml_tensor * tensor) {+uint64_t vk_tensor_offset(const ggml_tensor * tensor) {if (tensor->view_src) {return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;}return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base;}-static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset);--static size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {+size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {// vk_tensor_offset() is relative to vk_ptr_base, but mapped host tensors need an offset relative to their Vulkan buffer.if (ctx->device->uma) {vk_buffer buf = nullptr;@@ -2617,8 +258,7 @@ static size_t ggml_vk_tensor_buffer_offset(const ggml_backend_vk_context * ctx,}return (size_t)(vk_tensor_offset(t) + t->view_offs);}--static size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) {+size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment, size_t type_size) {// Move the descriptor back until its distance to the tensor is divisible by the tensor type size.size_t descriptor_offset = tensor_offset & ~(alignment - 1);while ((tensor_offset - descriptor_offset) % type_size != 0) {@@ -2628,8 +268,7 @@ static size_t ggml_vk_descriptor_offset(size_t tensor_offset, size_t alignment,return descriptor_offset;}--static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {+uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t) {const size_t tensor_offset = ggml_vk_tensor_buffer_offset(ctx, t);const size_t descriptor_offset = ggml_vk_descriptor_offset(tensor_offset, ctx->device->properties.limits.minStorageBufferOffsetAlignment, ggml_type_size(t->type));@@ -2637,7 +276,7 @@ static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggreturn tensor_offset - descriptor_offset;}-static uint32_t ggml_vk_concat_unit_size(ggml_type type) {+uint32_t ggml_vk_concat_unit_size(ggml_type type) {const uint32_t type_size = ggml_type_size(type);if (!ggml_is_quantized(type)) {@@ -2656,8 +295,7 @@ static uint32_t ggml_vk_concat_unit_size(ggml_type type) {}return 1;}--static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {+bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) {if (src0->type != src1->type || src0->type != dst->type) {return false;}@@ -2670,162 +308,9 @@ static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor// Quantized tensor rows are block-aligned when created.return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst);}--template <typename T> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- GGML_UNUSED(p);- GGML_UNUSED(src0);- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);- GGML_UNUSED(dst);- static_assert(!std::is_const<T>::value, "unexpected type");- GGML_ASSERT(!src0 || get_misalign_bytes(ctx, src0) == 0);- GGML_ASSERT(!src1 || get_misalign_bytes(ctx, src1) == 0);- GGML_ASSERT(!src2 || get_misalign_bytes(ctx, src2) == 0);- GGML_ASSERT(!src3 || get_misalign_bytes(ctx, src3) == 0);- GGML_ASSERT(!dst || get_misalign_bytes(ctx, dst) == 0);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_p021_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.b_offset = b_offset;- p.d_offset = d_offset;-- GGML_UNUSED(src0);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_mat_vec_nc_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.b_offset = b_offset;- p.d_offset = d_offset;-- GGML_UNUSED(src0);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_fwht_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- p.src_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- p.dst_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_comb_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- p.m_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- p.s_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- p.b_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_pre_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- p.w_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_dsv4_hc_post_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- p.x_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- p.r_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- p.p_offset = get_misalign_bytes(ctx, src2) / ggml_type_size(src2->type);- p.c_offset = src3 ? get_misalign_bytes(ctx, src3) / ggml_type_size(src3->type) : 0;- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-}--struct ggml_backend_vk_buffer_context {- vk_device_ref device;- vk_buffer dev_buffer;- std::string name;-- ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) :- device(device),- dev_buffer(dev_buffer),- name(name) {- }-- ~ggml_backend_vk_buffer_context() {- ggml_vk_destroy_buffer(dev_buffer);- }-};--void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {- if (!vk_memory_logger_enabled) {- return;- }- std::lock_guard<std::mutex> guard(log_mutex);- vk_buffer buf = buf_ref.lock();- const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);- const std::string type = device ? "device" : "host";- allocations[buf->buffer] = size;- total_device += device ? size : 0;- total_host += device ? 0 : size;- VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));-}--void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) {- if (buf_ref.expired() || buf_ref.lock()->size == 0 || !vk_memory_logger_enabled) {- return;- }-- std::lock_guard<std::mutex> guard(log_mutex);- vk_buffer buf = buf_ref.lock();- const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);- std::string type = device ? "device" : "host";- auto it = allocations.find(buf->buffer);- if (it != allocations.end()) {- total_device -= device ? it->second : 0;- total_host -= device ? 0 : it->second;- VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));- allocations.erase(it);- } else {- VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer);- }-}--struct vk_instance_t {- vk::Instance instance;-- bool debug_utils_support = false; // VK_EXT_debug_utils enabled- PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = {};- PFN_vkQueueBeginDebugUtilsLabelEXT pfn_vkQueueBeginDebugUtilsLabelEXT = {};- PFN_vkQueueEndDebugUtilsLabelEXT pfn_vkQueueEndDebugUtilsLabelEXT = {};- PFN_vkCmdBeginDebugUtilsLabelEXT pfn_vkCmdBeginDebugUtilsLabelEXT = {};- PFN_vkCmdEndDebugUtilsLabelEXT pfn_vkCmdEndDebugUtilsLabelEXT = {};- PFN_vkCmdInsertDebugUtilsLabelEXT pfn_vkCmdInsertDebugUtilsLabelEXT = {};-- std::vector<size_t> device_indices;- std::vector<bool> device_supports_membudget;- vk_device devices[GGML_VK_MAX_DEVICES];-};-static bool vk_instance_initialized = false;-static vk_instance_t vk_instance;-#ifdef GGML_VULKAN_CHECK_RESULTS-static size_t vk_skip_checks;-static size_t vk_output_tensor;--static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name);-static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);-static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx);-#endif--typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);--static void ggml_backend_vk_free(ggml_backend_t backend);+vk_instance_t vk_instance;static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},@@ -2833,8 +318,7 @@ static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context *return range;}-// Wait for ctx->fence to be signaled.-static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {+void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {// Use waitForFences while most of the graph executes. Hopefully the CPU can sleep// during this wait.if (ctx->almost_ready_fence_pending) {@@ -2876,16 +360,6 @@ static void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx) {ctx->device->device.resetFences({ ctx->fence });}-static constexpr uint32_t kSpvOpCooperativeMatrixLoadTensorNV = 5367;-static constexpr uint32_t kSpvCapabilityCooperativeMatrixDecodeVectorNV = 5447;-static constexpr uint32_t kSpvTensorAddressingDecodeVectorFuncBit = 0x4;--// Remove SPV_NV_cooperative_matrix_decode_vector usage from a SPIR-V module so it-// can be loaded on drivers that only support SPV_NV_cooperative_matrix2. Drops the-// OpExtension declaration, the CooperativeMatrixDecodeVectorNV OpCapability, and the-// DecodeVectorFunc operand from any OpCooperativeMatrixLoadTensorNV instruction.-// Returns true when the input used the extension (and `out` was populated with a-// stripped copy); returns false otherwise without touching `out`.static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) {static const char kDecodeVectorExt[] = "SPV_NV_cooperative_matrix_decode_vector";@@ -3006,14 +480,6 @@ static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_countreturn true;}-// Remove the loop unrolling hint of the matmul shader's BK loop-// and replace it with the dont_unroll hint for better performance on-// hardware like Apple M1/M2.-// Assumes 1. code comes from mul_mm.comp 2. the K-tile loop has no loop-// control hint and 3. the BK loop is the last loop nested directly inside-// the K-tile loop.-// Returns true when the input was modified; returns false otherwise-// without touching `out`.static bool ggml_vk_roll_bk_loop(const uint32_t * code, size_t word_count, std::vector<uint32_t> & out) {if (word_count < 5) {return false;@@ -3338,7 +804,7 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelindevice->compile_cv.notify_all();}-static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {+void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {VK_LOG_DEBUG("ggml_pipeline_destroy_pipeline(" << pipeline->name << ")");device.destroyPipelineLayout(pipeline->layout);@@ -3347,7 +813,7 @@ static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline)device.destroyPipeline(pipeline->pipeline);}-static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) {+void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n) {VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")");ctx->pipeline_descriptor_set_requirements += n;if (!pipeline->compiled) {@@ -3356,7 +822,7 @@ static void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx,ggml_pipeline_allocate_descriptor_sets(ctx);}-static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) {+void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx) {if (ctx->descriptor_sets.size() >= ctx->pipeline_descriptor_set_requirements) {// Enough descriptors are available@@ -3405,7 +871,7 @@ static vk_command_buffer* ggml_vk_create_cmd_buffer(vk_device& device, vk_commanreturn &p.cmd_buffers[p.cmd_buffers.size()-1];}-static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {+void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {if (ctx->seqs.empty()) {if (fence) {ctx->p->q->handle->submit({}, fence);@@ -3482,7 +948,7 @@ static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {ctx->seqs.clear();}-static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) {+uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) {VK_LOG_DEBUG("ggml_vk_find_queue_family_index()");const uint32_t qfsize = queue_family_props.size();@@ -3528,7 +994,7 @@ static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyPropeabort();}-static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) {+std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) {VK_LOG_DEBUG("ggml_vk_create_queue()");std::lock_guard<std::recursive_mutex> guard(device->mutex);@@ -3562,7 +1028,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_queue(vk_device& device, uint32_return q;}-static std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) {+std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device, const std::unique_ptr<vk_queue>& source) {std::lock_guard<std::recursive_mutex> guard(device->mutex);auto q = std::make_unique<vk_queue>();q->handle = source->handle;@@ -3573,7 +1039,7 @@ static std::unique_ptr<vk_queue> ggml_vk_create_aliased_queue(vk_device& device,return q;}-static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) {+vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_command_pool& p) {vk_context result = std::make_shared<vk_context_struct>();VK_LOG_DEBUG("ggml_vk_create_context(" << result << ")");ctx->gc.contexts.emplace_back(result);@@ -3581,7 +1047,7 @@ static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_commareturn result;}-static vk_context ggml_vk_create_temporary_context(vk_command_pool& p) {+vk_context ggml_vk_create_temporary_context(vk_command_pool& p) {vk_context result = std::make_shared<vk_context_struct>();VK_LOG_DEBUG("ggml_vk_create_temporary_context(" << result << ")");result->p = &p;@@ -3617,7 +1083,7 @@ static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) {return ctx->gc.events[ctx->event_idx++];}-static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) {+void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) {VK_LOG_DEBUG("ggml_vk_command_pool_cleanup()");// Requires command buffers to be done@@ -3629,7 +1095,7 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p)}}-static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {+void ggml_vk_queue_command_pools_cleanup(vk_device& device) {VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()");// Arbitrary frequency to cleanup/reuse command buffers@@ -3643,243 +1109,11 @@ static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {}}-static std::vector<uint32_t> ggml_vk_find_memory_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {- std::vector<uint32_t> indices;-- for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {- vk::MemoryType memory_type = mem_props->memoryTypes[i];- if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&- (flags & memory_type.propertyFlags) == flags &&- mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {- indices.push_back(i);- }- }- return indices;-}--static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std::initializer_list<vk::MemoryPropertyFlags> & req_flags_list,- void *import_ptr = nullptr) {- VK_LOG_DEBUG("ggml_vk_create_buffer(" << device->name << ", " << size << ", " << to_string(req_flags_list.begin()[0]) << ", " << to_string(req_flags_list.begin()[req_flags_list.size()-1]) << ")");- if (size > device->max_buffer_size) {- throw vk::OutOfDeviceMemoryError("Requested buffer size exceeds device buffer size limit");- }-- vk_buffer buf = std::make_shared<vk_buffer_struct>();-- if (size == 0) {- buf->size = 0;- return buf;- }-- vk::BufferUsageFlags usage_flags = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst;- vk::MemoryAllocateFlags mem_flags {};- if (device->buffer_device_address) {- usage_flags |= vk::BufferUsageFlagBits::eShaderDeviceAddress;- mem_flags |= vk::MemoryAllocateFlagBits::eDeviceAddress;- }-- vk::BufferCreateInfo buffer_create_info{- vk::BufferCreateFlags(),- size,- usage_flags,- vk::SharingMode::eExclusive,- 0,- nullptr,- };-- vk::ExternalMemoryBufferCreateInfo external_memory_bci;- if (import_ptr) {- external_memory_bci.handleTypes = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;- buffer_create_info.setPNext(&external_memory_bci);- }-- buf->buffer = device->device.createBuffer(buffer_create_info);-- vk::MemoryRequirements mem_req = device->device.getBufferMemoryRequirements(buf->buffer);-- vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();-- const vk::MemoryPriorityAllocateInfoEXT mem_priority_info { 1.0f };-- vk::MemoryAllocateFlagsInfo mem_flags_info { mem_flags };-- if (device->memory_priority) {- mem_flags_info.setPNext(&mem_priority_info);- }-- if (import_ptr) {- vk::MemoryHostPointerPropertiesEXT host_pointer_props;- try {- host_pointer_props = device->device.getMemoryHostPointerPropertiesEXT(vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT, import_ptr);- } catch (vk::SystemError& e) {- GGML_LOG_WARN("ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (%s)\n", e.what());- device->device.destroyBuffer(buf->buffer);- return {};- }- vk::PhysicalDeviceMemoryProperties mem_props = device->physical_device.getMemoryProperties();-- uint32_t memory_type_idx;- vk::MemoryPropertyFlags property_flags = *req_flags_list.begin();- for (memory_type_idx = 0; memory_type_idx < 32; ++memory_type_idx) {- if (!(host_pointer_props.memoryTypeBits & (1u << memory_type_idx))) {- continue;- }- if (!(mem_req.memoryTypeBits & (1u << memory_type_idx))) {- continue;- }-- vk::MemoryType memory_type = mem_props.memoryTypes[memory_type_idx];- // check for visible+coherent+cached. Other flags (e.g. devicelocal) are allowed- if ((memory_type.propertyFlags & property_flags) == property_flags) {- property_flags = memory_type.propertyFlags;- break;- }- }- if (memory_type_idx == 32) {- GGML_LOG_WARN("ggml_vulkan: Memory type for host allocation not found\n");- device->device.destroyBuffer(buf->buffer);- return {};- }-- buf->memory_property_flags = mem_props.memoryTypes[memory_type_idx].propertyFlags;- try {- vk::ImportMemoryHostPointerInfoEXT import_info;- import_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eHostAllocationEXT;- import_info.pHostPointer = import_ptr;- import_info.setPNext(&mem_flags_info);- buf->device_memory = device->device.allocateMemory({ size, memory_type_idx, &import_info });- } catch (const vk::SystemError& e) {- }- } else {- for (auto it = req_flags_list.begin(); it != req_flags_list.end(); it++) {- const auto & req_flags = *it;-- const std::vector<uint32_t> memory_type_indices = ggml_vk_find_memory_properties(&mem_props, &mem_req, req_flags);-- if (memory_type_indices.empty()) {- continue;- }-- bool done = false;-- for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) {- try {- buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info });- buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags;- done = true;- break;- } catch (const vk::SystemError& e) {- // loop and retry- // during last attempt throw the exception- if (it + 1 == req_flags_list.end() && mtype_it + 1 == memory_type_indices.end()) {- device->device.destroyBuffer(buf->buffer);- throw e;- }- }- }-- if (done) {- break;- }- }- }-- if (!buf->device_memory) {- device->device.destroyBuffer(buf->buffer);- throw vk::OutOfDeviceMemoryError("No suitable memory type found");- }-- buf->ptr = nullptr;-- if (import_ptr) {- buf->ptr = import_ptr;- } else {- if (buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {- buf->ptr = device->device.mapMemory(buf->device_memory, 0, VK_WHOLE_SIZE);- }- }-- device->device.bindBufferMemory(buf->buffer, buf->device_memory, 0);-- buf->device = device;- buf->size = size;-- if (device->buffer_device_address) {- const vk::BufferDeviceAddressInfo addressInfo(buf->buffer);- buf->bda_addr = device->device.getBufferAddress(addressInfo);- }-- device->memory_logger->log_allocation(buf, size);-- return buf;-}--static vk_buffer ggml_vk_create_buffer_check(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) {- try {- return ggml_vk_create_buffer(device, size, {req_flags, fallback_flags});- } catch (const vk::SystemError& e) {- std::cerr << "ggml_vulkan: Memory allocation of size " << size << " failed." << std::endl;- std::cerr << "ggml_vulkan: " << e.what() << std::endl;- throw e;- }-}--static vk_buffer ggml_vk_create_buffer_device(vk_device& device, size_t size) {- vk_buffer buf;- try {- if (device->prefer_host_memory) {- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,- vk::MemoryPropertyFlagBits::eDeviceLocal});- } else if (device->uma) {- // On UMA, prefer host-visible memory so direct tensor borrowing works.- // If unavailable, fall back to device-local memory.- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,- vk::MemoryPropertyFlagBits::eDeviceLocal,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});- } else if (device->disable_host_visible_vidmem) {- if (device->allow_sysmem_fallback) {- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});- } else {- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal});- }- } else {- // use rebar if available, otherwise fallback to device only visible memory- if (device->allow_sysmem_fallback) {- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,- vk::MemoryPropertyFlagBits::eDeviceLocal,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});- } else {- buf = ggml_vk_create_buffer(device, size, {vk::MemoryPropertyFlagBits::eDeviceLocal | vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent,- vk::MemoryPropertyFlagBits::eDeviceLocal});- }- }- } catch (const vk::SystemError& e) {- std::cerr << "ggml_vulkan: Device memory allocation of size " << size << " failed." << std::endl;- std::cerr << "ggml_vulkan: " << e.what() << std::endl;- throw e;- }-- return buf;-}--static void ggml_vk_destroy_buffer(vk_buffer& buf) {- if (buf == nullptr) {- return;- }-- if (buf->device != nullptr) {- buf->device->memory_logger->log_deallocation(buf);- }-- buf.reset();-}--static vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset = 0) {+vk_subbuffer ggml_vk_subbuffer(const ggml_backend_vk_context* ctx, const vk_buffer& buf, size_t offset) {return { buf, offset, ggml_vk_get_max_buffer_range(ctx, buf, offset) };}-static void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) {+void ggml_vk_sync_buffers(ggml_backend_vk_context* ctx, vk_context& subctx) {VK_LOG_DEBUG("ggml_vk_sync_buffers()");const bool transfer_queue = subctx->p->q->transfer_only;@@ -3910,7 +1144,7 @@ static void ggml_vk_reset_event(vk_context& ctx, vk::Event& event) {);}-static void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {+void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {VK_LOG_DEBUG("ggml_vk_set_event()");ctx->s->buffer->buf.setEvent(@@ -3919,7 +1153,7 @@ static void ggml_vk_set_event(vk_context& ctx, vk::Event& event) {);}-static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) {+void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events) {VK_LOG_DEBUG("ggml_vk_wait_events()");if (events.empty()) {return;@@ -3935,29 +1169,6 @@ static void ggml_vk_wait_events(vk_context& ctx, std::vector<vk::Event>&& events);}-struct vk_fa_tuning_params {- FaCodePath path;- uint32_t workgroup_size;- uint32_t subgroup_size;- uint32_t block_rows;- uint32_t block_cols;- uint32_t d_split;- uint32_t row_split;- bool shmem_staging;- bool disable_subgroups;- uint32_t limit_occupancy_shmem;-- void print() const {- std::cerr << "path=" << path << " workgroup_size=" << workgroup_size << " subgroup_size=" << subgroup_size <<- " block_rows=" << block_rows << " block_cols=" << block_cols << " d_split=" << d_split <<- " row_split=" << row_split << " shmem_staging=" << shmem_staging << " disable_subgroups=" << disable_subgroups <<- " limit_occupancy_shmem=" << limit_occupancy_shmem << std::endl;- }-};--static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type);-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16);-static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {vk_fa_tuning_params result{};@@ -4092,7 +1303,7 @@ static vk_fa_tuning_params get_fa_tuning_params_coopmat2(const vk_device& devicereturn result;}-static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {+vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) {FaCodePath path = device->coopmat2 ? FA_COOPMAT2 :device->coopmat1_fa_support ? FA_COOPMAT1 : FA_SCALAR;@@ -4136,7 +1347,7 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_}}-static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc,+vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool aligned, bool f32acc,bool use_mask, bool use_mask_opt, bool use_logit_softcap, bool use_sparse, ggml_type k_type, ggml_type v_type) {const bool old_amd_windows = device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eAmdProprietary &&(device->architecture == AMD_GCN || device->architecture == AMD_RDNA1 || device->architecture == AMD_RDNA2);@@ -4152,8 +1363,6 @@ static vk_fa_pipeline_state get_fa_pipeline_state(const vk_device& device, constreturn vk_fa_pipeline_state{hsk, hsv, params.block_rows, params.block_cols, params.d_split, params.row_split, params.shmem_staging, params.path, params.workgroup_size, subgroup_size, aligned, f32acc, flags, params.limit_occupancy_shmem, k_type, v_type};}-// Bytes per buffer block for the FaBlockBytesK/V spec constants. F32 is fed as-// a vec4 "block" of 4 floats, everything else uses its ggml block size.static uint32_t fa_block_bytes(ggml_type t) {if (t == GGML_TYPE_F32) {return 16u;@@ -4242,10 +1451,6 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vecreturn supported;}-// Shmem usage for the q8_1 mmq shader (mul_mmq.comp), which uses-// block_a_cache / block_b_cache layouts (see mul_mmq_shmem_types.glsl) rather-// than the float load buffers checked by ggml_vk_matmul_shmem_support.-// Sizes follow std430 rules. Returns false for types without a q8_1 pipeline.static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {// FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.@@ -4307,35 +1512,16 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:return supported;}-struct GpuPipelineConfig {- // GPU architecture identifier.- // Example: vk_device_architecture::AMD_GCN- vk_device_architecture arch;-- // Mapping of pipeline names to their specific subgroup sizes.- // Example: {"soft_max_f32", 64}- std::unordered_map<std::string, uint32_t> pipelines;-- // Default subgroup size for this GPU.- // Defaults to 0 if not explicitly provided.- uint32_t default_subgroup_size = 0;-};--// Pipeline configuration for RDNA1 GPUs.static const std::unordered_map<std::string, uint32_t> rdna1_pipelines = {{"soft_max", 64}, {"im2col", 64},{"argmax", 64}, {"mul_mat_vec", 64},{"mul_mat_vec_f16", 32}, {"mul_mat_vec_f32_f16", 32}};-// Pipeline configuration for RDNA2 GPUs.static const std::unordered_map<std::string, uint32_t> rdna2_pipelines = {{"soft_max", 64}, {"im2col", 64},};-static constexpr uint32_t RDNA_DEFAULT_SUBGROUP_SIZE = 32;--// Define configurations for different GPUs.static std::vector<GpuPipelineConfig> gpu_pipeline_configs = {{vk_device_architecture::AMD_RDNA1,@@ -4353,7 +1539,7 @@ static std::vector<GpuPipelineConfig> gpu_pipeline_configs = {},};-static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) {+uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_device_architecture &arch) {for (const auto &config : gpu_pipeline_configs) {if (config.arch == arch) {auto pipIt = config.pipelines.find(pipeline_name);@@ -4374,7 +1560,6 @@ static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_devreturn 0; // If no matching configuration is found}-// Whether scalar flash attention will use the MMQ path for the given K/V types.static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {switch (type) {case GGML_TYPE_IQ4_NL:@@ -4399,24 +1584,7 @@ static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type#endif}-// load_shaders walks the pipeline list under compile_mutex and either claims-// the requested pipeline for compilation or, if another thread is already-// compiling it, drops the lock and waits on compile_cv. Compiles themselves-// run unlocked.-struct CompileTask {- vk_pipeline pipeline;- size_t spv_size;- const void * spv_data;- std::string entrypoint;- uint32_t parameter_count;- std::array<uint32_t, 3> wg_denoms;- std::vector<uint32_t> specialization_constants;- bool disable_robustness;- bool require_full_subgroups;- uint32_t required_subgroup_size;-};--static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {+void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {VK_LOG_DEBUG("ggml_vk_load_shaders(" << device->name << ")");// some shaders have a minimum subgroup size@@ -6560,10 +3728,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {}}-static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch);-static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev);--static vk_device ggml_vk_get_device(size_t idx) {+vk_device ggml_vk_get_device(size_t idx) {VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");if (vk_instance.devices[idx] == nullptr) {@@ -7738,17 +4903,13 @@ static void ggml_vk_print_gpu_info(size_t idx) {}}-static bool ggml_vk_instance_layer_settings_available();-static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions);-static bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions);-static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev);-static DispatchLoaderDynamic ggml_vk_default_dispatcher_instance;+DispatchLoaderDynamic & ggml_vk_default_dispatcher() {return ggml_vk_default_dispatcher_instance;}-static void ggml_vk_instance_init() {+void ggml_vk_instance_init() {if (vk_instance_initialized) {return;}@@ -7998,7 +5159,7 @@ static void ggml_vk_instance_init() {}}-static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {+void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {VK_LOG_DEBUG("ggml_vk_init(" << ctx->name << ", " << idx << ")");ggml_vk_instance_init();GGML_ASSERT(idx < vk_instance.device_indices.size());@@ -8042,7 +5203,7 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {#endif}-static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) {+vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type) {VK_LOG_DEBUG("ggml_vk_get_to_fp16()");switch (type) {case GGML_TYPE_F32:@@ -8248,77 +5409,14 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_contextif (ctx->device->vendor_id == VK_VENDOR_ID_INTEL) {dmmv_wg = DMMV_WG_SIZE_SUBGROUP;}- return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type];- }-- return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type];-}--static void * ggml_vk_host_malloc(vk_device& device, size_t size) {- VK_LOG_MEMORY("ggml_vk_host_malloc(" << size << ")");- vk_buffer buf = ggml_vk_create_buffer(device, size,- {vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent});-- if(!(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible)) {- fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory\n",- size/1024.0/1024.0);- device->device.freeMemory(buf->device_memory);- device->device.destroyBuffer(buf->buffer);- return nullptr;- }-- std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);- device->pinned_memory.push_back(std::make_tuple(buf->ptr, size, buf));-- return buf->ptr;-}--static void ggml_vk_host_free(vk_device& device, void* ptr) {- if (ptr == nullptr) {- return;- }- VK_LOG_MEMORY("ggml_vk_host_free(" << ptr << ")");- std::lock_guard<std::shared_mutex> guard(device->pinned_memory_mutex);-- vk_buffer buf;- size_t index;- for (size_t i = 0; i < device->pinned_memory.size(); i++) {- const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);- const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);- if (ptr >= addr && ptr < endr) {- buf = std::get<2>(device->pinned_memory[i]);- index = i;- break;- }- }- if (buf == nullptr) {- fprintf(stderr, "WARNING: failed to free pinned memory: memory not in map\n");- return;- }-- ggml_vk_destroy_buffer(buf);-- device->pinned_memory.erase(device->pinned_memory.begin() + index);-}--static void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, size_t& buf_offset) {- std::shared_lock<std::shared_mutex> guard(device->pinned_memory_mutex);- buf = nullptr;- buf_offset = 0;- for (size_t i = 0; i < device->pinned_memory.size(); i++) {- const uint8_t* addr = (const uint8_t*) std::get<0>(device->pinned_memory[i]);- const uint8_t* endr = addr + std::get<1>(device->pinned_memory[i]);- if (ptr >= addr && ptr < endr) {- buf = std::get<2>(device->pinned_memory[i]);- buf_offset = ((const uint8_t *)ptr) - addr;- break;- }+ return ctx->device->pipeline_dequant_mul_mat_vec_id_q8_1_f32[dmmv_wg][a_type];}++ return ctx->device->pipeline_dequant_mul_mat_vec_id_f32[dmmv_wg][a_type];}-static vk_subbuffer ggml_vk_tensor_subbuffer(- const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign = false) {+vk_subbuffer ggml_vk_tensor_subbuffer(+ const ggml_backend_vk_context * ctx, const ggml_tensor * tensor, bool allow_misalign) {vk_buffer buffer = nullptr;size_t offset = 0;@@ -8345,7 +5443,6 @@ static vk_subbuffer ggml_vk_tensor_subbuffer(return vk_subbuffer{buffer, offset, size};}-// Get a command buffer from pool. Create a new one if no reusable buffer is availablestatic vk_command_buffer* ggml_vk_get_or_create_cmd_buffer(vk_device& device, vk_command_pool& pool) {for (auto& cmd_buffer : pool.cmd_buffers) {if (!cmd_buffer.in_use) {@@ -8369,156 +5466,14 @@ static vk_submission ggml_vk_begin_submission(vk_device& device, vk_command_poolreturn s;}-template <typename T> size_t push_constant_size(const T &t) {- static_assert(std::is_class<T>::value, "T must be a struct/class");- GGML_UNUSED(t);- return sizeof(T);-}-template <typename T> size_t push_constant_size(const std::vector<T> &t) {- GGML_UNUSED(t);- return sizeof(T) * t.size();-}-template <typename T, uint32_t N> size_t push_constant_size(const std::array<T, N> &t) {- GGML_UNUSED(t);- return sizeof(T) * N;-}--template <typename T> const T *push_constant_data(const T &t) {- static_assert(std::is_class<T>::value, "T must be a struct/class");- return &t;-}-template <typename T> const T *push_constant_data(const std::vector<T> &t) {- return t.data();-}-template <typename T, uint32_t N> const T *push_constant_data(const std::array<T, N> &t) {- return t.data();-}--static void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) {+void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name) {vk::DebugUtilsLabelEXT label = {};label.pLabelName = name;label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};vk_instance.pfn_vkCmdBeginDebugUtilsLabelEXT(buf, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));}-// no-op unless GGML_VK_DEBUG_MARKERS is set-struct ggml_vk_debug_label {- // at most one of these is set, depending on the scope the label was opened in- vk_context_struct * subctx {};- vk_queue_handle * qhandle {};-- // one region per dispatch, e.g. "matmul_q4_k_f32_f16acc_aligned_m (192,8,1)".- // RGP cannot recover the pipeline name on its own, it only has the hash- ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) {- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {- return;- }- begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")");- }-- // one region per graph node- // fused nodes are joined with '+', e.g. "RMS_NORM+MUL+ROPE Qcur-19"- ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) {- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {- return;- }- std::string name = ggml_op_name(cgraph->nodes[node_idx]->op);- for (int i = 1; i <= n_fused; i++) {- name += "+";- name += ggml_op_name(cgraph->nodes[node_idx + i]->op);- }- name += " ";- name += cgraph->nodes[node_idx]->name;- begin(ctx, name);- }-- // one region per graph evaluation, opened on the queue instead of a command buffer- // so it spans every submit the evaluation makes- ggml_vk_debug_label(vk_queue_handle * handle, const char * name) {- if (!vk_instance.debug_utils_support || handle == nullptr) {- return;- }- vk::DebugUtilsLabelEXT label = {};- label.pLabelName = name;- label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};-- qhandle = handle;- std::lock_guard<vk_queue_handle> guard(*qhandle);- vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));- }-- // call before the command buffer can end, the destructor covers the rest- void close() {- if (subctx != nullptr) {- // close on the current command buffer, which may differ from the one begin used- if (subctx->s != nullptr) {- vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf);- }- subctx->debug_labels.pop_back();- subctx = nullptr;- }- if (qhandle != nullptr) {- std::lock_guard<vk_queue_handle> guard(*qhandle);- vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue);- qhandle = nullptr;- }- }-- ~ggml_vk_debug_label() {- close();- }-- ggml_vk_debug_label(const ggml_vk_debug_label &) = delete;- ggml_vk_debug_label & operator=(const ggml_vk_debug_label &) = delete;--private:- // the constructors check this too, so the name is not built when markers are off- void begin(vk_context & ctx, const std::string & name) {- if (!vk_instance.debug_utils_support || ctx->s == nullptr) {- return;- }- subctx = ctx.get();- subctx->debug_labels.push_back(name);- ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str());- }-};--template <typename T>-static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {- const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);- const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);- const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);- VK_LOG_DEBUG("ggml_vk_dispatch_pipeline(" << pipeline->name << ", {";- for (auto& buffer : descriptor_buffer_infos) {- std::cerr << "(" << buffer.buffer << ", " << buffer.offset << ", " << buffer.range << "), ";- }- std::cerr << "}, (" << wg0 << "," << wg1 << "," << wg2 << "))");- GGML_ASSERT(wg0 <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] &&- wg1 <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&- wg2 <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]);- GGML_ASSERT(ctx->descriptor_set_idx < ctx->descriptor_sets.size());- GGML_ASSERT(descriptor_buffer_infos.size() <= MAX_PARAMETER_COUNT);- GGML_ASSERT(pipeline->parameter_count == descriptor_buffer_infos.size());- GGML_ASSERT(pipeline->push_constant_size == push_constant_size(push_constants));-- vk::DescriptorSet& descriptor_set = ctx->descriptor_sets[ctx->descriptor_set_idx++];- vk::WriteDescriptorSet write_descriptor_set{ descriptor_set, 0, 0, pipeline->parameter_count, vk::DescriptorType::eStorageBuffer, nullptr, descriptor_buffer_infos.begin() };- ctx->device->device.updateDescriptorSets({ write_descriptor_set }, {});-- subctx->s->buffer->buf.pushConstants(pipeline->layout, vk::ShaderStageFlagBits::eCompute, 0, push_constant_size(push_constants), push_constant_data(push_constants));- subctx->s->buffer->buf.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline->pipeline);- subctx->s->buffer->buf.bindDescriptorSets(vk::PipelineBindPoint::eCompute,- pipeline->layout,- 0,- { descriptor_set },- {});- {- ggml_vk_debug_label dbg(subctx, pipeline->name, wg0, wg1, wg2);- subctx->s->buffer->buf.dispatch(wg0, wg1, wg2);- }-}--static void ggml_vk_ctx_end(vk_context& ctx) {+void ggml_vk_ctx_end(vk_context& ctx) {VK_LOG_DEBUG("ggml_vk_ctx_end(" << ctx << ", " << ctx->seqs.size() << ")");if (ctx->s == nullptr) {return;@@ -8537,7 +5492,7 @@ static void ggml_vk_ctx_end(vk_context& ctx) {ctx->s = nullptr;}-static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {+void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {VK_LOG_DEBUG("ggml_vk_ctx_begin(" << device->name << ")");if (subctx->s != nullptr) {ggml_vk_ctx_end(subctx);@@ -8558,7 +5513,7 @@ static void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx) {}}-static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {+vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {vk_context result;if (!ctx->compute_ctx.expired()) {result = ctx->compute_ctx.lock();@@ -8577,7 +5532,7 @@ static vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx) {return result;}-static vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {+vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {vk_context result;if (!ctx->transfer_ctx.expired()) {result = ctx->transfer_ctx.lock();@@ -8591,10 +5546,7 @@ static vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx) {return result;}-// Submit any pending transfer queue work and signal the transfer semaphore.-// The next compute context created via ggml_vk_get_compute_ctx will wait on this semaphore.-// Returns true if work was submitted.-static bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {+bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {if (!ctx->device->async_use_transfer_queue || ctx->transfer_ctx.expired()) {return false;}@@ -8614,12 +5566,12 @@ static bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx) {return true;}-static size_t ggml_vk_align_size(size_t width, size_t align) {+size_t ggml_vk_align_size(size_t width, size_t align) {VK_LOG_DEBUG("ggml_vk_align_size(" << width << ", " << align << ")");return CEIL_DIV(width, align) * align;}-static void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys = nullptr) {+void deferred_memcpy(void * dst, const void * src, size_t size, std::vector<vk_staging_memcpy>* memcpys) {if (memcpys == nullptr) {memcpy(dst, src, size);} else {@@ -8627,7 +5579,7 @@ static void deferred_memcpy(void * dst, const void * src, size_t size, std::vect}}-static void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets = nullptr) {+void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_staging_memset>* memsets) {if (memsets == nullptr) {memset(dst, val, size);} else {@@ -8635,444 +5587,6 @@ static void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<v}}-static void ggml_vk_ensure_sync_staging_buffer(vk_device& device, size_t size) {- if (device->sync_staging == nullptr || device->sync_staging->size < size) {- VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");- ggml_vk_destroy_buffer(device->sync_staging);- device->sync_staging = ggml_vk_create_buffer_check(device, size,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);- }-}--static void ggml_vk_ensure_sync_staging_buffer(ggml_backend_vk_context * ctx, size_t size) {- if (ctx->sync_staging == nullptr || ctx->sync_staging->size < size) {- VK_LOG_MEMORY("ggml_vk_ensure_sync_staging_buffer(" << size << ")");- ggml_vk_destroy_buffer(ctx->sync_staging);- ctx->sync_staging = ggml_vk_create_buffer_check(ctx->device, size,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached,- vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent);- }-}--static void ggml_vk_buffer_write_nc_async(ggml_backend_vk_context * ctx, vk_context& subctx, vk_buffer& dst, size_t offset, const ggml_tensor * tensor, bool sync_staging = false) {- VK_LOG_DEBUG("ggml_vk_buffer_write_nc_async(" << tensor << ")");- GGML_ASSERT(!ggml_is_contiguous(tensor));- // Buffer is already mapped- if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {- std::cerr << "ggml_vulkan: buffer_write_nc_async dst buffer is host_visible. Use synchronous write." << std::endl;- GGML_ABORT("fatal error");- }- // Check if src is pinned memory- vk_buffer buf = nullptr;- size_t buf_offset = 0;- ggml_vk_host_get(ctx->device, tensor->data, buf, buf_offset);-- const uint64_t ne0 = tensor->ne[0];- const uint64_t ne1 = tensor->ne[1];- const uint64_t ne2 = tensor->ne[2];- const uint64_t ne3 = tensor->ne[3];- const uint64_t nb0 = tensor->nb[0];- const uint64_t nb1 = tensor->nb[1];- const uint64_t nb2 = tensor->nb[2];- const uint64_t nb3 = tensor->nb[3];- const ggml_type type = tensor->type;- const uint64_t ts = ggml_type_size(type);- const uint64_t bs = ggml_blck_size(type);-- const uint64_t dstnb0 = ts;- const uint64_t dstnb1 = dstnb0*(ne0/bs);- const uint64_t dstnb2 = dstnb1*ne1;- const uint64_t dstnb3 = dstnb2*ne2;-- const uint64_t ne = ggml_nelements(tensor);-- if (buf != nullptr) {- // Memory is pinned, use as staging buffer- std::vector<vk::BufferCopy> slices;-- for (uint64_t i3 = 0; i3 < ne3; i3++) {- for (uint64_t i2 = 0; i2 < ne2; i2++) {- // Find longest contiguous slice- if (ne1*nb1 == dstnb2) {- slices.push_back({ buf_offset + i3*nb3 + i2*nb2, offset + i3*dstnb3 + i2*dstnb2, dstnb2 });- } else {- for (uint64_t i1 = 0; i1 < ne1; i1++) {- if (ne0*nb0/bs == dstnb1) {- slices.push_back({ buf_offset + i3*nb3 + i2*nb2 + i1*nb1, offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, dstnb1 });- } else {- const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;- const uint64_t d_off = offset + i3*dstnb3 + i2*dstnb2 + i1*dstnb1;- for (uint64_t i0 = 0; i0 < ne0; i0++) {- slices.push_back({ s_off + i0*nb0, d_off + i0*dstnb0, dstnb0 });- }- }- }- }- }- }-- ggml_vk_sync_buffers(ctx, subctx);- subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);- return;- }-- if (!sync_staging) {- GGML_ABORT("Asynchronous write to non-pinned memory not supported");- }-- // Staging buffer required- vk_buffer& staging = ctx->device->sync_staging;- const uint64_t copy_size = ts*ne/bs;- ggml_vk_ensure_sync_staging_buffer(ctx->device, copy_size);- VkBufferCopy buf_copy{ 0, offset, copy_size };-- ggml_vk_sync_buffers(ctx, subctx);- vkCmdCopyBuffer(subctx->s->buffer->buf, (VkBuffer)staging->buffer, (VkBuffer)dst->buffer, 1, &buf_copy);-- for (uint64_t i3 = 0; i3 < ne3; i3++) {- for (uint64_t i2 = 0; i2 < ne2; i2++) {- // Find longest contiguous slice- if (ne1*nb1 == dstnb2) {- deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2, dstnb2, &subctx->in_memcpys);- } else {- for (uint64_t i1 = 0; i1 < ne1; i1++) {- if (ne0*nb0/bs == dstnb1) {- deferred_memcpy((uint8_t *)staging->ptr + i3*dstnb3 + i2*dstnb2 + i1*dstnb1, (const uint8_t *) tensor->data + buf_offset + i3*nb3 + i2*nb2 + i1*nb1, dstnb1, &subctx->in_memcpys);- } else {- const uint64_t s_off = buf_offset + i3*nb3 + i2*nb2 + i1*nb1;- const uint64_t d_off = i3*dstnb3 + i2*dstnb2 + i1*dstnb1;- for (uint64_t i0 = 0; i0 < ne0; i0++) {- deferred_memcpy((uint8_t *)staging->ptr + d_off + i0*dstnb0, (const uint8_t *) tensor->data + s_off + i0*nb0, dstnb0, &subctx->in_memcpys);- }- }- }- }- }- }-}--static bool ggml_vk_buffer_write_2d_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) {- VK_LOG_DEBUG("ggml_vk_buffer_write_2d_async(" << width << ", " << height << ")");- // Check if src is pinned memory- vk_buffer buf = nullptr;- size_t buf_offset = 0;- ggml_vk_host_get(dst->device, src, buf, buf_offset);-- if (buf != nullptr) {- // Memory is pinned, use as staging buffer- std::vector<vk::BufferCopy> slices(1);- if (width == spitch && width == dpitch) {- // Only do single write if stride is equal- slices[0].srcOffset = buf_offset;- slices[0].dstOffset = offset;- slices[0].size = width * height;- } else {- slices.resize(height);- for (size_t i = 0; i < height; i++) {- slices[i].srcOffset = buf_offset + i * spitch;- slices[i].dstOffset = offset + i * dpitch;- slices[i].size = width;- }- }-- ggml_vk_sync_buffers(nullptr, subctx);- subctx->s->buffer->buf.copyBuffer(buf->buffer, dst->buffer, slices);- return true;- }- VK_LOG_DEBUG("STAGING");-- if (!sync_staging) {- // copy was not handled caller needs to fall back- return false;- }-- // Staging buffer required- const size_t staging_size = width * height;- ggml_vk_ensure_sync_staging_buffer(dst->device, staging_size);-- vk_buffer& staging_buffer = dst->device->sync_staging;-- std::vector<vk::BufferCopy> slices(1);- if (width == dpitch) {- slices[0].srcOffset = 0;- slices[0].dstOffset = offset;- slices[0].size = staging_size;- } else {- slices.resize(height);- for (size_t i = 0; i < height; i++) {- slices[i].srcOffset = i * width;- slices[i].dstOffset = offset + i * dpitch;- slices[i].size = width;- }- }-- ggml_vk_sync_buffers(nullptr, subctx);- subctx->s->buffer->buf.copyBuffer(staging_buffer->buffer, dst->buffer, slices);-- if (width == spitch) {- deferred_memcpy((uint8_t *)staging_buffer->ptr, src, staging_size, &subctx->in_memcpys);- } else {- for (size_t i = 0; i < height; i++) {- deferred_memcpy((uint8_t *)staging_buffer->ptr + i * width, (const uint8_t *) src + i * spitch, width, &subctx->in_memcpys);- }- }- return true;-}--static bool ggml_vk_buffer_write_async(vk_context subctx, vk_buffer& dst, size_t offset, const void * src, size_t size, bool sync_staging = false) {- VK_LOG_DEBUG("ggml_vk_buffer_write_async(" << size << ")");- return ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, size, size, size, 1, sync_staging);-}--static void ggml_vk_buffer_write_2d(vk_buffer& dst, size_t offset, const void * src, size_t spitch, size_t dpitch, size_t width, size_t height) {- VK_LOG_DEBUG("ggml_vk_buffer_write_2d(" << width << ", " << height << ")");- // Buffer is already mapped- if(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible) {- GGML_ASSERT(dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);-- if (width == spitch && width == dpitch) {- memcpy((uint8_t *)dst->ptr + offset, src, width * height);- } else {- for (size_t i = 0; i < height; i++) {- memcpy((uint8_t *)dst->ptr + offset + i * dpitch, (const uint8_t *) src + i * spitch, width);- }- }- } else {- std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);-- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);- ggml_vk_ctx_begin(dst->device, subctx);- bool ret = ggml_vk_buffer_write_2d_async(subctx, dst, offset, src, spitch, dpitch, width, height, true);- GGML_ASSERT(ret);- ggml_vk_ctx_end(subctx);-- for (auto& cpy : subctx->in_memcpys) {- memcpy(cpy.dst, cpy.src, cpy.n);- }-- for (auto& mset : subctx->memsets) {- memset(mset.dst, mset.val, mset.n);- }-- ggml_vk_submit(subctx, dst->device->fence);- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);- dst->device->device.resetFences({ dst->device->fence });- ggml_vk_queue_command_pools_cleanup(dst->device);- }-}--static void ggml_vk_buffer_write(vk_buffer& dst, size_t offset, const void * src, size_t size) {- VK_LOG_DEBUG("ggml_vk_buffer_write(" << size << ")");- ggml_vk_buffer_write_2d(dst, offset, src, size, size, size, 1);-}--static bool ggml_vk_buffer_read_2d_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height, bool sync_staging = false) {- VK_LOG_DEBUG("ggml_vk_buffer_read_2d_async(offset=" << offset << ", width=" << width << ", height=" << height << ")");- GGML_ASSERT(width > 0);- GGML_ASSERT(height > 0);- GGML_ASSERT(src != nullptr);-- // TODO: staging_offset is not used-- // Check if dst is pinned memory- vk_buffer buf = nullptr;- size_t buf_offset = 0;- ggml_vk_host_get(src->device, dst, buf, buf_offset);-- std::vector<vk::BufferCopy> slices(1);- if (width == spitch && width == dpitch) {- // Only do single write if stride is equal- slices[0].srcOffset = offset;- slices[0].dstOffset = buf_offset;- slices[0].size = width * height;- } else {- slices.resize(height);- for (size_t i = 0; i < height; i++) {- slices[i].srcOffset = offset + i * spitch;- slices[i].dstOffset = buf_offset + i * dpitch;- slices[i].size = width;- }- }-- if (buf != nullptr) {- // Memory is pinned, use as staging buffer- ggml_vk_sync_buffers(nullptr, subctx);- subctx->s->buffer->buf.copyBuffer(src->buffer, buf->buffer, slices);-- return true;- }- VK_LOG_DEBUG("STAGING");-- if (!sync_staging) {- // copy was not handled caller needs to fall back- return false;- }-- // Fall back to staging buffer- const size_t staging_size = width * height;- ggml_vk_ensure_sync_staging_buffer(src->device, staging_size);-- vk_buffer& staging_buffer = src->device->sync_staging;-- std::vector<vk::BufferCopy> staging_slices(1);- if (width == spitch) {- staging_slices[0].srcOffset = offset;- staging_slices[0].dstOffset = 0;- staging_slices[0].size = staging_size;- } else {- staging_slices.resize(height);- for (size_t i = 0; i < height; i++) {- staging_slices[i].srcOffset = offset + i * spitch;- staging_slices[i].dstOffset = i * width;- staging_slices[i].size = width;- }- }-- ggml_vk_sync_buffers(nullptr, subctx);- subctx->s->buffer->buf.copyBuffer(src->buffer, staging_buffer->buffer, staging_slices);-- if (width == dpitch) {- deferred_memcpy(dst, staging_buffer->ptr, staging_size, &subctx->out_memcpys);- } else {- for (size_t i = 0; i < height; i++) {- deferred_memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) staging_buffer->ptr + i * width, width, &subctx->out_memcpys);- }- }- return true;-}--static bool ggml_vk_buffer_read_async(vk_context subctx, vk_buffer& src, size_t offset, void * dst, size_t size, bool sync_staging = false) {- return ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, size, size, size, 1, sync_staging);-}--static void ggml_vk_buffer_read_2d(vk_buffer& src, size_t offset, void * dst, size_t spitch, size_t dpitch, size_t width, size_t height) {- VK_LOG_DEBUG("ggml_vk_buffer_read_2d(" << src->buffer << ", " << offset << ", " << width << ", " << height << ")");-- // If the device is not an UMA device the memory is host-accessible through rebar. While writing- // through PCIe is sufficient fast reading back data from PCIe is slower than going through- // the HW device to host copy path.- if(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible && src->device->uma) {- GGML_ASSERT(src->memory_property_flags & vk::MemoryPropertyFlagBits::eHostCoherent);-- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);- vk_context subctx = ggml_vk_create_temporary_context(src->device->compute_queue->cmd_pool);- ggml_vk_ctx_begin(src->device, subctx);- subctx->s->buffer->buf.pipelineBarrier(- vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer,- vk::PipelineStageFlagBits::eHost,- {},- { { vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite,- vk::AccessFlagBits::eHostRead } },- {}, {});- ggml_vk_ctx_end(subctx);- ggml_vk_submit(subctx, src->device->fence);- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),- "vk_buffer_read_2d uma waitForFences", src->device);- src->device->device.resetFences({ src->device->fence });- ggml_vk_queue_command_pools_cleanup(src->device);-- if (width == spitch && width == dpitch) {- memcpy(dst, (const uint8_t *) src->ptr + offset, width * height);- } else {- for (size_t i = 0; i < height; i++) {- memcpy((uint8_t *) dst + i * dpitch, (const uint8_t *) src->ptr + offset + i * spitch, width);- }- }- } else {- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);-- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);- ggml_vk_ctx_begin(src->device, subctx);- bool ret = ggml_vk_buffer_read_2d_async(subctx, src, offset, dst, spitch, dpitch, width, height, true);- GGML_ASSERT(ret);- ggml_vk_ctx_end(subctx);-- ggml_vk_submit(subctx, src->device->fence);- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);- src->device->device.resetFences({ src->device->fence });- ggml_vk_queue_command_pools_cleanup(src->device);-- for (auto& cpy : subctx->out_memcpys) {- memcpy(cpy.dst, cpy.src, cpy.n);- }- }-}--static void ggml_vk_buffer_read(vk_buffer& src, size_t offset, void * dst, size_t size) {- VK_LOG_DEBUG("ggml_vk_buffer_read(" << src->buffer << ", " << offset << ", " << size << ")");- ggml_vk_buffer_read_2d(src, offset, dst, size, size, size, 1);-}--static void ggml_vk_buffer_copy_async(vk_context& ctx, vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {- VK_LOG_DEBUG("ggml_vk_buffer_copy_async(" << size << ")");- // Make sure both buffers are on same device- GGML_ASSERT(src->device == dst->device);-- VkBufferCopy bc{ src_offset, dst_offset, size };-- vkCmdCopyBuffer(ctx->s->buffer->buf, (VkBuffer)src->buffer, (VkBuffer)dst->buffer, 1, &bc);-}--static void ggml_vk_buffer_copy(vk_buffer& dst, size_t dst_offset, vk_buffer& src, size_t src_offset, size_t size) {- if (src->device == dst->device) {- std::lock_guard<std::recursive_mutex> guard(src->device->mutex);- VK_LOG_DEBUG("ggml_vk_buffer_copy(SINGLE_DEVICE, " << size << ")");- // Copy within the device- vk_context subctx = ggml_vk_create_temporary_context(src->device->transfer_queue->cmd_pool);- ggml_vk_ctx_begin(src->device, subctx);- ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);- ggml_vk_ctx_end(subctx);- ggml_vk_submit(subctx, src->device->fence);- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);- src->device->device.resetFences({ src->device->fence });- ggml_vk_queue_command_pools_cleanup(src->device);- } else {- VK_LOG_DEBUG("ggml_vk_buffer_copy(MULTI_DEVICE, " << size << ")");- // Copy device to device- ggml_vk_ensure_sync_staging_buffer(src->device, size);-- // Copy to src staging buffer- ggml_vk_buffer_copy(src->device->sync_staging, 0, src, src_offset, size);- // Copy to dst buffer- ggml_vk_buffer_write(dst, dst_offset, src->device->sync_staging->ptr, size);- }-}--static void ggml_vk_buffer_memset_async(vk_context& ctx, vk_buffer& dst, size_t offset, uint32_t c, size_t size) {- VK_LOG_DEBUG("ggml_vk_buffer_memset_async(" << offset << ", " << c << ", " << size << ")");-- if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&- dst->device->uma) {- deferred_memset((uint8_t*)dst->ptr + offset, c, size, &ctx->memsets);- return;- }-- // Fall back to GPU fillBuffer for non-UMA or non-host-visible buffers- ctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);-}--static void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t size) {- VK_LOG_DEBUG("ggml_vk_buffer_memset(" << offset << ", " << c << ", " << size << ")");-- if (dst->memory_property_flags & vk::MemoryPropertyFlagBits::eHostVisible &&- dst->device->uma) {- memset((uint8_t*)dst->ptr + offset, c, size);- return;- }-- std::lock_guard<std::recursive_mutex> guard(dst->device->mutex);- vk_context subctx = ggml_vk_create_temporary_context(dst->device->transfer_queue->cmd_pool);- ggml_vk_ctx_begin(dst->device, subctx);- subctx->s->buffer->buf.fillBuffer(dst->buffer, offset, size, c);- ggml_vk_ctx_end(subctx);-- ggml_vk_submit(subctx, dst->device->fence);- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);- dst->device->device.resetFences({ dst->device->fence });- ggml_vk_queue_command_pools_cleanup(dst->device);-}-static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, uint32_t n, uint32_t k, bool disable_split_k, const vk_pipeline& pipeline) {VK_LOG_DEBUG("ggml_vk_guess_split_k(" << m << ", " << n << ", " << k << ", " << disable_split_k << ")");@@ -9115,9 +5629,7 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,return split_k;}---static void ggml_vk_matmul(+void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer,uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,@@ -9166,7 +5678,6 @@ static void ggml_vk_matmul(ctx->prealloc_split_k_need_sync = true;}-static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) {if (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16) return false;if (src1_type == GGML_TYPE_Q8_1) return false;@@ -9234,14 +5745,14 @@ static void ggml_vk_matmul_id(ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as });}-static bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {+bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {returntensor->nb[0] == ggml_type_size(tensor->type) &&tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&(tensor->ne[3] == 1 || tensor->nb[3] == tensor->nb[2]*tensor->ne[2]);}-static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) {+vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to) {// Choose "contiguous copy" shader if src/dst are contiguousbool contig = ggml_is_contiguous(src) && (!dst || ggml_is_contiguous(dst));@@ -9382,7 +5893,7 @@ static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, constGGML_ABORT("fatal error");}-static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) {+void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor, const vk_subbuffer & in, const vk_subbuffer & out) {VK_LOG_DEBUG("ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), ";std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")");@@ -9407,8 +5918,6 @@ static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context&ggml_vk_sync_buffers(ctx, subctx);}-// Copy/convert tensor into a caller-defined dense layout. Destination strides-// are in output elements, not bytes.static void ggml_vk_cpy_to_strided(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline pipeline, const ggml_tensor * tensor,const vk_subbuffer & in, const vk_subbuffer & out,@@ -9437,7 +5946,7 @@ static void ggml_vk_cpy_to_strided(ggml_vk_sync_buffers(ctx, subctx);}-static vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) {+vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type) {switch(type) {case GGML_TYPE_Q8_1:return ctx->device->pipeline_quantize_q8_1_x4;@@ -9447,7 +5956,7 @@ static vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx,}}-static void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) {+void ggml_vk_quantize_q8_1(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_subbuffer & in, const vk_subbuffer & out, uint32_t ne) {VK_LOG_DEBUG("ggml_vk_quantize_q8_1(" << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ", " << ne << ")");vk_pipeline pipeline = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);@@ -9752,7 +6261,6 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub}}-// Device tuningstatic bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_t n, uint32_t k, ggml_type src0_type) {if (device->mmvq_mode == 1) {return true;@@ -10273,7 +6781,7 @@ static int ggml_vk_fwht_pipeline_idx(int64_t n) {}}-static bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) {+bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor * src1, const ggml_tensor * dst) {if (ctx->num_additional_fused_ops != 0) {return false;}@@ -10299,7 +6807,7 @@ static bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggmlreturn true;}-static void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) {+void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) {const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]);vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx];@@ -10329,7 +6837,7 @@ static uint32_t ggml_vk_nb_elem(const ggml_tensor * t, int i) {return (uint32_t)(t->nb[i] / ggml_type_size(t->type));}-static void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst) {+void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * mixes, const ggml_tensor * scale, const ggml_tensor * base, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_dsv4_hc_comb(" << mixes << ", " << scale << ", " << base << ", " << dst << ")");vk_pipeline pipeline = ctx->device->pipeline_dsv4_hc_comb_f32;@@ -10359,7 +6867,7 @@ static void ggml_vk_dsv4_hc_comb(ggml_backend_vk_context * ctx, vk_context& subcggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { mixes_buf, scale_buf, base_buf, dst_buf }, pc, { n_tokens, 1, 1 });}-static void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst) {+void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * weights, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_dsv4_hc_pre(" << x << ", " << weights << ", " << dst << ")");const float scale = ggml_get_op_params_f32(dst, 0);@@ -10390,7 +6898,7 @@ static void ggml_vk_dsv4_hc_pre(ggml_backend_vk_context * ctx, vk_context& subctggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, w_buf, d_buf }, pc, { n_embd, n_tokens, 1 });}-static void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst) {+void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * x, const ggml_tensor * residual, const ggml_tensor * post, const ggml_tensor * comb, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_dsv4_hc_post(" << x << ", " << residual << ", " << post << ", " << comb << ", " << dst << ")");vk_pipeline pipeline = comb ? ctx->device->pipeline_dsv4_hc_post_f32 : ctx->device->pipeline_dsv4_hc_post_nocomb_f32;@@ -10421,7 +6929,7 @@ static void ggml_vk_dsv4_hc_post(ggml_backend_vk_context * ctx, vk_context& subcggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, r_buf, p_buf, c_buf, d_buf }, pc, { n_embd, n_tokens, 1 });}-static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {ggml_tensor * dst = cgraph->nodes[node_idx];ggml_tensor * src0 = dst->src[0];ggml_tensor * src1 = dst->src[1];@@ -11099,14 +7607,14 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte}}-static bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {+bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {ggml_tensor * dst = cgraph->nodes[node_idx];ggml_tensor * src0 = dst->src[0];ggml_tensor * src2 = dst->src[2];return (src2->ne[1] <= 8) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));}-static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {ggml_tensor * dst = cgraph->nodes[node_idx];ggml_tensor * src0 = dst->src[0];ggml_tensor * src1 = dst->src[1];@@ -11119,7 +7627,7 @@ static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx}}-static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {+bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {GGML_UNUSED(f32acc);// Needs to be kept up to date on shader changesconst uint32_t wg_size = params.workgroup_size;@@ -11170,7 +7678,7 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, conreturn supported;}-static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {+bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) {GGML_UNUSED(v_type);// Needs to be kept up to date on shader changesconst uint32_t Br = params.block_rows;@@ -11218,7 +7726,7 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, coreturn supported;}-static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) {+void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v, const ggml_tensor * mask, const ggml_tensor * sinks, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_flash_attn((" << q << ", name=" << q->name << ", type=" << q->type << ", ne0=" << q->ne[0] << ", ne1=" << q->ne[1] << ", ne2=" << q->ne[2] << ", ne3=" << q->ne[3] << ", nb0=" << q->nb[0] << ", nb1=" << q->nb[1] << ", nb2=" << q->nb[2] << ", nb3=" << q->nb[3];std::cerr << "), (" << k << ", name=" << k->name << ", type=" << k->type << ", ne0=" << k->ne[0] << ", ne1=" << k->ne[1] << ", ne2=" << k->ne[2] << ", ne3=" << k->ne[3] << ", nb0=" << k->nb[0] << ", nb1=" << k->nb[1] << ", nb2=" << k->nb[2] << ", nb3=" << k->nb[3];std::cerr << "), (" << v << ", name=" << v->name << ", type=" << v->type << ", ne0=" << v->ne[0] << ", ne1=" << v->ne[1] << ", ne2=" << v->ne[2] << ", ne3=" << v->ne[3] << ", nb0=" << v->nb[0] << ", nb1=" << v->nb[1] << ", nb2=" << v->nb[2] << ", nb3=" << v->nb[3];@@ -12405,122 +8913,15 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, constif (dst->type == GGML_TYPE_F32) {return ctx->device->pipeline_fill_f32;}- if (dst->type == GGML_TYPE_F16) {- return ctx->device->pipeline_fill_f16;- }- return nullptr;- default:- return nullptr;- }-- GGML_UNUSED(src2);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_unary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.misalign_offsets = (a_offset << 16) | d_offset;-- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_glu_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t b_offset = src1 ? get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type) : a_offset;- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- GGML_ASSERT(a_offset < (1u << 8));- GGML_ASSERT(b_offset < (1u << 8));- GGML_ASSERT(d_offset < (1u << 8));-- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;-- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_sum_rows_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.misalign_offsets = (a_offset << 16) | d_offset;-- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_pad_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.misalign_offsets = (a_offset << 16) | d_offset;-- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_im2col_3d_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.misalign_offsets = (a_offset << 16) | d_offset;-- GGML_UNUSED(src0);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_binary_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / ggml_type_size(src1->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- GGML_ASSERT(a_offset <= 0xFFFF);- GGML_ASSERT(b_offset <= 0xFF);- GGML_ASSERT(d_offset <= 0xFF);-- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;-- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size;- const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size;- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size;-- p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset;-- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);-- p.a_offset = a_offset;- p.d_offset = d_offset;-- GGML_UNUSED(src1);- GGML_UNUSED(src2);- GGML_UNUSED(src3);-}--template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_rope_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) {- p.a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type);- p.d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type);+ if (dst->type == GGML_TYPE_F16) {+ return ctx->device->pipeline_fill_f16;+ }+ return nullptr;+ default:+ return nullptr;+ }- GGML_UNUSED(src1);GGML_UNUSED(src2);- GGML_UNUSED(src3);}template<typename PC>@@ -12925,7 +9326,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co}}-static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -12940,7 +9341,7 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context& subctx,});}-static void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -12955,7 +9356,7 @@ static void ggml_vk_get_rows_back(ggml_backend_vk_context * ctx, vk_context& sub});}-static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -12975,7 +9376,7 @@ static void ggml_vk_acc(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {const ggml_tensor *first_node = cgraph->nodes[node_idx];const ggml_tensor *dst = cgraph->nodes[node_idx + ctx->num_additional_fused_ops];@@ -13082,7 +9483,7 @@ static void ggml_vk_multi_add(ggml_backend_vk_context * ctx, vk_context& subctx,}, pc, elements);}-static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13097,7 +9498,7 @@ static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-static void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13115,7 +9516,7 @@ static void ggml_vk_out_prod(ggml_backend_vk_context * ctx, vk_context& subctx,});}-static void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13130,7 +9531,7 @@ static void ggml_vk_sub(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13145,8 +9546,7 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-// index into device->pipeline_unary_mul for the supported unary ops, or -1-static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {+int ggml_vk_unary_mul_op_index(ggml_unary_op op) {switch (op) {case GGML_UNARY_OP_GELU: return 0;case GGML_UNARY_OP_SIGMOID: return 1;@@ -13156,7 +9556,7 @@ static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {}}-static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {const ggml_tensor * unary = cgraph->nodes[node_idx];ggml_tensor * mul = cgraph->nodes[node_idx + 1];@@ -13191,7 +9591,7 @@ static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx,}, pipeline);}-static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13206,7 +9606,7 @@ static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-static void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {+void ggml_vk_add_id(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t src2_type_size = ggml_type_size(src2->type);@@ -13262,7 +9662,7 @@ static void ggml_vk_op_f32_wkv(ggml_backend_vk_context * ctx, vk_context& subctx}}-static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const size_t seq_length = dst->src[0]->ne[2];const size_t n_embed = dst->ne[0];const size_t n_heads = dst->src[0]->ne[1];@@ -13280,7 +9680,7 @@ static void ggml_vk_rwkv_wkv6(ggml_backend_vk_context * ctx, vk_context& subctx,);}-static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const size_t seq_length = dst->src[0]->ne[2];const size_t n_embed = dst->ne[0];const size_t n_heads = dst->src[0]->ne[1];@@ -13298,7 +9698,7 @@ static void ggml_vk_rwkv_wkv7(ggml_backend_vk_context * ctx, vk_context& subctx,);}-static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const size_t seq_length = dst->src[0]->ne[2];const size_t n_embed = dst->ne[0];const size_t n_heads = dst->src[0]->ne[1];@@ -13333,7 +9733,7 @@ static void ggml_vk_gated_linear_attn(ggml_backend_vk_context * ctx, vk_context&pc, { (uint32_t)(n_seqs * n_heads), 1, 1 });}-static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const ggml_tensor * q = dst->src[0];const ggml_tensor * k = dst->src[1];const ggml_tensor * w = dst->src[2];@@ -13382,7 +9782,7 @@ static void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context&pc, {dispatch_x, dispatch_y, 1});}-static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const ggml_tensor * src_q = dst->src[0];const ggml_tensor * src_v = dst->src[2];const ggml_tensor * src_beta = dst->src[4];@@ -13439,7 +9839,7 @@ static void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& spc, { H, n_seqs, S_v });}-static void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];const ggml_tensor * src2 = dst->src[2];@@ -13496,7 +9896,7 @@ static void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx,pc, elements);}-static void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_ssm_conv(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {ggml_tensor * conv = cgraph->nodes[node_idx];const ggml_tensor * src0 = conv->src[0];const ggml_tensor * src1 = conv->src[1];@@ -13570,7 +9970,7 @@ static void ggml_vk_op_f32_opt_step_adamw(ggml_backend_vk_context * ctx, vk_contpc, elements);}-static void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const size_t n = ggml_nelements(dst->src[0]);ggml_vk_op_f32_opt_step_adamw(@@ -13579,13 +9979,13 @@ static void ggml_vk_opt_step_adamw(ggml_backend_vk_context * ctx, vk_context& su);}-static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {+void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {const size_t n = ggml_nelements(dst->src[0]);ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_OPT_STEP_SGD, { (uint32_t)n, 0, 0.0f, 0.0f, 0.0f, 0.0f });}-static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {int * op_params = (int *)dst->op_params;const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type);@@ -13612,7 +10012,7 @@ static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, coggml_vk_op_f32<vk_op_concat_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc));}-static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t mode = (uint32_t)ggml_get_op_params_i32(dst, 0);@@ -13639,7 +10039,7 @@ static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, c});}-static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = ggml_get_op_params_f32(dst, 0);p.param2 = ggml_get_op_params_f32(dst, 1);@@ -13647,15 +10047,15 @@ static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context& subctx, conggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SCALE, std::move(p));}-static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQR, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_sqrt(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SQRT, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13670,7 +10070,7 @@ static void ggml_vk_add1(ggml_backend_vk_context * ctx, vk_context& subctx, cons});}-static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_arange(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")");vk_op_push_constants pc = {@@ -13692,7 +10092,7 @@ static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);}-static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")");const uint64_t n = ggml_nelements(dst);GGML_ASSERT(n > 0);@@ -13721,32 +10121,32 @@ static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggmlggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);}-static void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_cos(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COS, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_log(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LOG, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = ggml_get_op_params_f32(dst, 0);ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_TRI, std::move(p));}-static void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_diag(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG, std::move(p));}-static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = ggml_get_op_params_f32(dst, 0);p.param2 = ggml_get_op_params_f32(dst, 1);@@ -13754,12 +10154,12 @@ static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context& subctx, conggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CLAMP, std::move(p));}-static void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_pad_push_constants p = vk_op_pad_push_constants_init(src0, dst);ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD, std::move(p));}-static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const uint32_t p0 = (uint32_t)dst->op_params[0];const uint32_t p1 = (uint32_t)dst->op_params[1];@@ -13770,7 +10170,7 @@ static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& suggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD_REFLECT_1D, std::move(p));}-static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const int32_t s0 = ggml_get_op_params_i32(dst, 0);const int32_t s1 = ggml_get_op_params_i32(dst, 1);const int32_t s2 = ggml_get_op_params_i32(dst, 2);@@ -13785,17 +10185,17 @@ static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, consggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ROLL, std::move(p));}-static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT, std::move(p));}-static void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_repeat_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_REPEAT_BACK, std::move(p));}-static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {uint32_t ne = (uint32_t)ggml_nelements(src0);if (ggml_is_quantized(src0->type) && ggml_is_quantized(dst->type)) {// Convert from number of logical elements to 2- or 4-byte units.@@ -13811,7 +10211,7 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context& subctx, constggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_CPY, std::move(p));}-static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -13833,11 +10233,11 @@ static void ggml_vk_set_rows(ggml_backend_vk_context * ctx, vk_context& subctx,});}-static void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_silu_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SILU_BACK, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f });}-static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = op_params[0];@@ -13845,7 +10245,7 @@ static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, consggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p));}-static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const int * int_op_params = (const int *)dst->op_params;const float * float_op_params = (const float *)dst->op_params;@@ -13863,7 +10263,7 @@ static uint32_t ggml_vk_rms_num_partials(ggml_backend_vk_context * ctx, const ggreturn num_partials;}-static uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) {+uint32_t ggml_vk_rms_partials_size(ggml_backend_vk_context * ctx, const ggml_tensor *node) {const uint32_t num_partials = ggml_vk_rms_num_partials(ctx, node);const uint32_t num_bytes = ROUNDUP_POW2(num_partials * sizeof(uint32_t), ctx->device->partials_binding_alignment);return num_bytes;@@ -13917,23 +10317,6 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *return rope;}-static vk_op_binary_push_constants ggml_vk_rms_norm_push_constants(- const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst,- float eps, uint32_t num_partials) {- const uint32_t src0_type_size = ggml_type_size(src0->type);- const uint32_t src1_type_size = ggml_type_size(src1->type);- const uint32_t dst_type_size = ggml_type_size(dst->type);-- return {- (uint32_t)ggml_nelements(src0),- (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,- (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,- (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size,- 0,- eps, 0.0f, (int32_t)num_partials,- };-}-static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_tensor * src0) {if (ctx->do_add_rms_partials_offset_calculation) {ctx->prealloc_size_add_rms_partials_offset += ggml_vk_rms_partials_size(ctx, src0);@@ -13942,7 +10325,7 @@ static void ggml_vk_rms_norm_finish(ggml_backend_vk_context * ctx, const ggml_te}}-static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) {+void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx, float * op_params) {ggml_tensor * rms = cgraph->nodes[node_idx];const ggml_tensor * src0 = rms->src[0];@@ -14116,23 +10499,23 @@ static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx,ggml_vk_rms_norm_finish(ctx, src0);}-static void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_rms_norm_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_RMS_NORM_BACK, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f });}-static void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_l2_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const float * op_params = (const float *)dst->op_params;vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = op_params[0];ggml_vk_op_f32<vk_op_unary_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_L2_NORM, std::move(p));}-static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, vk_op_unary_push_constants_init(src0, dst));}-static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);p.param1 = op_params[1];@@ -14142,7 +10525,7 @@ static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, conggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p));}-static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const float * op_params_f = (const float *)dst->op_params;const bool swapped = (bool)dst->op_params[1];@@ -14190,12 +10573,12 @@ static void ggml_vk_glu(ggml_backend_vk_context * ctx, vk_context& subctx, const});}-static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {int32_t * op_params = (int32_t *)dst->op_params;ggml_vk_op_f32<vk_op_diag_mask_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] });}-static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {+void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;float scale = op_params[0];@@ -14279,12 +10662,12 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx,}}-static void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_soft_max_back(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {float * op_params = (float *)dst->op_params;ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_SOFT_MAX_BACK, { (uint32_t)src0->ne[0], (uint32_t)ggml_nrows(src0), op_params[0], op_params[1], 0.0f, 0.0f });}-static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {topk_moe_mode mode = ctx->fused_topk_moe_mode;const bool has_bias = mode == TOPK_MOE_SIGMOID_NORM_BIAS || mode == TOPK_MOE_SQRT_SOFTPLUS_NORM_BIAS;ggml_tensor * logits = cgraph->nodes[node_idx + 0]->src[0];@@ -14370,7 +10753,7 @@ static void ggml_vk_topk_moe(ggml_backend_vk_context * ctx, vk_context& subctx,ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, {logits_buf, bias_buf, weights_buf, ids_buf}, pc, elements);}-static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) {+void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx, bool backprop) {ggml_tensor * dst = cgraph->nodes[node_idx];const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];@@ -14404,7 +10787,7 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, consggml_vk_make_rope_constants(cgraph->nodes[node_idx], src0, src2 != nullptr, backprop, set_rows_stride));}-static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const uint32_t * op_params = (const uint32_t *)dst->op_params;uint32_t ncols = src0->ne[0];@@ -14490,7 +10873,7 @@ static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context& subctx, c}}-static void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {uint32_t ncols = src0->ne[0];uint32_t nrows = ggml_nrows(src0);uint32_t k = dst->ne[0];@@ -14623,7 +11006,7 @@ static void ggml_vk_topk(ggml_backend_vk_context * ctx, vk_context& subctx, consctx->prealloc_x_need_sync = true;}-static void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_cgraph * cgraph, int node_idx) {const ggml_tensor * get_rows = cgraph->nodes[node_idx + 0];const ggml_tensor * add = cgraph->nodes[node_idx + ctx->num_additional_fused_ops - 1];ggml_tensor * top_k = cgraph->nodes[node_idx + ctx->num_additional_fused_ops];@@ -14672,23 +11055,23 @@ static void ggml_vk_topk_qsa(ggml_backend_vk_context * ctx, vk_context& subctx,ctx->prealloc_x_need_sync = true;}-static void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_sum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, ggml_nelements(src0));ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM, p);}-static void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_sum_rows(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SUM_ROWS, p);}-static void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_mean(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_sum_rows_push_constants p = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);p.weight = 1.0f / (float)src0->ne[0];ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_MEAN, p);}-static void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_cumsum(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {vk_op_sum_rows_push_constants pc = vk_op_sum_rows_push_constants_init(src0, dst, src0->ne[0]);// Use the single pass shader when the rows are small or there are enough rows to fill the GPU.// For fewer, larger rows, use the multipass shader to spread each row across SMs.@@ -14745,7 +11128,7 @@ static std::array<uint32_t, 3> ggml_vk_nrows_elements(uint32_t nr) {return { nr, 1, 1 };}-static void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];@@ -14798,7 +11181,7 @@ static void ggml_vk_cross_entropy_loss(ggml_backend_vk_context * ctx, vk_contextctx->prealloc_x_need_sync = true;}-static void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {+void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {const ggml_tensor * grad = dst->src[0];const ggml_tensor * logits = dst->src[1];const ggml_tensor * labels = dst->src[2];@@ -14832,15 +11215,15 @@ static void ggml_vk_cross_entropy_loss_back(ggml_backend_vk_context * ctx, vk_coggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { grad_buf, logits_buf, labels_buf, dst_buf }, pc, ggml_vk_nrows_elements(nrows));}-static void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_argmax(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_ARGMAX, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f, 0.0f, 0.0f });}-static void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_count_equal(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_COUNT_EQUAL, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f, 0.0f, 0.0f });}-static void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);const uint32_t dst_type_size = ggml_type_size(dst->type);@@ -14855,7 +11238,7 @@ static void ggml_vk_solve_tri(ggml_backend_vk_context * ctx, vk_context& subctx,});}-static void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const int32_t s0 = dst->op_params[0];const int32_t s1 = dst->op_params[1];const int32_t p0 = dst->op_params[2];@@ -14895,7 +11278,7 @@ static void ggml_vk_im2col(ggml_backend_vk_context * ctx, vk_context& subctx, co});}-static void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {GGML_TENSOR_BINARY_OP_LOCALSconst int32_t s0 = ((const int32_t *)(dst->op_params))[0];@@ -14961,7 +11344,7 @@ static void ggml_vk_im2col_3d(ggml_backend_vk_context * ctx, vk_context& subctx,ggml_vk_op_f32<vk_op_im2col_3d_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_IM2COL_3D, std::move(pc));}-static void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const uint32_t dim = dst->op_params[0];const uint32_t max_period = dst->op_params[1];const uint32_t nb1 = dst->nb[1] / ggml_type_size(dst->type);@@ -14971,7 +11354,7 @@ static void ggml_vk_timestep_embedding(ggml_backend_vk_context * ctx, vk_context});}-static void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {// src0: (K, Cout, Cin, 1) -- kernel// src1: (L, Cin, 1, 1) -- input// dst: (*, Cout, 1, 1)@@ -15002,7 +11385,7 @@ static void ggml_vk_conv_transpose_1d(ggml_backend_vk_context * ctx, vk_context&ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_TRANSPOSE_1D, std::move(p));}-static void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {// src0: [K_OC, T_in] columns from matmul// dst: [T_out, OC]@@ -15028,11 +11411,7 @@ static void ggml_vk_col2im_1d(ggml_backend_vk_context * ctx, vk_context& subctx,ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_COL2IM_1D, std::move(p));}-// Dispatch the fused snake activation: y = x + sin^2(a * x) * inv_b.-// Match the naive mul -> sin -> sqr -> mul -> add chain and run the-// dedicated kernel directly. The pattern is validated by-// ggml_vk_can_fuse_snake before this call.-static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {+void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_cgraph * cgraph, int node_idx) {const ggml_tensor * mul0 = cgraph->nodes[node_idx + 0];const ggml_tensor * sqr = cgraph->nodes[node_idx + 2];const ggml_tensor * mul1 = cgraph->nodes[node_idx + 3];@@ -15067,7 +11446,7 @@ static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_conteggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements);}-static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {uint32_t op = static_cast<uint32_t>(dst->op_params[0]);const int32_t k0 = dst->op_params[1];const int32_t s0 = dst->op_params[2];@@ -15090,7 +11469,7 @@ static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, c});}-static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {uint32_t op = static_cast<uint32_t>(dst->op_params[0]);const int32_t k1 = dst->op_params[1];const int32_t k0 = dst->op_params[2];@@ -15118,7 +11497,7 @@ static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, c});}-static void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,+void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -15158,7 +11537,7 @@ static void ggml_vk_conv_2d(ggml_backend_vk_context * ctx, vk_context & subctx,ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, dst->op, std::move(p));}-static void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,+void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx, const ggml_tensor * src0,const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -15188,762 +11567,54 @@ static void ggml_vk_conv_3d(ggml_backend_vk_context * ctx, vk_context & subctx,// total input element count must fit in a uint32.GGML_ASSERT((uint64_t)p.IC * p.N * p.IW * p.IH * p.ID <= 0xFFFFFFFFull);- p.nb01 = static_cast<uint32_t>(nb01 / nb00);- p.nb02 = static_cast<uint32_t>(nb02 / nb00);- p.nb03 = static_cast<uint32_t>(nb03 / nb00);-- p.nb11 = static_cast<uint32_t>(nb11 / nb10);- p.nb12 = static_cast<uint32_t>(nb12 / nb10);- p.nb13 = static_cast<uint32_t>(nb13 / nb10);-- p.nb1 = static_cast<uint32_t>(nb1 / nb0);- p.nb2 = static_cast<uint32_t>(nb2 / nb0);- p.nb3 = static_cast<uint32_t>(nb3 / nb0);-- ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p));-}--static void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {- vk_op_conv2d_dw_push_constants p{};- p.ne = ggml_nelements(dst);- p.channels = dst->ne[2];- p.batches = dst->ne[3];- p.dst_w = dst->ne[0];- p.dst_h = dst->ne[1];- p.src_w = src1->ne[0];- p.src_h = src1->ne[1];- p.knl_w = src0->ne[0];- p.knl_h = src0->ne[1];- p.stride_x = dst->op_params[0];- p.stride_y = dst->op_params[1];- p.pad_x = dst->op_params[2];- p.pad_y = dst->op_params[3];- p.dilation_x = dst->op_params[4];- p.dilation_y = dst->op_params[5];-- GGML_ASSERT(src0->ne[3] == p.channels);- GGML_ASSERT(src1->ne[3] == p.batches);-- ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p));-}--static void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {- const float * op_params = (const float *)dst->op_params;- vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);- p.param1 = op_params[0];-- ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));-}--#ifdef GGML_VULKAN_RUN_TESTS-static void ggml_vk_print_matrix_area(const void * data, ggml_type type, int ne0, int ne1, int i0, int i1, int i2) {- if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16) {- return;- }- i0 = std::max(i0, 5);- i1 = std::max(i1, 5);- i2 = std::max(i2, 0);- fprintf(stderr, " ");- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- fprintf(stderr, "%7d ", idx1);- }- fprintf(stderr, "\n");- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {- fprintf(stderr, "%7d: ", idx0);- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- if (idx0 >= 0 && idx0 < ne0 && idx1 >= 0 && idx1 < ne1) {- float val;- if (type == GGML_TYPE_F32) {- val = *((const float *) data + i2*ne1*ne0 + idx1*ne0 + idx0);- } else if (type == GGML_TYPE_F16) {- val = ggml_fp16_to_fp32(*((const ggml_fp16_t *) data + i2*ne1*ne0 + idx1*ne0 + idx0));- } else {- GGML_ABORT("fatal error");- }- fprintf(stderr, "% 7.2f ", val);- } else {- fprintf(stderr, " ");- }- }- fprintf(stderr, "\n");- }-}--template <typename X_TYPE, typename Y_TYPE>-static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, int split_k, int shader_size) {- VK_LOG_DEBUG("ggml_vk_test_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << shader_size << ")");- const size_t x_ne = m * k * batch;- const size_t y_ne = k * n * batch;- const size_t d_ne = m * n * batch;-- ggml_type x_type = std::is_same<float, X_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;- ggml_type y_type = std::is_same<float, Y_TYPE>() ? GGML_TYPE_F32 : GGML_TYPE_F16;- vk_matmul_pipeline_key mm_test_key{x_type, y_type, false, false};- auto mm_test_it = ctx->device->pipeline_matmul.find(mm_test_key);- GGML_ASSERT(mm_test_it != ctx->device->pipeline_matmul.end() && !mm_test_it->second.empty());- auto& mm_test_configs = mm_test_it->second;- GGML_ASSERT(shader_size >= 0 && shader_size < (int)mm_test_configs.size());-- std::string shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_ALIGNED_" + std::to_string(shader_size);- vk_pipeline p = mm_test_configs[shader_size].aligned ? mm_test_configs[shader_size].aligned : mm_test_configs[shader_size].unaligned;-- const size_t kpad = ggml_vk_align_size(k, mm_test_configs[shader_size].align);-- if (k != kpad) {- p = mm_test_configs[shader_size].unaligned;- shname = std::string(ggml_type_name(x_type)) + "_" + std::string(ggml_type_name(y_type)) + "_" + std::to_string(shader_size);- }-- if (split_k > 1) {- ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);-- if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {- // Resize buffer- if (ctx->prealloc_split_k != nullptr) {- ggml_vk_destroy_buffer(ctx->prealloc_split_k);- }- ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});- }- }-- ggml_pipeline_allocate_descriptor_sets(ctx);-- vk_buffer d_X = ggml_vk_create_buffer_check(ctx->device, sizeof(X_TYPE) * x_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer d_Y = ggml_vk_create_buffer_check(ctx->device, sizeof(Y_TYPE) * y_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer d_D = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne, {vk::MemoryPropertyFlagBits::eDeviceLocal});-- X_TYPE* x = (X_TYPE *) malloc(sizeof(X_TYPE) * x_ne);- Y_TYPE* y = (Y_TYPE *) malloc(sizeof(Y_TYPE) * y_ne);- float* d = (float *) malloc(sizeof(float) * d_ne);-- for (size_t i = 0; i < x_ne; i++) {- if (std::is_same<float, X_TYPE>()) {- x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;- // x[i] = 1.0f;- // x[i] = i + 1;- // x[i] = (i % k == i / k) ? 1.0f : 0.0f;- } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {- x[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);- // x[i] = ggml_fp32_to_fp16(1.0f);- // x[i] = ggml_fp32_to_fp16(i + 1);- // x[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);- } else {- GGML_ABORT("fatal error");- }- }- for (size_t i = 0; i < y_ne; i++) {- if (std::is_same<float, Y_TYPE>()) {- y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;- // y[i] = (i % k == i / k) ? 1.0f : 0.0f;- // y[i] = i + 1;- } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {- y[i] = ggml_fp32_to_fp16((rand() / (float)RAND_MAX) * 2.0f - 1.0f);- // y[i] = ggml_fp32_to_fp16((i % k == i / k) ? 1.0f : 0.0f);- // y[i] = ggml_fp32_to_fp16(i + 1);- } else {- GGML_ABORT("fatal error");- }- }-- ggml_vk_buffer_write(d_X, 0, x, sizeof(X_TYPE) * k * m * batch);- ggml_vk_buffer_write(d_Y, 0, y, sizeof(Y_TYPE) * k * n * batch);-- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);- ggml_vk_ctx_begin(ctx->device, subctx);- for (size_t i = 0; i < num_it; i++) {- ggml_vk_matmul(- ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),- m, n, k,- k, k, m, k*m, k*n, m*n,- split_k, batch, batch, batch, 1, 1, n- );- }- ggml_vk_ctx_end(subctx);-- auto begin = std::chrono::high_resolution_clock::now();- ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);- ctx->device->device.resetFences({ ctx->fence });- ggml_vk_queue_command_pools_cleanup(ctx->device);-- auto end = std::chrono::high_resolution_clock::now();- double time = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;-- // copy dst to host- ggml_vk_buffer_read(d_D, 0, d, sizeof(float) * d_ne);-- float * d_chk = (float *) malloc(sizeof(float) * d_ne);-- ggml_init_params iparams = {- /*.mem_size =*/ 1024*1024*1024,- /*.mem_buffer =*/ NULL,- /*.no_alloc =*/ true,- };-- ggml_context * ggml_ctx = ggml_init(iparams);-- ggml_type src0_type;- ggml_type src1_type;-- if (std::is_same<float, X_TYPE>()) {- src0_type = GGML_TYPE_F32;- } else if (std::is_same<ggml_fp16_t, X_TYPE>()) {- src0_type = GGML_TYPE_F16;- } else {- GGML_ABORT("fatal error");- }- if (std::is_same<float, Y_TYPE>()) {- src1_type = GGML_TYPE_F32;- } else if (std::is_same<ggml_fp16_t, Y_TYPE>()) {- src1_type = GGML_TYPE_F16;- } else {- GGML_ABORT("fatal error");- }-- ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, src0_type, k, m, batch);- ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, src1_type, k, n, batch);- ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);-- src0_ggml->data = x;- src1_ggml->data = y;- tensor_ggml->data = d_chk;-- ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);- ggml_build_forward_expand(cgraph, tensor_ggml);-- ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);-- ggml_free(ggml_ctx);-- double avg_err = 0.0;- int first_err_n = -1;- int first_err_m = -1;- int first_err_b = -1;-- for (size_t i = 0; i < m*n*batch; i++) {- double err = std::fabs(d[i] - d_chk[i]);- avg_err += err;-- if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {- first_err_b = i / (m * n);- first_err_n = (i % (m * n)) / m;- first_err_m = (i % (m * n)) % m;- }- }-- avg_err /= m * n;-- double tflops = 2.0*m*n*k*batch*num_it / (time / 1000.0) / (1000.0*1000.0*1000.0*1000.0);-- std::cerr << "TEST " << shname << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;-- if (avg_err > 0.1 || std::isnan(avg_err)) {- std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;- std::cerr << "Actual result: " << std::endl << std::endl;- ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);- std::cerr << "Expected result: " << std::endl << std::endl;- ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- if (split_k > 1) {- float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);- ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);-- std::cerr << "d_buf0: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- std::cerr << "d_buf1: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- std::cerr << "d_buf2: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- std::cerr << "d_buf3: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- free(split_k_buf);- }- }-- free(d_chk);-- ggml_vk_command_pool_cleanup(ctx->device, ctx->compute_cmd_pool);-- ggml_vk_destroy_buffer(d_X);- ggml_vk_destroy_buffer(d_Y);- ggml_vk_destroy_buffer(d_D);-- free(x);- free(y);- free(d);-}--static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, int i0, int i1, int i2, int i3) {- if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) {- return;- }- i0 = std::max(i0, 5);- i1 = std::max(i1, 5);- i2 = std::max(i2, 0);- i3 = std::max(i3, 0);- fprintf(stderr, " ");- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- fprintf(stderr, "%7d ", idx1);- }- fprintf(stderr, "\n");- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {- fprintf(stderr, "%7d: ", idx0);- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {- float val;- if (tensor->type == GGML_TYPE_F32) {- val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);- } else if (tensor->type == GGML_TYPE_F16) {- val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));- } else {- GGML_ABORT("fatal error");- }- fprintf(stderr, "% 7.2f ", val);- } else {- fprintf(stderr, " ");- }- }- fprintf(stderr, "\n");- }-}--static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml_type quant) {- ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr);-}--static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) {- if (quant == GGML_TYPE_F32) {- memcpy(to, from, sizeof(float) * ne);- return;- }-- const auto * tt = ggml_get_type_traits(quant);-- ggml_to_float_t dequant_fn = tt->to_float;-- dequant_fn(from, to, ne);-}--static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {- VK_LOG_DEBUG("ggml_vk_test_dequant(" << ne << ")");- const size_t x_sz = sizeof(float) * ne;- const size_t x_sz_f16 = sizeof(ggml_fp16_t) * ne;- const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);- float * x = (float *) malloc(x_sz);- void * qx = malloc(qx_sz);- vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz_f16, {vk::MemoryPropertyFlagBits::eDeviceLocal});- float * x_ref = (float *) malloc(x_sz);- ggml_fp16_t * x_chk = (ggml_fp16_t *) malloc(x_sz_f16);-- for (size_t i = 0; i < ne; i++) {- x[i] = rand() / (float)RAND_MAX;- }-- vk_pipeline p = ggml_vk_get_to_fp16(ctx, quant);-- ggml_vk_quantize_data(x, qx, ne, quant);- ggml_vk_dequantize_data(qx, x_ref, ne, quant);-- ggml_pipeline_request_descriptor_sets(ctx, p, 1);-- ggml_pipeline_allocate_descriptor_sets(ctx);-- ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);-- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);- ggml_vk_ctx_begin(ctx->device, subctx);- const std::vector<uint32_t> pc = { 1, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne, (uint32_t)ne };- ggml_vk_dispatch_pipeline(ctx, subctx, p, { vk_subbuffer{ qx_buf, 0, qx_sz }, vk_subbuffer{ x_buf, 0, x_sz_f16 } }, pc, { (uint32_t)ne, 1, 1});- ggml_vk_ctx_end(subctx);-- auto begin = std::chrono::high_resolution_clock::now();-- ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);- ctx->device->device.resetFences({ ctx->fence });- ggml_vk_queue_command_pools_cleanup(ctx->device);-- auto end = std::chrono::high_resolution_clock::now();-- double ms_dequant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;- ggml_vk_buffer_read(x_buf, 0, x_chk, x_sz_f16);-- int first_err = -1;-- double avg_err = 0.0;- for (size_t i = 0; i < ne; i++) {- double error = std::fabs(x_ref[i] - ggml_fp16_to_fp32(x_chk[i]));- avg_err += error;-- if (first_err < 0 && error > 0.05) {- first_err = i;- }- }-- avg_err /= ne;-- std::cerr << "TEST DEQUANT " << ggml_type_name(quant) << " time=" << ms_dequant << "ms avg_err=" << avg_err << std::endl;-- if (avg_err > 0.1) {- std::cerr << "first_error = " << first_err << std::endl;- std::cerr << "Actual result: " << std::endl << std::endl;- for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {- std::cerr << ggml_fp16_to_fp32(x_chk[i]) << ", ";- }- std::cerr << std::endl << "Expected result: " << std::endl << std::endl;- for (int i = std::max(0, first_err - 5); i < std::min((int)ne, first_err + 5); i++) {- std::cerr << x_ref[i] << ", ";- }- std::cerr << std::endl;- }-- ggml_vk_destroy_buffer(x_buf);- ggml_vk_destroy_buffer(qx_buf);-- free(x);- free(qx);- free(x_ref);- free(x_chk);-}--// This does not work without ggml q8_1 quantization support-//-// typedef uint16_t ggml_half;-// typedef uint32_t ggml_half2;-//-// #define QK8_1 32-// typedef struct {-// union {-// struct {-// ggml_half d; // delta-// ggml_half s; // d * sum(qs[i])-// } GGML_COMMON_AGGR_S;-// ggml_half2 ds;-// } GGML_COMMON_AGGR_U;-// int8_t qs[QK8_1]; // quants-// } block_q8_1;-//-// static void ggml_vk_test_quantize(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) {-// VK_LOG_DEBUG("ggml_vk_test_quantize(" << ne << ")");-// GGML_ASSERT(quant == GGML_TYPE_Q8_1);-//-// const size_t x_sz = sizeof(float) * ne;-// const size_t qx_sz = ne * ggml_type_size(quant)/ggml_blck_size(quant);-// float * x = (float *) malloc(x_sz);-// block_q8_1 * qx = (block_q8_1 *)malloc(qx_sz);-// block_q8_1 * qx_res = (block_q8_1 *)malloc(qx_sz);-// vk_buffer x_buf = ggml_vk_create_buffer_check(ctx->device, x_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});-// vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});-//-// for (size_t i = 0; i < ne; i++) {-// x[i] = rand() / (float)RAND_MAX;-// }-//-// vk_pipeline p = ggml_vk_get_quantize_pipeline(ctx, quant);-//-// ggml_pipeline_request_descriptor_sets(ctx, p, 1);-//-// ggml_pipeline_allocate_descriptor_sets(ctx);-//-// ggml_vk_buffer_write(x_buf, 0, x, x_sz);-//-// vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);-// ggml_vk_ctx_begin(ctx->device, subctx);-// ggml_vk_quantize_q8_1(ctx, subctx, ggml_vk_subbuffer(ctx, x_buf), ggml_vk_subbuffer(ctx, qx_buf), ne);-// ggml_vk_ctx_end(subctx);-//-// auto begin = std::chrono::high_resolution_clock::now();-//-// ggml_vk_submit(subctx, ctx->fence);-// VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_quantize waitForFences");-// ctx->device->device.resetFences({ ctx->fence });-// ggml_vk_queue_command_pools_cleanup(ctx->device);-//-// auto end = std::chrono::high_resolution_clock::now();-//-// double ms_quant = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;-// ggml_vk_buffer_read(qx_buf, 0, qx, qx_sz);-//-// ggml_vk_quantize_data(x, qx_res, ne, quant);-//-// int first_err = -1;-//-// for (size_t i = 0; i < ne / 32; i++) {-// double error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d));-//-// if (first_err < 0 && error > 0.1) {-// first_err = i;-// }-//-// error = std::fabs(ggml_fp16_to_fp32(qx_res[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) - ggml_fp16_to_fp32(qx[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s));-//-// if (first_err < 0 && error > 0.1) {-// first_err = i;-// }-//-// for (size_t j = 0; j < 32; j++) {-// uint64_t error = std::abs(qx_res[i].qs[j] - qx[i].qs[j]);-//-// if (first_err < 0 && error > 1) {-// first_err = i;-// }-// }-// }-//-// std::cerr << "TEST QUANTIZE " << ggml_type_name(quant) << " time=" << ms_quant << "ms " << (first_err == -1 ? "CORRECT" : "INCORRECT") << std::endl;-//-// if (first_err != -1) {-// std::cerr << "first_error = " << first_err << std::endl;-// std::cerr << "Actual result: " << std::endl << std::endl;-// std::cout << "d=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";-// for (size_t j = 0; j < 32; j++) {-// std::cout << " qs" << j << "=" << (uint32_t)qx[first_err].qs[j] << " ";-// }-// std::cerr << std::endl << std::endl << "Expected result: " << std::endl << std::endl;-// std::cout << "d=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d) << " s=" << ggml_fp16_to_fp32(qx_res[first_err].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.s) << " ";-// for (size_t j = 0; j < 32; j++) {-// std::cout << " qs" << j << "=" << (uint32_t)qx_res[first_err].qs[j] << " ";-// }-// std::cerr << std::endl;-// }-//-// ggml_vk_destroy_buffer(x_buf);-// ggml_vk_destroy_buffer(qx_buf);-//-// free(x);-// free(qx);-// free(qx_res);-// }--static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size_t k, size_t batch, size_t num_it, size_t split_k, size_t shader_size, ggml_type quant, bool mmq = false) {- VK_LOG_DEBUG("ggml_vk_test_dequant_matmul(" << m << ", " << n << ", " << k << ", " << batch << ", " << num_it << ", " << split_k << ", " << ggml_type_name(quant) << ")");- const size_t x_ne = m * k * batch;- const size_t y_ne = k * n * batch;- const size_t d_ne = m * n * batch;-- ggml_type b_type = mmq ? GGML_TYPE_Q8_1 : GGML_TYPE_F32;- bool f16acc = ctx->device->fp16 && !mmq;- vk_matmul_pipeline_key dq_key{quant, b_type, false, f16acc};- auto dq_it = ctx->device->pipeline_matmul.find(dq_key);- if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {- if (f16acc) {- dq_key.f16acc = false;- dq_it = ctx->device->pipeline_matmul.find(dq_key);- }- }- if (dq_it == ctx->device->pipeline_matmul.end() || dq_it->second.empty()) {- std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;- return;- }- auto& dq_configs = dq_it->second;- if (shader_size >= (int)dq_configs.size()) {- std::cerr << "error: shader_size " << shader_size << " >= configs.size() " << dq_configs.size() << " for " << ggml_type_name(quant) << std::endl;- return;- }-- std::string shname = std::string(ggml_type_name(quant)) + "_ALIGNED_" + std::to_string(shader_size);- vk_pipeline p = dq_configs[shader_size].aligned ? dq_configs[shader_size].aligned : dq_configs[shader_size].unaligned;-- const size_t kpad = mmq ? 0 : ggml_vk_align_size(k, dq_configs[shader_size].align);-- if (mmq || k != kpad) {- p = dq_configs[shader_size].unaligned;- shname = std::string(ggml_type_name(quant)) + "_" + std::to_string(shader_size);- }-- if (p == nullptr) {- std::cerr << "error: no pipeline for ggml_vk_test_dequant_matmul " << ggml_type_name(quant) << std::endl;- return;- }-- const size_t x_sz = sizeof(float) * x_ne;- const size_t y_sz = sizeof(float) * y_ne;- const size_t qx_sz = x_ne * ggml_type_size(quant)/ggml_blck_size(quant);- const size_t qy_sz = mmq ? y_ne * ggml_type_size(GGML_TYPE_Q8_1)/ggml_blck_size(GGML_TYPE_Q8_1) : y_sz;- const size_t d_sz = sizeof(float) * d_ne;- float * x = (float *) malloc(x_sz);- float * y = (float *) malloc(y_sz);- void * qx = malloc(qx_sz);- vk_buffer qx_buf = ggml_vk_create_buffer_check(ctx->device, qx_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer y_buf = ggml_vk_create_buffer_check(ctx->device, y_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer qy_buf = ggml_vk_create_buffer_check(ctx->device, qy_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});- vk_buffer d_buf = ggml_vk_create_buffer_check(ctx->device, d_sz, {vk::MemoryPropertyFlagBits::eDeviceLocal});- float * d = (float *) malloc(d_sz);- float * d_chk = (float *) malloc(d_sz);-- for (size_t i = 0; i < x_ne; i++) {- x[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;- // x[i] = (i % k == i / k) ? 1.0f : 0.0f;- // x[i] = i % k;- }-- ggml_vk_quantize_data(x, qx, x_ne, quant);-- for (size_t i = 0; i < y_ne; i++) {- y[i] = (rand() / (float)RAND_MAX) * 2.0f - 1.0f;- // y[i] = (i % k == i / k) ? 1.0f : 0.0f;- // y[i] = i % k;- }-- if (split_k > 1) {- ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, num_it);-- if (ctx->prealloc_split_k == nullptr || ctx->prealloc_split_k->size < sizeof(float) * d_ne * split_k) {- // Resize buffer- if (ctx->prealloc_split_k != nullptr) {- ggml_vk_destroy_buffer(ctx->prealloc_split_k);- }- ctx->prealloc_split_k = ggml_vk_create_buffer_check(ctx->device, sizeof(float) * d_ne * split_k, {vk::MemoryPropertyFlagBits::eDeviceLocal});- }- }- if (mmq) {- vk_pipeline pipeline_quantize_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);- ggml_pipeline_request_descriptor_sets(ctx, pipeline_quantize_q8_1, num_it);- }-- ggml_pipeline_allocate_descriptor_sets(ctx);-- ggml_vk_buffer_write(qx_buf, 0, qx, qx_sz);- ggml_vk_buffer_write(y_buf, 0, y, y_sz);-- vk_context subctx = ggml_vk_create_context(ctx, ctx->compute_cmd_pool);- ggml_vk_ctx_begin(ctx->device, subctx);- if (mmq) {- for (size_t i = 0; i < num_it; i++) {- ggml_vk_quantize_q8_1(ctx, subctx, { y_buf, 0, y_sz }, { qy_buf, 0, qy_sz }, y_ne);- ggml_vk_matmul(- ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },- m, n, k,- k, k, m, k*m, k*n, m*n,- split_k, batch, batch, batch, 1, 1, n- );- }- } else {- for (size_t i = 0; i < num_it; i++) {- ggml_vk_matmul(- ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },- m, n, k,- k, k, m, k*m, k*n, m*n,- split_k, batch, batch, batch, 1, 1, n- );- }- }- ggml_vk_ctx_end(subctx);-- auto begin = std::chrono::high_resolution_clock::now();-- ggml_vk_submit(subctx, ctx->fence);- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);- ctx->device->device.resetFences({ ctx->fence });- ggml_vk_queue_command_pools_cleanup(ctx->device);-- auto end = std::chrono::high_resolution_clock::now();-- double time_ms = std::chrono::duration_cast<std::chrono::microseconds>(end-begin).count() / 1000.0;- ggml_vk_buffer_read(d_buf, 0, d, d_sz);-- ggml_init_params iparams = {- /*.mem_size =*/ 1024*1024*1024,- /*.mem_buffer =*/ NULL,- /*.no_alloc =*/ true,- };-- ggml_context * ggml_ctx = ggml_init(iparams);-- ggml_tensor * src0_ggml = ggml_new_tensor_3d(ggml_ctx, quant, k, m, batch);- ggml_tensor * src1_ggml = ggml_new_tensor_3d(ggml_ctx, GGML_TYPE_F32, k, n, batch);- ggml_tensor * tensor_ggml = ggml_mul_mat(ggml_ctx, src0_ggml, src1_ggml);-- src0_ggml->data = qx;- src1_ggml->data = y;- tensor_ggml->data = d_chk;-- ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx);- ggml_build_forward_expand(cgraph, tensor_ggml);-- ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1);-- ggml_free(ggml_ctx);-- double avg_err = 0.0;- int first_err_n = -1;- int first_err_m = -1;- int first_err_b = -1;-- for (size_t i = 0; i < m*n*batch; i++) {- double err = std::fabs(d[i] - d_chk[i]);- avg_err += err;-- if ((err > 0.05f || std::isnan(err)) && first_err_n == -1) {- first_err_b = i / (m * n);- first_err_n = (i % (m * n)) / m;- first_err_m = (i % (m * n)) % m;- }- }-- avg_err /= m * n;-- double tflops = 2.0*m*n*k*batch*num_it / (time_ms / 1000.0) / (1000.0*1000.0*1000.0*1000.0);-- std::cerr << "TEST dequant matmul " << shname;- if (mmq) {- std::cerr << " mmq";- }- std::cerr << " m=" << m << " n=" << n << " k=" << k << " batch=" << batch << " split_k=" << split_k << " matmul " << time_ms / num_it << "ms " << tflops << " TFLOPS avg_err=" << avg_err << std::endl;-- if (avg_err > 0.01 || std::isnan(avg_err)) {- std::cerr << "m = " << first_err_m << " n = " << first_err_n << " b = " << first_err_b << std::endl;- std::cerr << "Actual result: " << std::endl << std::endl;- ggml_vk_print_matrix_area(d, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);- std::cerr << std::endl;- std::cerr << "Expected result: " << std::endl << std::endl;- ggml_vk_print_matrix_area(d_chk, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);-- std::cerr << "src0: " << std::endl << std::endl;- ggml_vk_print_matrix_area(x, GGML_TYPE_F32, k, m, first_err_m, first_err_n, first_err_b);- std::cerr << std::endl;- std::cerr << "src1: " << std::endl << std::endl;- ggml_vk_print_matrix_area(y, GGML_TYPE_F32, k, n, first_err_m, first_err_n, first_err_b);+ p.nb01 = static_cast<uint32_t>(nb01 / nb00);+ p.nb02 = static_cast<uint32_t>(nb02 / nb00);+ p.nb03 = static_cast<uint32_t>(nb03 / nb00);- if (split_k > 1) {- float * split_k_buf = (float *) malloc(sizeof(float) * d_ne * split_k);- ggml_vk_buffer_read(ctx->prealloc_split_k, 0, split_k_buf, sizeof(float) * d_ne * split_k);+ p.nb11 = static_cast<uint32_t>(nb11 / nb10);+ p.nb12 = static_cast<uint32_t>(nb12 / nb10);+ p.nb13 = static_cast<uint32_t>(nb13 / nb10);- std::cerr << "d_buf0: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+ p.nb1 = static_cast<uint32_t>(nb1 / nb0);+ p.nb2 = static_cast<uint32_t>(nb2 / nb0);+ p.nb3 = static_cast<uint32_t>(nb3 / nb0);- std::cerr << "d_buf1: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+ ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_3D, std::move(p));+}- std::cerr << "d_buf2: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + 2 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+void ggml_vk_conv_2d_dw(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ vk_op_conv2d_dw_push_constants p{};+ p.ne = ggml_nelements(dst);+ p.channels = dst->ne[2];+ p.batches = dst->ne[3];+ p.dst_w = dst->ne[0];+ p.dst_h = dst->ne[1];+ p.src_w = src1->ne[0];+ p.src_h = src1->ne[1];+ p.knl_w = src0->ne[0];+ p.knl_h = src0->ne[1];+ p.stride_x = dst->op_params[0];+ p.stride_y = dst->op_params[1];+ p.pad_x = dst->op_params[2];+ p.pad_y = dst->op_params[3];+ p.dilation_x = dst->op_params[4];+ p.dilation_y = dst->op_params[5];- std::cerr << "d_buf3: " << std::endl << std::endl;- ggml_vk_print_matrix_area(split_k_buf + 3 * d_ne, GGML_TYPE_F32, m, n, first_err_m, first_err_n, first_err_b);+ GGML_ASSERT(src0->ne[3] == p.channels);+ GGML_ASSERT(src1->ne[3] == p.batches);- free(split_k_buf);- }- }+ ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONV_2D_DW, std::move(p));+}- ggml_vk_destroy_buffer(qx_buf);- ggml_vk_destroy_buffer(y_buf);- ggml_vk_destroy_buffer(qy_buf);- ggml_vk_destroy_buffer(d_buf);+void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+ const float * op_params = (const float *)dst->op_params;+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);+ p.param1 = op_params[0];- free(x);- free(qx);- free(y);- free(d);- free(d_chk);+ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));}-#endif-static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) {+void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx) {#if defined(GGML_VULKAN_RUN_TESTS)const std::vector<size_t> vals {512, 512, 128,@@ -16081,11 +11752,7 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_contex}}-static void ggml_vk_compute_forward(ggml_backend_vk_context* ctx, ggml_cgraph * cgraph, ggml_tensor* tensor, int tensor_idx, bool almost_ready);--// Returns true if node has enqueued work into the queue, false otherwise-// If submit is true the current all operations queued so far are being submitted to Vulkan to overlap cmdlist creation and GPU execution.-static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){+bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int node_idx, ggml_tensor *node_begin, int node_idx_begin, bool last_node, bool almost_ready, bool submit){ggml_tensor * node = cgraph->nodes[node_idx];if (ggml_is_empty(node) || ggml_op_is_empty(node->op) || !node->buffer) {return false;@@ -16658,7 +12325,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgrreturn true;}-static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready = false) {+void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, ggml_tensor * tensor, int tensor_idx, bool almost_ready) {GGML_UNUSED(cgraph);GGML_UNUSED(tensor);@@ -16708,8 +12375,7 @@ static void ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_cgraph *}}-// Clean up after graph processing is done-static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {+void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {VK_LOG_DEBUG("ggml_vk_graph_cleanup()");ctx->prealloc_y_last_pipeline_used = {};ctx->prealloc_y_last_tensor_used = nullptr;@@ -16747,8 +12413,7 @@ static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {ctx->descriptor_set_idx = 0;}-// Clean up on backend free-static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {+void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {VK_LOG_DEBUG("ggml_vk_cleanup(" << ctx->name << ")");// discard any unsubmitted command buffersctx->compute_ctx.reset();@@ -16796,13 +12461,13 @@ static void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {}}-static int ggml_vk_get_device_count() {+int ggml_vk_get_device_count() {ggml_vk_instance_init();return vk_instance.device_indices.size();}-static void ggml_vk_get_device_description(int device, char * description, size_t description_size) {+void ggml_vk_get_device_description(int device, char * description, size_t description_size) {ggml_vk_instance_init();std::vector<vk::PhysicalDevice> devices = vk_instance.instance.enumeratePhysicalDevices();@@ -16813,30 +12478,24 @@ static void ggml_vk_get_device_description(int device, char * description, size_snprintf(description, description_size, "%s", props.deviceName.data());}-// backend interface--#define UNUSED GGML_UNUSED--// device backend--static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {+bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;}-static void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) {+void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) {VK_LOG_MEMORY("ggml_backend_vk_buffer_free_buffer()");ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;ggml_vk_destroy_buffer(ctx->dev_buffer);delete ctx;}-static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) {+void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) {return vk_ptr_base;UNUSED(buffer);}-static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");if (tensor->view_src != nullptr) {GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);@@ -16844,7 +12503,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_treturn GGML_STATUS_SUCCESS;}-static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {+void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;@@ -16857,7 +12516,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, gggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);}-static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;@@ -16869,7 +12528,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggmlggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}-static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,+void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset,size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<n_copies << ", " << stride_tensor << ", " << stride_data << ")");@@ -16883,7 +12542,7 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, gggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);}-static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;@@ -16896,7 +12555,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, consggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}-static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,+void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset,size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<n_copies << ", " << stride_tensor << ", " << stride_data << ")");@@ -16911,7 +12570,7 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, cggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);}-static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {+bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {if (ggml_nbytes(src) == 0) {return true;}@@ -16932,34 +12591,19 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, consUNUSED(buffer);}-static void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+void ggml_backend_vk_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);}-static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {- /* .free_buffer = */ ggml_backend_vk_buffer_free_buffer,- /* .get_base = */ ggml_backend_vk_buffer_get_base,- /* .init_tensor = */ ggml_backend_vk_buffer_init_tensor,- /* .memset_tensor = */ ggml_backend_vk_buffer_memset_tensor,- /* .set_tensor = */ ggml_backend_vk_buffer_set_tensor,- /* .get_tensor = */ ggml_backend_vk_buffer_get_tensor,- /* .set_tensor_2d = */ ggml_backend_vk_buffer_set_tensor_2d,- /* .get_tensor_2d = */ ggml_backend_vk_buffer_get_tensor_2d,- /* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,- /* .clear = */ ggml_backend_vk_buffer_clear,- /* .reset = */ NULL,-};--// vk buffer type-static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) {+const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft) {ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *)buft->context;return ctx->name.c_str();}-static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;@@ -16975,17 +12619,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backereturn ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);}-static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;return ctx->device->properties.limits.minStorageBufferOffsetAlignment;}-static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;return ctx->device->suballocation_block_size;}-static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {return ggml_nbytes(tensor);UNUSED(buft);@@ -17001,8 +12645,6 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {return &dev->buffer_type;}-// host buffer type-static const char * ggml_backend_vk_host_buffer_type_name(ggml_backend_buffer_type_t buft) {return GGML_VK_NAME "_Host";@@ -17048,8 +12690,6 @@ static size_t ggml_backend_vk_host_buffer_type_get_max_size(ggml_backend_buffer_UNUSED(buft);}-// Should be changed to return device-specific host buffer type-// but that probably requires changes in llama.cppggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {static struct ggml_backend_buffer_type ggml_backend_vk_buffer_type_host = {/* .iface = */ {@@ -17071,16 +12711,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {return &ggml_backend_vk_buffer_type_host;}--// backend-static const char * ggml_backend_vk_name(ggml_backend_t backend) {ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;return ctx->name.c_str();}-static void ggml_backend_vk_free(ggml_backend_t backend) {+void ggml_backend_vk_free(ggml_backend_t backend) {ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");@@ -17286,7 +12923,7 @@ static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_bareturn false;}-static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {+void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {VK_LOG_DEBUG("ggml_vk_synchronize()");bool do_transfer = !ctx->compute_ctx.expired();@@ -17366,7 +13003,7 @@ static void ggml_backend_vk_synchronize(ggml_backend_t backend) {ggml_vk_graph_cleanup(ctx);}-static bool ggml_vk_is_empty(ggml_tensor * node) {+bool ggml_vk_is_empty(ggml_tensor * node) {return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;}@@ -17407,7 +13044,7 @@ static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, iggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1);}-static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {+bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) {return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx);}@@ -17603,8 +13240,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct greturn true;}-// Match SSM_CONV + UNARY(SILU) or SSM_CONV + ADD + UNARY(SILU). num_extra is 1 or 2.-static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,+bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,int node_idx, int num_extra) {const ggml_tensor * conv = cgraph->nodes[node_idx];if (conv->op != GGML_OP_SSM_CONV) {@@ -17659,7 +13295,7 @@ static bool ggml_vk_can_fuse_ssm_conv(const ggml_backend_vk_context * ctx, constreturn true;}-static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,+bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,int node_idx, topk_moe_mode mode) {const ggml_tensor * softmax;@@ -17770,7 +13406,6 @@ static bool ggml_vk_can_fuse_topk_moe(ggml_backend_vk_context * ctx, const strucreturn true;}-// Manual op-sequence match (ggml_can_fuse_subgraph rejects the mask's external reshape/cpy).static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx,const std::initializer_list<ggml_op> & ops) {if (node_idx + (int) ops.size() > cgraph->n_nodes) {@@ -17787,8 +13422,7 @@ static bool ggml_vk_match_ops(const struct ggml_cgraph * cgraph, int node_idx,return true;}-// True if the qwen4 QSA indexer top-k can be fused at node_idx (the get_rows).-static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {+bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {if (ctx->device->disable_fusion || !ctx->device->pipeline_topk_radix_qsa) {return false;}@@ -17856,7 +13490,7 @@ static bool ggml_vk_can_fuse_topk_qsa(ggml_backend_vk_context * ctx, const strucreturn true;}-static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,+bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,int node_idx) {const ggml_tensor *rope = cgraph->nodes[node_idx + 0];const ggml_tensor *view = cgraph->nodes[node_idx + 1];@@ -17895,7 +13529,7 @@ static bool ggml_vk_can_fuse_rope_set_rows(ggml_backend_vk_context * ctx, constreturn true;}-static bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,+bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,int node_idx) {const ggml_tensor * rms = cgraph->nodes[node_idx];const ggml_tensor * view = cgraph->nodes[node_idx + 1];@@ -17921,10 +13555,7 @@ static bool ggml_vk_can_fuse_rms_norm_set_rows(ggml_backend_vk_context * ctx, coreturn true;}-// Pattern check for the 5-op Snake fusion: mul -> sin -> sqr -> mul -> add.-// Verifies the chain shape, the closure x_in_add == x_in_mul0, and that-// the broadcast operands a and inv_b share a [1, C] layout.-static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {+bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {GGML_UNUSED(ctx);if (!ggml_can_fuse(cgraph, node_idx, snake_pattern)) {return false;@@ -17980,11 +13611,7 @@ static bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct greturn true;}-// Check whether the tensors overlap in memory.-// Fusions can potentially overwrite src tensors in ways that are not prevented-// by ggml-alloc. If the fusion src is being applied in a way that's elementwise-// with the destination, then it's OK for them to overlap if they are exactly equal.-static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) {+bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise) {ggml_backend_vk_buffer_context * a_buf_ctx = (ggml_backend_vk_buffer_context *)a->buffer->context;vk_buffer a_buf = a_buf_ctx->dev_buffer;ggml_backend_vk_buffer_context * b_buf_ctx = (ggml_backend_vk_buffer_context *)b->buffer->context;@@ -18007,7 +13634,7 @@ static bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * breturn false;}-static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,+bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph,int node_idx) {const ggml_tensor *rms = cgraph->nodes[node_idx + 0];const ggml_tensor *mul = cgraph->nodes[node_idx + 1];@@ -18040,7 +13667,7 @@ static bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, coreturn true;}-static uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {+uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx) {const ggml_tensor *first_node = cgraph->nodes[node_idx];if (first_node->op != GGML_OP_ADD) {@@ -18599,8 +14226,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgUNUSED(backend);}-// Sort the graph for improved parallelism.-static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params)+void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params){VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;@@ -19037,7 +14663,6 @@ static void ggml_backend_vk_event_wait(ggml_backend_t backend, ggml_backend_even}}-// TODO: enable async and synchronizestatic ggml_backend_i ggml_backend_vk_interface = {/* .get_name = */ ggml_backend_vk_name,/* .free = */ ggml_backend_vk_free,@@ -19178,17 +14803,6 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {return std::string(pci_bus_id);}-//////////////////////////--struct ggml_backend_vk_device_context {- size_t device;- std::string name;- std::string description;- bool is_integrated_gpu;- std::string pci_bus_id;- int op_offload_min_batch_size;-};-static const char * ggml_backend_vk_device_get_name(ggml_backend_dev_t dev) {ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;return ctx->name.c_str();@@ -19909,21 +15523,6 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_bareturn buft_ctx->device->idx == ctx->device;}-static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {- switch (op->op) {- case GGML_OP_GET_ROWS:- return 0;- case GGML_OP_MUL_MAT:- return op->ne[1];- case GGML_OP_MUL_MAT_ID:- case GGML_OP_ROPE:- case GGML_OP_ROPE_BACK:- return op->ne[2];- default:- return ggml_nrows(op);- }-}-static bool ggml_backend_vk_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {ggml_backend_vk_device_context * dev_ctx = (ggml_backend_vk_device_context *)dev->context;@@ -20005,31 +15604,6 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm}}-static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size) {- if (!device->external_memory_host) {- return {};- }-- uintptr_t uptr = reinterpret_cast<uintptr_t>(ptr);- if (uptr & (device->min_imported_host_pointer_alignment - 1)) {- return {};- }- if (size & (device->min_imported_host_pointer_alignment - 1)) {- return {};- }-- const vk::MemoryPropertyFlags property_flags = vk::MemoryPropertyFlagBits::eHostVisible | vk::MemoryPropertyFlagBits::eHostCoherent | vk::MemoryPropertyFlagBits::eHostCached;-- vk_buffer buf {};- try {- buf = ggml_vk_create_buffer(device, size, { property_flags }, ptr);- } catch (vk::SystemError& e) {- GGML_LOG_WARN("ggml_vulkan: Failed ggml_vk_create_buffer (%s)\n", e.what());- }-- return buf;-}-static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");GGML_UNUSED(max_tensor_size);@@ -20140,8 +15714,7 @@ ggml_backend_reg_t ggml_backend_vk_reg() {}}-// Extension availability-static bool ggml_vk_instance_layer_settings_available() {+bool ggml_vk_instance_layer_settings_available() {#ifdef GGML_VULKAN_VALIDATE// Check if validation layer provides the extensionconst std::string layer_name = "VK_LAYER_KHRONOS_validation";@@ -20159,7 +15732,8 @@ static bool ggml_vk_instance_layer_settings_available() {#endifreturn false;}-static bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) {++bool ggml_vk_instance_portability_enumeration_ext_available(const std::vector<vk::ExtensionProperties>& instance_extensions) {#ifdef __APPLE__// Check for portability enumeration extension for MoltenVK supportfor (const auto& properties : instance_extensions) {@@ -20174,8 +15748,7 @@ static bool ggml_vk_instance_portability_enumeration_ext_available(const std::veUNUSED(instance_extensions);}-// Extension availability-static bool ggml_vk_instance_debug_utils_ext_available(+bool ggml_vk_instance_debug_utils_ext_available(const std::vector<vk::ExtensionProperties> & instance_extensions) {// Check for portability enumeration extension for MoltenVK supportfor (const auto & properties : instance_extensions) {@@ -20190,7 +15763,7 @@ static bool ggml_vk_instance_debug_utils_ext_available(UNUSED(instance_extensions);}-static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {+bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {VkPhysicalDeviceFeatures2 device_features2;device_features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;@@ -20204,7 +15777,7 @@ static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) {return vk11_features.storageBuffer16BitAccess;}-static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) {+bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) {switch (props.vendorID) {case VK_VENDOR_ID_INTEL:// Only allowing Xe2/Xe3 GPU and integrated Xe GPUs at the moment since older hardware (ex. Arc A770) has performance regressions.@@ -20221,7 +15794,7 @@ static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDevicePrope}}-static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) {+uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) {VkPhysicalDeviceProperties2 props = vkdev.getProperties2();if (props.properties.vendorID != VK_VENDOR_ID_INTEL) {@@ -20264,8 +15837,7 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev)}}-// checks whether lower <= driver_version < upper, with each bound given as xxx.yyyy-static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) {+bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint32_t lower_major, uint32_t lower_minor, uint32_t upper_major, uint32_t upper_minor) {#if defined(_WIN32)// Intel Windows encodes xxx.yyyy as [31:14].[13:0].const uint32_t major = driver_version >> 14;@@ -20285,788 +15857,251 @@ static bool ggml_vk_intel_windows_driver_in_range(uint32_t driver_version, uint3#endif}+GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg)-// checks--#ifdef GGML_VULKAN_CHECK_RESULTS-static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vector<const ggml_tensor *>& done, int level = 0) {- if (std::find(done.begin(), done.end(), tensor) != done.end() || level > 10) {- return;- }- for (int j = 0; j < level; j++) {- std::cerr << " ";- }- std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl;- done.push_back(tensor);+// out-of-lined header method definitions- for (int i = 0; i < GGML_MAX_SRC; i++) {- if (tensor->src[i] != nullptr) {- ggml_vk_print_graph_origin(tensor->src[i], done, level + 1);+void vk_queue_handle_synchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) {+ // Workaround for NVIDIA driver bug+ std::unique_lock<std::mutex> device_guard;+ if (device_submit_mutex) {+ device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);+ }+ std::lock_guard<std::mutex> guard(mutex);+ try {+ queue.submit(submits, fence);+ } catch (vk::DeviceLostError &) {+ if (auto dev = device.lock()) {+ ggml_vk_print_device_lost_info(dev);}+ throw;}}-static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * data, int i0, int i1, int i2, int i3) {- if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) {- return;+void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) {+ // Workaround for NVIDIA driver bug+ std::unique_lock<std::mutex> device_guard;+ if (device_submit_mutex) {+ device_guard = std::unique_lock<std::mutex>(*device_submit_mutex);}- i0 = std::max(i0, 5);- i1 = std::max(i1, 5);- i2 = std::max(i2, 0);- i3 = std::max(i3, 0);- fprintf(stderr, " ");- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- fprintf(stderr, "%7d ", idx1);- }- fprintf(stderr, "\n");- for (int idx0 = i0 - 5; idx0 < i0 + 5; idx0++) {- fprintf(stderr, "%7d: ", idx0);- for (int idx1 = i1 - 5; idx1 < i1 + 5; idx1++) {- if (idx0 >= 0 && idx0 < tensor->ne[0] && idx1 >= 0 && idx1 < tensor->ne[1] && i2 >= 0 && i2 < tensor->ne[2] && i3 >= 0 && i3 < tensor->ne[3]) {- float val;- if (tensor->type == GGML_TYPE_F32) {- val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);- } else if (tensor->type == GGML_TYPE_F16) {- val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]));- } else if (tensor->type == GGML_TYPE_I32) {- val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]);- } else {- GGML_ABORT("fatal error");- }- fprintf(stderr, "% 7.2f ", val);- } else {- fprintf(stderr, " ");- }+ try {+ queue.submit(submits, fence);+ } catch (vk::DeviceLostError &) {+ if (auto dev = device.lock()) {+ ggml_vk_print_device_lost_info(dev);}- fprintf(stderr, "\n");+ throw;}}-static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name) {- void * tensor_data = tensor->data;+vk_device_struct::~vk_device_struct() {+ VK_LOG_DEBUG("destroy device " << name);- const bool is_gpu = tensor->buffer != nullptr && ggml_backend_buffer_is_vk(tensor->buffer);+ device.destroyFence(fence);- if (is_gpu) {- const size_t tensor_size = ggml_nbytes(tensor);- tensor_data = malloc(tensor_size);+ ggml_vk_destroy_buffer(sync_staging);- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;+ if (compute_queue) compute_queue->cmd_pool.destroy(device);+ if (transfer_queue) transfer_queue->cmd_pool.destroy(device);- vk_buffer buffer_gpu = buf_ctx->dev_buffer;- ggml_vk_buffer_read(buffer_gpu, vk_tensor_offset(tensor) + tensor->view_offs, tensor_data, tensor_size);- }+ // Explicitly clear to ensure queues drop their shared_ptrs to handles+ // before the Vulkan logical device instance is destroyed+ compute_queue.reset();+ transfer_queue.reset();- std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl;- std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl;- if (tensor->src[0] != nullptr) {- std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl;- }- if (tensor->src[1] != nullptr) {- std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl;- }- std::cerr << std::endl << "Result:" << std::endl;- ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);- std::cerr << std::endl;- std::vector<const ggml_tensor *> done;- ggml_vk_print_graph_origin(tensor, done);+ for (auto& pipeline : all_pipelines) {+ if (pipeline.expired()) {+ continue;+ }- if (is_gpu) {- free(tensor_data);+ vk_pipeline pl = pipeline.lock();+ ggml_vk_destroy_pipeline(device, pl);}+ all_pipelines.clear();++ device.destroyDescriptorSetLayout(dsl);++ device.destroy();}-void * comp_result;-size_t comp_size;-size_t comp_nb[GGML_MAX_DIMS];-size_t check_counter = 0;-static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {- ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];- if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {+void vk_perf_logger::print_timings(bool force) {+ if (timings.empty()) {return;}-- check_counter++;- if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {+ print_count++;+ if ((print_count % vk_perf_logger_frequency) != 0 && !force) {return;}-- VK_LOG_DEBUG("ggml_vk_check_results_0(" << tensor->name << ")");-- struct ggml_init_params iparams = {- /*.mem_size =*/ 2ul*1024ul*1024ul*1024ul,- /*.mem_buffer =*/ NULL,- /*.no_alloc =*/ false,- };-- struct ggml_context * ggml_ctx = ggml_init(iparams);-- std::array<struct ggml_tensor *, GGML_MAX_SRC> src_clone = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr};- const char * srci_name[GGML_MAX_SRC] = {"src0", "src1", "src2", "src3", "src4", "src5", "src6", "src7", "src8", "src9"};-- std::map<ggml_tensor *, ggml_tensor *> cloned_tensors;- std::vector<void *> cloned_mallocs;-- struct ggml_tensor * tensor_clone = nullptr;-- for (int f = 0; f < ctx->num_additional_fused_ops + 1; ++f) {- tensor = cgraph->nodes[tensor_idx + f];- for (int i = 0; i < GGML_MAX_SRC; i++) {- ggml_tensor * srci = tensor->src[i];- if (srci == nullptr) {- continue;- }- // If a src tensor has been cloned, use that one- auto it = cloned_tensors.find(srci);- if (it != cloned_tensors.end()) {- src_clone[i] = it->second;- continue;- }- ggml_tensor * srci_clone = ggml_dup_tensor(ggml_ctx, srci);- size_t srci_size = ggml_nbytes(srci);-- src_clone[i] = srci_clone;- void *src_buffer = malloc(srci_size);- cloned_mallocs.push_back(src_buffer);-- srci_clone->data = src_buffer;- if (ggml_backend_buffer_is_host(srci->buffer)) {- memcpy(srci_clone->data, srci->data, srci_size);- memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);- } else if (ggml_backend_buffer_is_vk(srci->buffer)) {- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)srci->buffer->context;- vk_buffer& buffer_gpu = buf_ctx->dev_buffer;- uint64_t offset = vk_tensor_offset(srci) + srci->view_offs;- if (!ggml_is_contiguous(srci) && ggml_vk_dim01_contiguous(srci)) {- for (int i3 = 0; i3 < srci->ne[3]; i3++) {- for (int i2 = 0; i2 < srci->ne[2]; i2++) {- const int idx = i3*srci->ne[2] + i2;- ggml_vk_buffer_read(buffer_gpu, offset + idx * srci->nb[2], ((char *)srci_clone->data + idx * srci_clone->nb[2]), srci->ne[1] * srci->nb[1]);- }- }-- srci_clone->nb[0] = srci->nb[0];- srci_clone->nb[1] = srci->nb[1];- for (int i = 2; i < GGML_MAX_DIMS; i++) {- srci_clone->nb[i] = srci_clone->nb[i - 1]*srci_clone->ne[i - 1];- }- } else {- if (offset + srci_size >= buffer_gpu->size) {- srci_size = buffer_gpu->size - offset;- }- ggml_vk_buffer_read(buffer_gpu, offset, srci_clone->data, srci_size);- memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);- }- } else {- GGML_ABORT("fatal error");- }-- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {- ggml_vk_print_tensor(srci, srci_name[i]);- }+ print_count = 0;+ uint64_t total_all_op_times = 0;+ std::cerr << "----------------\nVulkan Timings:" << std::endl;+ for (const auto & t : timings) {+ uint64_t total_op_times = 0;+ for (const auto & time : t.second) {+ total_op_times += time;}+ std::cerr << t.first << ": " << t.second.size() << " x " << (total_op_times / t.second.size() / 1000.0)+ << " us = " << (total_op_times / 1000.0) << " us";- if (tensor->op == GGML_OP_FLASH_ATTN_EXT) {- const float * params = (const float *)tensor->op_params;- tensor_clone = ggml_flash_attn_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3], params[0], params[1], params[2]);- if (src_clone[4]) {- ggml_flash_attn_ext_add_sinks(tensor_clone, src_clone[4]);- }- } else if (tensor->op == GGML_OP_MUL_MAT) {- tensor_clone = ggml_mul_mat(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_MUL_MAT_ID) {- tensor_clone = ggml_mul_mat_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);- } else if (tensor->op == GGML_OP_SUB) {- tensor_clone = ggml_sub(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_MUL) {- tensor_clone = ggml_mul(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_DIV) {- tensor_clone = ggml_div(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_CONCAT) {- tensor_clone = ggml_concat(ggml_ctx, src_clone[0], src_clone[1], *(int *)tensor->op_params);- } else if (tensor->op == GGML_OP_UPSCALE) {- tensor_clone = ggml_interpolate(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], (ggml_scale_mode) tensor->op_params[0]);- } else if (tensor->op == GGML_OP_SCALE) {- const float * params = (const float *)tensor->op_params;- tensor_clone = ggml_scale_bias(ggml_ctx, src_clone[0], params[0], params[1]);- } else if (tensor->op == GGML_OP_ADD1) {- tensor_clone = ggml_add1(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_ARANGE) {- const float start = ggml_get_op_params_f32(tensor, 0);- const float stop = ggml_get_op_params_f32(tensor, 1);- const float step = ggml_get_op_params_f32(tensor, 2);- tensor_clone = ggml_arange(ggml_ctx, start, stop, step);- } else if (tensor->op == GGML_OP_FILL) {- const float value = ggml_get_op_params_f32(tensor, 0);- tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);- } else if (tensor->op == GGML_OP_SQR) {- tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_SQRT) {- tensor_clone = ggml_sqrt(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_SIN) {- tensor_clone = ggml_sin(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_COS) {- tensor_clone = ggml_cos(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_LOG) {- tensor_clone = ggml_log(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_TRI) {- tensor_clone = ggml_tri(ggml_ctx, src_clone[0], (ggml_tri_type)ggml_get_op_params_i32(tensor, 0));- } else if (tensor->op == GGML_OP_DIAG) {- tensor_clone = ggml_diag(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_CLAMP) {- const float * params = (const float *)tensor->op_params;- tensor_clone = ggml_clamp(ggml_ctx, src_clone[0], params[0], params[1]);- } else if (tensor->op == GGML_OP_PAD) {- tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3],- tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]);- } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) {- tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]);- } else if (tensor->op == GGML_OP_REPEAT) {- tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor);- } else if (tensor->op == GGML_OP_REPEAT_BACK) {- tensor_clone = ggml_repeat_back(ggml_ctx, src_clone[0], tensor);- } else if (tensor->op == GGML_OP_ADD) {- tensor_clone = ggml_add(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_ACC) {- tensor_clone = ggml_acc(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);- } else if (tensor->op == GGML_OP_SET) {- tensor_clone = ggml_set(ggml_ctx, src_clone[0], src_clone[1], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3]);- } else if (tensor->op == GGML_OP_NORM) {- tensor_clone = ggml_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);- } else if (tensor->op == GGML_OP_GROUP_NORM) {- const float * float_params = (const float *)tensor->op_params;- tensor_clone = ggml_group_norm(ggml_ctx, src_clone[0], tensor->op_params[0], float_params[1]);- } else if (tensor->op == GGML_OP_RMS_NORM) {- tensor_clone = ggml_rms_norm(ggml_ctx, src_clone[0], *(float *)tensor->op_params);- } else if (tensor->op == GGML_OP_RMS_NORM_BACK) {- const float eps = ((float *) tensor->op_params)[0];- tensor_clone = ggml_rms_norm_back(ggml_ctx, src_clone[0], src_clone[1], eps);- } else if (tensor->op == GGML_OP_SILU_BACK) {- tensor_clone = ggml_silu_back(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_L2_NORM) {- const float eps = ((float *) tensor->op_params)[0];- tensor_clone = ggml_l2_norm(ggml_ctx, src_clone[0], eps);- } else if (tensor->op == GGML_OP_SOFT_MAX) {- if (tensor->src[1] != nullptr) {- const float * params = (const float *)tensor->op_params;- tensor_clone = ggml_soft_max_ext(ggml_ctx, src_clone[0], src_clone[1], params[0], params[1]);- } else {- tensor_clone = ggml_soft_max(ggml_ctx, src_clone[0]);- }- } else if (tensor->op == GGML_OP_SOFT_MAX_BACK) {- tensor_clone = ggml_soft_max_ext_back(ggml_ctx, src_clone[0], src_clone[1], ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]);- } else if (tensor->op == GGML_OP_DIAG_MASK_INF) {- tensor_clone = ggml_diag_mask_inf(ggml_ctx, src_clone[0], tensor->op_params[0]);- } else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) {- const int n_dims = ((int32_t *) tensor->op_params)[1];- const int mode = ((int32_t *) tensor->op_params)[2];- //const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3];- const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4];- const float freq_base = ((float *) tensor->op_params)[5];- const float freq_scale = ((float *) tensor->op_params)[6];- const float ext_factor = ((float *) tensor->op_params)[7];- const float attn_factor = ((float *) tensor->op_params)[8];- const float beta_fast = ((float *) tensor->op_params)[9];- const float beta_slow = ((float *) tensor->op_params)[10];- if (mode & GGML_ROPE_TYPE_MROPE) {- int32_t *sections = ((int32_t *) tensor->op_params) + 11;- if (tensor->op == GGML_OP_ROPE) {- tensor_clone = ggml_rope_multi(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);- } else {- tensor_clone = ggml_rope_multi_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, sections, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);- }- } else {- if (tensor->op == GGML_OP_ROPE) {- tensor_clone = ggml_rope_ext(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);- } else {- tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);- }- }- const int n_offs = ((int32_t *) tensor->op_params)[15];- if (n_offs != 0) {- tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs);- }- } else if (tensor->op == GGML_OP_UNARY) {- switch (ggml_get_unary_op(tensor)) {- case GGML_UNARY_OP_EXP:- tensor_clone = ggml_exp(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_EXPM1:- tensor_clone = ggml_expm1(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_ELU:- tensor_clone = ggml_elu(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_SILU:- tensor_clone = ggml_silu(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_GELU:- tensor_clone = ggml_gelu(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_GELU_ERF:- tensor_clone = ggml_gelu_erf(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_GELU_QUICK:- tensor_clone = ggml_gelu_quick(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_RELU:- tensor_clone = ggml_relu(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_XIELU:- tensor_clone = ggml_xielu(ggml_ctx, src_clone[0], 0, 0, 0, 0);- ggml_set_op_params_f32(tensor_clone, 1, ggml_get_op_params_f32(tensor, 1));- ggml_set_op_params_f32(tensor_clone, 2, ggml_get_op_params_f32(tensor, 2));- ggml_set_op_params_f32(tensor_clone, 3, ggml_get_op_params_f32(tensor, 3));- ggml_set_op_params_f32(tensor_clone, 4, ggml_get_op_params_f32(tensor, 4));- break;- case GGML_UNARY_OP_NEG:- tensor_clone = ggml_neg(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_TANH:- tensor_clone = ggml_tanh(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_SIGMOID:- tensor_clone = ggml_sigmoid(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_HARDSIGMOID:- tensor_clone = ggml_hardsigmoid(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_HARDSWISH:- tensor_clone = ggml_hardswish(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_ABS:- tensor_clone = ggml_abs(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_SOFTPLUS:- tensor_clone = ggml_softplus(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_STEP:- tensor_clone = ggml_step(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_ROUND:- tensor_clone = ggml_round(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_CEIL:- tensor_clone = ggml_ceil(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_FLOOR:- tensor_clone = ggml_floor(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_TRUNC:- tensor_clone = ggml_trunc(ggml_ctx, src_clone[0]);- break;- case GGML_UNARY_OP_SGN:- tensor_clone = ggml_sgn(ggml_ctx, src_clone[0]);- break;- default:- std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;- GGML_ABORT("fatal error");- }- } else if (tensor->op == GGML_OP_GLU) {- if (src_clone[1] == nullptr) {- tensor_clone = ggml_glu(ggml_ctx, src_clone[0], (ggml_glu_op) tensor->op_params[0], tensor->op_params[1]);- } else {- tensor_clone = ggml_glu_split(ggml_ctx, src_clone[0], src_clone[1], (ggml_glu_op) tensor->op_params[0]);- }- ggml_set_op_params_i32(tensor_clone, 2, ggml_get_op_params_i32(tensor, 2));- ggml_set_op_params_i32(tensor_clone, 3, ggml_get_op_params_i32(tensor, 3));- } else if (tensor->op == GGML_OP_CPY || tensor->op == GGML_OP_DUP) {- if (tensor->src[1] == nullptr) {- tensor_clone = ggml_dup(ggml_ctx, src_clone[0]);- tensor_clone->type = tensor->type;- } else {- tensor_clone = ggml_cpy(ggml_ctx, src_clone[0], src_clone[1]);- }- } else if (tensor->op == GGML_OP_CONT) {- tensor_clone = ggml_cont_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);- } else if (tensor->op == GGML_OP_RESHAPE) {- tensor_clone = ggml_reshape_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);- } else if (tensor->op == GGML_OP_VIEW) {- tensor_clone = ggml_view_4d(ggml_ctx, src_clone[0], tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], tensor->nb[1], tensor->nb[2], tensor->nb[3], ((int32_t *) tensor->op_params)[0]);- } else if (tensor->op == GGML_OP_PERMUTE) {- int32_t * params = (int32_t *)tensor->op_params;- tensor_clone = ggml_permute(ggml_ctx, src_clone[0], params[0], params[1], params[2], params[3]);- } else if (tensor->op == GGML_OP_TRANSPOSE) {- tensor_clone = ggml_transpose(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_GET_ROWS) {- tensor_clone = ggml_get_rows(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_ARGSORT) {- tensor_clone = ggml_argsort(ggml_ctx, src_clone[0], (ggml_sort_order) *(int *)tensor->op_params);- } else if (tensor->op == GGML_OP_TOP_K) {- tensor_clone = ggml_top_k(ggml_ctx, src_clone[0], tensor->ne[0]);- } else if (tensor->op == GGML_OP_SUM) {- tensor_clone = ggml_sum(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_SUM_ROWS) {- tensor_clone = ggml_sum_rows(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_CUMSUM) {- tensor_clone = ggml_cumsum(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_DSV4_HC_COMB) {- tensor_clone = ggml_dsv4_hc_comb(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],- ggml_get_op_params_f32(tensor, 0), ggml_get_op_params_i32(tensor, 1));- } else if (tensor->op == GGML_OP_DSV4_HC_PRE) {- if (ggml_get_op_params_i32(tensor, 1) != 0) {- tensor_clone = ggml_dsv4_hc_pre_gated(ggml_ctx, src_clone[0], src_clone[1], ggml_get_op_params_f32(tensor, 0));- } else {- tensor_clone = ggml_dsv4_hc_pre(ggml_ctx, src_clone[0], src_clone[1]);+ // If we have as many flops entries as timing entries for the op, then compute and log the flops/S.+ auto it = flops.find(t.first);+ if (it != flops.end() && (it->second).size() == t.second.size()) {+ uint64_t total_op_flops = 0;+ for (const auto & elem : it->second) {+ total_op_flops += elem;}- } else if (tensor->op == GGML_OP_DSV4_HC_POST) {- tensor_clone = ggml_dsv4_hc_post(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);- } else if (tensor->op == GGML_OP_MEAN) {- tensor_clone = ggml_mean(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_ARGMAX) {- tensor_clone = ggml_argmax(ggml_ctx, src_clone[0]);- } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS) {- tensor_clone = ggml_cross_entropy_loss(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_CROSS_ENTROPY_LOSS_BACK) {- tensor_clone = ggml_cross_entropy_loss_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);- } else if (tensor->op == GGML_OP_COUNT_EQUAL) {- tensor_clone = ggml_count_equal(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_SOLVE_TRI) {- tensor_clone = ggml_solve_tri(ggml_ctx, src_clone[0], src_clone[1], true, true, false);- } else if (tensor->op == GGML_OP_IM2COL) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t p0 = tensor->op_params[2];- const int32_t p1 = tensor->op_params[3];- const int32_t d0 = tensor->op_params[4];- const int32_t d1 = tensor->op_params[5];-- const bool is_2D = tensor->op_params[6] == 1;- tensor_clone = ggml_im2col(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1, is_2D, tensor->type);- } else if (tensor->op == GGML_OP_IM2COL_3D) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t s2 = tensor->op_params[2];- const int32_t p0 = tensor->op_params[3];- const int32_t p1 = tensor->op_params[4];- const int32_t p2 = tensor->op_params[5];- const int32_t d0 = tensor->op_params[6];- const int32_t d1 = tensor->op_params[7];- const int32_t d2 = tensor->op_params[8];- const int32_t IC = tensor->op_params[9];-- tensor_clone = ggml_im2col_3d(ggml_ctx, src_clone[0], src_clone[1], IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, tensor->type);- } else if (tensor->op == GGML_OP_TIMESTEP_EMBEDDING) {- const int32_t dim = tensor->op_params[0];- const int32_t max_period = tensor->op_params[1];- tensor_clone = ggml_timestep_embedding(ggml_ctx, src_clone[0], dim, max_period);- } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_1D){- const int32_t s0 = tensor->op_params[0];- const int32_t p0 = tensor->op_params[1];- const int32_t d0 = tensor->op_params[2];- tensor_clone = ggml_conv_transpose_1d(ggml_ctx, src_clone[0], src_clone[1], s0, p0, d0);- } else if (tensor->op == GGML_OP_COL2IM_1D) {- const int32_t stride = tensor->op_params[0];- const int32_t oc = tensor->op_params[1];- const int32_t p0 = tensor->op_params[2];- tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0);- } else if (tensor->op == GGML_OP_POOL_1D) {- enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);- const int32_t k0 = tensor->op_params[1];- const int32_t s0 = tensor->op_params[2];- const int32_t p0 = tensor->op_params[3];-- tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0);- } else if (tensor->op == GGML_OP_POOL_2D) {- enum ggml_op_pool op = static_cast<ggml_op_pool>(tensor->op_params[0]);- const int32_t k0 = tensor->op_params[1];- const int32_t k1 = tensor->op_params[2];- const int32_t s0 = tensor->op_params[3];- const int32_t s1 = tensor->op_params[4];- const int32_t p0 = tensor->op_params[5];- const int32_t p1 = tensor->op_params[6];-- tensor_clone = ggml_pool_2d(ggml_ctx, src_clone[0], op, k0, k1, s0, s1, p0, p1);- } else if (tensor->op == GGML_OP_CONV_2D) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t p0 = tensor->op_params[2];- const int32_t p1 = tensor->op_params[3];- const int32_t d0 = tensor->op_params[4];- const int32_t d1 = tensor->op_params[5];- tensor_clone = ggml_conv_2d(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);- } else if (tensor->op == GGML_OP_CONV_3D) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t s2 = tensor->op_params[2];- const int32_t p0 = tensor->op_params[3];- const int32_t p1 = tensor->op_params[4];- const int32_t p2 = tensor->op_params[5];- const int32_t d0 = tensor->op_params[6];- const int32_t d1 = tensor->op_params[7];- const int32_t d2 = tensor->op_params[8];- const int32_t IC = tensor->op_params[9];- const int32_t N = tensor->op_params[10];- const int32_t OC = tensor->op_params[11];- tensor_clone = ggml_conv_3d_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, s2, p0, p1, p2, d0, d1, d2, IC, N, OC);- } else if (tensor->op == GGML_OP_CONV_2D_DW) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t p0 = tensor->op_params[2];- const int32_t p1 = tensor->op_params[3];- const int32_t d0 = tensor->op_params[4];- const int32_t d1 = tensor->op_params[5];- tensor_clone = ggml_conv_2d_dw_direct(ggml_ctx, src_clone[0], src_clone[1], s0, s1, p0, p1, d0, d1);- } else if (tensor->op == GGML_OP_CONV_TRANSPOSE_2D) {- const int32_t s = tensor->op_params[0];- tensor_clone = ggml_conv_transpose_2d_p0(ggml_ctx, src_clone[0], src_clone[1], s);- } else if (tensor->op == GGML_OP_LEAKY_RELU) {- const float * op_params = (const float *)tensor->op_params;- tensor_clone = ggml_leaky_relu(ggml_ctx, src_clone[0], op_params[0], false);- } else if (tensor->op == GGML_OP_RWKV_WKV6) {- tensor_clone = ggml_rwkv_wkv6(ggml_ctx, src_clone[0], src_clone[1],- src_clone[2], src_clone[3], src_clone[4], src_clone[5]);- } else if (tensor->op == GGML_OP_RWKV_WKV7) {- tensor_clone = ggml_rwkv_wkv7(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3],- src_clone[4], src_clone[5], src_clone[6]);- } else if (tensor->op == GGML_OP_GATED_LINEAR_ATTN) {- const float * op_params = (const float *)tensor->op_params;- tensor_clone = ggml_gated_linear_attn(ggml_ctx, src_clone[0], src_clone[1],- src_clone[2], src_clone[3], src_clone[4], op_params[0]);- } else if (tensor->op == GGML_OP_LIGHTNING_INDEXER) {- tensor_clone = ggml_lightning_indexer(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], src_clone[3]);- } else if (tensor->op == GGML_OP_GATED_DELTA_NET) {- tensor_clone = ggml_gated_delta_net(ggml_ctx, src_clone[0], src_clone[1],- src_clone[2], src_clone[3], src_clone[4], src_clone[5],- ggml_get_op_params_i32(tensor, 0));- } else if (tensor->op == GGML_OP_OPT_STEP_ADAMW) {- src_clone[0]->flags = tensor->src[0]->flags;- tensor_clone = ggml_opt_step_adamw(ggml_ctx, src_clone[0], src_clone[1],- src_clone[2], src_clone[3], src_clone[4]);- } else if (tensor->op == GGML_OP_OPT_STEP_SGD) {- src_clone[0]->flags = tensor->src[0]->flags;- tensor_clone = ggml_opt_step_sgd(ggml_ctx, src_clone[0], src_clone[1],- src_clone[2]);- } else if (tensor->op == GGML_OP_ADD_ID) {- tensor_clone = ggml_add_id(ggml_ctx, src_clone[0], src_clone[1], src_clone[2]);- } else if (tensor->op == GGML_OP_SSM_SCAN) {- const int32_t K = ggml_get_op_params_i32(tensor, 0);- tensor_clone = ggml_ssm_scan(ggml_ctx, src_clone[0], src_clone[1], src_clone[2],- src_clone[3], src_clone[4], src_clone[5], src_clone[6], K);- } else if (tensor->op == GGML_OP_SSM_CONV) {- tensor_clone = ggml_ssm_conv(ggml_ctx, src_clone[0], src_clone[1]);- } else if (tensor->op == GGML_OP_ROLL) {- const int32_t s0 = tensor->op_params[0];- const int32_t s1 = tensor->op_params[1];- const int32_t s2 = tensor->op_params[2];- const int32_t s3 = tensor->op_params[3];- tensor_clone = ggml_roll(ggml_ctx, src_clone[0], s0, s1, s2, s3);- }- else {- std::cerr << "Missing vk_check_results OP: " << ggml_op_name(tensor->op) << std::endl;- GGML_ABORT("fatal error");+ std::cerr << " ("+ << (double(total_op_flops) / (1000.0 * 1000.0 * 1000.0)) /+ (double(total_op_times) / (1000.0 * 1000.0 * 1000.0))+ << " GFLOPS/s)";}- cloned_tensors[tensor] = tensor_clone;- }-- ggml_cgraph * cgraph_cpu = ggml_new_graph(ggml_ctx);- ggml_build_forward_expand(cgraph_cpu, tensor_clone);- ggml_graph_compute_with_ctx(ggml_ctx, cgraph_cpu, 8);+ total_all_op_times += total_op_times;- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {- ggml_vk_print_tensor(tensor_clone, "tensor_clone");+ std::cerr << std::endl;}- comp_size = ggml_nbytes(tensor_clone);-- comp_result = malloc(comp_size);- memcpy(comp_result, tensor_clone->data, comp_size);- memcpy(comp_nb, tensor_clone->nb, sizeof(size_t) * GGML_MAX_DIMS);-- for (auto m : cloned_mallocs) {- free(m);+ if (timings.size() > 0) {+ std::cerr << "Total time: " << total_all_op_times / 1000.0 << " us." << std::endl;}- ggml_free(ggml_ctx);-- VK_LOG_DEBUG("END ggml_vk_check_results_0(" << tensor->name << ")");+ timings.clear();+ flops.clear();}-static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, int tensor_idx) {- ggml_tensor * tensor = cgraph->nodes[tensor_idx + ctx->num_additional_fused_ops];- if (tensor->op == GGML_OP_TRANSPOSE || tensor->op == GGML_OP_SET_ROWS) {+std::string vk_perf_logger::get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) {+ *n_flops = ggml_vk_get_node_flops(node);+ std::string fusion_str;+ if (fusion_name) {+ fusion_str = fusion_name + std::string(" ");+ }+ if (node->op == GGML_OP_UNARY) {+ return fusion_str + ggml_unary_op_name(ggml_get_unary_op(node));+ }+ if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {+ const uint64_t m = node->ne[0];+ const uint64_t n = node->ne[1];+ const uint64_t k = node->src[1]->ne[0];+ const uint64_t batch = node->ne[2] * node->ne[3];+ std::string name = ggml_op_name(node->op);+ if ((node->op == GGML_OP_MUL_MAT && n <= mul_mat_vec_max_cols) ||+ (node->op == GGML_OP_MUL_MAT_ID && node->src[2]->ne[1] == 1)) {+ name += "_VEC";+ }+ name += " ";+ name += ggml_type_name(node->src[0]->type);+ name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k);+ if (node->op == GGML_OP_MUL_MAT_ID) {+ name += " n_expert=" + std::to_string(node->src[0]->ne[2]);+ }+ if (batch > 1) {+ name += " batch=" + std::to_string(batch);+ }+ return fusion_str + name;+ }+ if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) {+ std::string name = ggml_op_name(node->op);+ const ggml_tensor * knl = node->src[0];+ uint64_t Cout = node->ne[2];+ uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1];+ uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1];+ name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) ++ ", N=N*OW*OH=" + std::to_string(size_N);+ return fusion_str + name;+ }+ if (node->op == GGML_OP_RMS_NORM) {+ std::string name = ggml_op_name(node->op);+ name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")";+ return fusion_str + name;+ }+ if (node->op == GGML_OP_FLASH_ATTN_EXT) {+ const ggml_tensor * dst = node;+ const ggml_tensor * q = node->src[0];+ const ggml_tensor * k = node->src[1];+ const ggml_tensor * v = node->src[2];+ const ggml_tensor * m = node->src[3];+ std::stringstream name;+ name << fusion_str;+ name << ggml_op_name(node->op) <<+ " dst(" << dst->ne[0] << "," << dst->ne[1] << "," << dst->ne[2] << "," << dst->ne[3] << "), " <<+ " q(" << q->ne[0] << "," << q->ne[1] << "," << q->ne[2] << "," << q->ne[3] << "), " <<+ " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " <<+ " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " <<+ " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")";+ return name.str();+ }+ if (node->op == GGML_OP_TOP_K) {+ std::stringstream name;+ name << fusion_str;+ name << ggml_op_name(node->op) <<+ " K=" << node->ne[0] <<+ " (" << node->src[0]->ne[0] << "," << node->src[0]->ne[1] << "," << node->src[0]->ne[2] << "," << node->src[0]->ne[3] << ")";+ return name.str();+ }+ return fusion_str + ggml_op_name(node->op);+}++ggml_backend_vk_buffer_context::~ggml_backend_vk_buffer_context() {+ ggml_vk_destroy_buffer(dev_buffer);+}++ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const std::string & pipeline_name, uint32_t wg0, uint32_t wg1, uint32_t wg2) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {return;}+ begin(ctx, pipeline_name + " (" + std::to_string(wg0) + "," + std::to_string(wg1) + "," + std::to_string(wg2) + ")");+}- if (!(vk_output_tensor > 0 && vk_output_tensor == check_counter) && check_counter <= vk_skip_checks) {+ggml_vk_debug_label::ggml_vk_debug_label(vk_context & ctx, const ggml_cgraph * cgraph, int node_idx, int n_fused) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {return;}+ std::string name = ggml_op_name(cgraph->nodes[node_idx]->op);+ for (int i = 1; i <= n_fused; i++) {+ name += "+";+ name += ggml_op_name(cgraph->nodes[node_idx + i]->op);+ }+ name += " ";+ name += cgraph->nodes[node_idx]->name;+ begin(ctx, name);+}- VK_LOG_DEBUG("ggml_vk_check_results_1(" << tensor->name << ")");-- ggml_tensor * src0 = tensor->src[0];- ggml_tensor * src1 = tensor->src[1];- ggml_tensor * src2 = tensor->src[2];- ggml_tensor * src3 = tensor->src[3];-- void * tensor_data = tensor->data;-- if (ggml_backend_buffer_is_vk(tensor->buffer)) {- size_t tensor_size = ggml_nbytes(tensor);- tensor_data = malloc(tensor_size);-- ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)tensor->buffer->context;-- vk_buffer& buffer_gpu = buf_ctx->dev_buffer;- uint64_t offset = vk_tensor_offset(tensor) + tensor->view_offs;- if (offset + tensor_size >= buffer_gpu->size) {- tensor_size = buffer_gpu->size - offset;- }-- ggml_vk_buffer_read(buffer_gpu, offset, tensor_data, tensor_size);- }-- float first_error_result = -1.0f;- float first_error_correct = -1.0f;- std::array<int, 4> first_error = { -1, -1, -1, -1 };- double avg_err = 0.0;- size_t counter = 0;-- for (int i3 = 0; i3 < tensor->ne[3]; i3++) {- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- const bool buffer_size_fit = i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0] < comp_size;- float correct = 0.0f;- float result = 0.0f;-- if (buffer_size_fit) {- if (tensor->type == GGML_TYPE_F32) {- correct = *(float *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);- result = *(float *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);- } else if (tensor->type == GGML_TYPE_F16) {- correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));- result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));- } else if (tensor->type == GGML_TYPE_BF16) {- correct = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]));- result = ggml_bf16_to_fp32(*(ggml_bf16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]));- } else if (tensor->type == GGML_TYPE_I32) {- correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);- result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);- } else if (tensor->type == GGML_TYPE_I64) {- correct = *(int64_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]);- result = *(int64_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]);- } else {- std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl;- }- } else {- std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl;- GGML_ABORT("fatal error");- }-- if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) {- std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl;- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;- if (src0 != nullptr) {- std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;- }- if (src1 != nullptr) {- std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;- }- if (src2 != nullptr) {- std::cerr << "src2=" << src2 << " src2->name=" << src2->name << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;- }- if (src3 != nullptr) {- std::cerr << "src3=" << src3 << " src3->name=" << src3->name << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;- }- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;- std::cerr << std::endl << "Result:" << std::endl;- ggml_vk_print_tensor_area(tensor, tensor_data, i0, i1, i2, i3);- std::cerr << std::endl << "Correct:" << std::endl;- ggml_vk_print_tensor_area(tensor, comp_result, i0, i1, i2, i3);- std::cerr << std::endl;- std::vector<const ggml_tensor *> done;- ggml_vk_print_graph_origin(tensor, done);- GGML_ABORT("fatal error");- }- const double denom = std::fabs(correct) > 1.0f ? (std::fabs(correct) > 1e-8 ? std::fabs(correct) : 1e-8) : 1.0f;- if (first_error[0] == -1 && std::fabs(correct - result) / denom > 0.5) {- first_error[0] = i0;- first_error[1] = i1;- first_error[2] = i2;- first_error[3] = i3;- first_error_result = result;- first_error_correct = correct;- }-- // Special case, value is infinite, avoid NaN result in avg_err- // NaN also appears in results, if both are nan error is 0- if (!std::isinf(correct) && !std::isinf(result) && !std::isnan(correct) && !std::isnan(result)) {- avg_err += std::fabs(correct - result) / denom;- }- counter++;- }- }- }+ggml_vk_debug_label::ggml_vk_debug_label(vk_queue_handle * handle, const char * name) {+ if (!vk_instance.debug_utils_support || handle == nullptr) {+ return;}+ vk::DebugUtilsLabelEXT label = {};+ label.pLabelName = name;+ label.color = std::array<float, 4>{1.0f, 1.0f, 1.0f, 1.0f};- avg_err /= counter;+ qhandle = handle;+ std::lock_guard<vk_queue_handle> guard(*qhandle);+ vk_instance.pfn_vkQueueBeginDebugUtilsLabelEXT(qhandle->queue, reinterpret_cast<VkDebugUtilsLabelEXT *>(&label));+}- if (vk_output_tensor > 0 && vk_output_tensor == check_counter) {- std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;- if (src0 != nullptr) {- std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;- }- if (src1 != nullptr) {- std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;- }- if (src2 != nullptr) {- std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;+void ggml_vk_debug_label::close() {+ if (subctx != nullptr) {+ // close on the current command buffer, which may differ from the one begin used+ if (subctx->s != nullptr) {+ vk_instance.pfn_vkCmdEndDebugUtilsLabelEXT(subctx->s->buffer->buf);}- if (src3 != nullptr) {- std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;- }- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;- std::cerr << std::endl << "Result:" << std::endl;- ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0);- std::cerr << std::endl << "Correct:" << std::endl;- ggml_vk_print_tensor_area(tensor, comp_result, 5, 5, 0, 0);- std::cerr << std::endl;- std::vector<const ggml_tensor *> done;- ggml_vk_print_graph_origin(tensor, done);+ subctx->debug_labels.pop_back();+ subctx = nullptr;}-- if (avg_err > 0.01 || std::isnan(avg_err)) {- std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl;- std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl;- if (src0 != nullptr) {- std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl;- }- if (src1 != nullptr) {- std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl;- }- if (src2 != nullptr) {- std::cerr << "src2=" << src2 << " op=" << ggml_op_name(src2->op) << " type=" << ggml_type_name(src2->type) << " ne0=" << src2->ne[0] << " nb0=" << src2->nb[0] << " ne1=" << src2->ne[1] << " nb1=" << src2->nb[1] << " ne2=" << src2->ne[2] << " nb2=" << src2->nb[2] << " ne3=" << src2->ne[3] << " nb3=" << src2->nb[3] << " offset=" << src2->view_offs << std::endl;- }- if (src3 != nullptr) {- std::cerr << "src3=" << src3 << " op=" << ggml_op_name(src3->op) << " type=" << ggml_type_name(src3->type) << " ne0=" << src3->ne[0] << " nb0=" << src3->nb[0] << " ne1=" << src3->ne[1] << " nb1=" << src3->nb[1] << " ne2=" << src3->ne[2] << " nb2=" << src3->nb[2] << " ne3=" << src3->ne[3] << " nb3=" << src3->nb[3] << " offset=" << src3->view_offs << std::endl;- }- std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl;- std::cerr << std::endl << "Result:" << std::endl;- ggml_vk_print_tensor_area(tensor, tensor_data, first_error[0], first_error[1], first_error[2], first_error[3]);- std::cerr << std::endl << "Correct:" << std::endl;- ggml_vk_print_tensor_area(tensor, comp_result, first_error[0], first_error[1], first_error[2], first_error[3]);- std::cerr << std::endl;- std::vector<const ggml_tensor *> done;- ggml_vk_print_graph_origin(tensor, done);- GGML_ABORT("fatal error");- } else {- std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl;+ if (qhandle != nullptr) {+ std::lock_guard<vk_queue_handle> guard(*qhandle);+ vk_instance.pfn_vkQueueEndDebugUtilsLabelEXT(qhandle->queue);+ qhandle = nullptr;}+}- free(comp_result);- comp_result = nullptr;- comp_size = 0;-- if (ggml_backend_buffer_is_vk(tensor->buffer)) {- free(tensor_data);+void ggml_vk_debug_label::begin(vk_context & ctx, const std::string & name) {+ if (!vk_instance.debug_utils_support || ctx->s == nullptr) {+ return;}-- VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")");+ subctx = ctx.get();+ subctx->debug_labels.push_back(name);+ ggml_vk_cmd_label_begin(subctx->s->buffer->buf, subctx->debug_labels.back().c_str());}-#endif-GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg)