LLM inference in C/C++
C++ ggml
vulkan : fix concurrent first device init (#29766)
losewayy47f5a64ae128
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 4205711407..436bfe3bb0 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -3990,10 +3990,12 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {vk_device ggml_vk_get_device(size_t idx) {VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");+ static std::mutex mutex;+ std::lock_guard<std::mutex> lock(mutex);+if (vk_instance.devices[idx] == nullptr) {VK_LOG_DEBUG("Initializing new vk_device");vk_device device = std::make_shared<vk_device_struct>();- vk_instance.devices[idx] = device;device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger());@@ -4891,14 +4893,14 @@ vk_device ggml_vk_get_device(size_t idx) {device->async_use_transfer_queue = false;}+ device->fence = device->device.createFence({});+device->buffer_type = {/* .iface = */ ggml_backend_vk_buffer_type_interface,/* .device = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx),/* .context = */ new ggml_backend_vk_buffer_type_context{ device->name, device },};- device->fence = device->device.createFence({});-device->idx = idx;device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;@@ -4920,6 +4922,9 @@ vk_device ggml_vk_get_device(size_t idx) {device->mmvq_mode = 1;}+ // publish the device only when it is fully built+ vk_instance.devices[idx] = device;+return device;}@@ -16463,6 +16468,11 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo>}vk_device_struct::~vk_device_struct() {+ // the build failed before the device was created, nothing to destroy+ if (device == VK_NULL_HANDLE) {+ return;+ }+VK_LOG_DEBUG("destroy device " << name);device.destroyFence(fence);diff --git a/tests/test-thread-safety.cpp b/tests/test-thread-safety.cppindex 4fbb1a2700..0e735d684d 100644--- a/tests/test-thread-safety.cpp+++ b/tests/test-thread-safety.cpp@@ -1,5 +1,5 @@// thread safety test-// - Loads a copy of the same model on each GPU, plus a copy on the CPU+// - Loads a copy of the same model on each GPU, plus a copy on the CPU, all in parallel// - Creates n_parallel (--parallel) contexts per model// - Runs inference in parallel on each context@@ -51,30 +51,43 @@ int main(int argc, char ** argv) {//const int num_models = std::max(1, gpu_dev_count);const int num_contexts = std::max(1, params.n_parallel);- std::vector<llama_model_ptr> models;+ std::vector<llama_model_ptr> models(num_models);+ std::vector<std::thread> load_threads;std::vector<std::thread> threads;std::atomic<bool> failed = false;for (int m = 0; m < num_models; ++m) {- auto mparams = common_model_params_to_llama(params);-- if (m < gpu_dev_count) {- mparams.split_mode = LLAMA_SPLIT_MODE_NONE;- mparams.devices = gpus[m].data();- } else if (m == gpu_dev_count) {- mparams.split_mode = LLAMA_SPLIT_MODE_NONE;- mparams.main_gpu = -1; // CPU model- } else {- mparams.split_mode = LLAMA_SPLIT_MODE_LAYER;- }+ load_threads.emplace_back([&, m]() {+ auto mparams = common_model_params_to_llama(params);++ if (m < gpu_dev_count) {+ mparams.split_mode = LLAMA_SPLIT_MODE_NONE;+ mparams.devices = gpus[m].data();+ } else if (m == gpu_dev_count) {+ mparams.split_mode = LLAMA_SPLIT_MODE_NONE;+ mparams.main_gpu = -1; // CPU model+ } else {+ mparams.split_mode = LLAMA_SPLIT_MODE_LAYER;+ }++ llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);+ if (model == NULL) {+ LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());+ failed.store(true);+ return;+ }++ models[m].reset(model);+ });+ }- llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);- if (model == NULL) {- LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());- return 1;- }+ for (auto & thread : load_threads) {+ thread.join();+ }- models.emplace_back(model);+ if (failed) {+ LOG_ERR("One or more models failed to load.\n");+ return 1;}for (int m = 0; m < num_models; ++m) {