LLM inference in C/C++
C++ ggml
common : add common_print_available_devices() (#26170)
Adrien Gallouët0e4a03622397
diff --git a/common/arg.cpp b/common/arg.cppindex 3bc9574d81..79480e06f9 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -1061,6 +1061,31 @@ static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & valreturn devices;}+void common_print_available_devices() {+ constexpr size_t MiB = 1024 * 1024;+ std::vector<ggml_backend_dev_t> devices;++ ggml_backend_load_all();++ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {+ auto * dev = ggml_backend_dev_get(i);+ if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {+ devices.push_back(dev);+ }+ }+ printf("Available devices:\n");++ if (devices.empty()) {+ printf(" (none)\n");+ return;+ }+ for (auto * dev : devices) {+ size_t free, total;+ ggml_backend_dev_memory(dev, &free, &total);+ printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / MiB, free / MiB);+ }+}+static void add_rpc_devices(const std::string & servers) {auto rpc_servers = string_split<std::string>(servers, ',');if (rpc_servers.empty()) {@@ -2588,20 +2613,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex{"--list-devices"},"print list of available devices and exit",[](common_params &) {- ggml_backend_load_all();- std::vector<ggml_backend_dev_t> devices;- for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {- auto * dev = ggml_backend_dev_get(i);- if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {- devices.push_back(dev);- }- }- printf("Available devices:\n");- for (auto * dev : devices) {- size_t free, total;- ggml_backend_dev_memory(dev, &free, &total);- printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);- }+ common_print_available_devices();exit(0);}));diff --git a/common/arg.h b/common/arg.hindex 54a38b9cce..8f609e356f 100644--- a/common/arg.h+++ b/common/arg.h@@ -123,6 +123,9 @@ struct common_params_context {// if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);+// load all backends and print the list of available (non-CPU) devices to stdout+void common_print_available_devices();+// parse input arguments from CLI into a mapbool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<common_arg, std::string> & out_map);diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex dc1c7caf4a..c17a27b540 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -670,22 +670,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {break;}} else if (arg == "--list-devices") {- std::vector<ggml_backend_dev_t> devices;- for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {- auto * dev = ggml_backend_dev_get(i);- if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {- devices.push_back(dev);- }- }- printf("Available devices:\n");- if (devices.empty()) {- printf(" (none)\n");- }- for (auto * dev : devices) {- size_t free, total;- ggml_backend_dev_memory(dev, &free, &total);- printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);- }+ common_print_available_devices();exit(0);} else if (arg == "-t" || arg == "--threads") {if (++i >= argc) {