LLM inference in C/C++
C++ ggml
mtmd: add --mmproj-device argument (#23255)
John-Henry Lim681c29d36a13
diff --git a/common/arg.cpp b/common/arg.cppindex 6f5fe377d5..0a479c6aaa 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.mmproj_use_gpu = value;}).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD"));+ add_opt(common_arg(+ // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet+ {"-mmdev", "--mmproj-device"}, "DEVICE",+ "device to use for multimodal projector (none = don't offload, default: auto)\n"+ "use --list-devices to see a list of available devices",+ [](common_params & params, const std::string & value) {+ if (value == "none") {+ params.mmproj_use_gpu = false;+ params.mmproj_device = nullptr;+ return;+ }+ auto devices = parse_device_list(value);+ // parse_device_list pushes nullptr at back so devices is length 2 for single device.+ if (devices.size() > 2) {+ throw std::invalid_argument("only one device may be specified for mmproj");+ }+ params.mmproj_use_gpu = true;+ params.mmproj_device = devices.front();+ }+ ).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reasonadd_opt(common_arg({"--image", "--audio", "--video"}, "FILE","path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n",diff --git a/common/common.h b/common/common.hindex d8a16897b8..de49dac9f6 100644--- a/common/common.h+++ b/common/common.h@@ -581,9 +581,10 @@ struct common_params {// multimodal models (see tools/mtmd)struct common_params_model mmproj;- bool mmproj_use_gpu = true; // use GPU for multimodal model- bool no_mmproj = false; // explicitly disable multimodal model- std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"+ bool mmproj_use_gpu = true; // use GPU for multimodal model+ ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model+ bool no_mmproj = false; // explicitly disable multimodal model+ std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"int image_min_tokens = -1;int image_max_tokens = -1;int mtmd_batch_max_tokens = 1024;diff --git a/tools/cli/README.md b/tools/cli/README.mdindex b3543ed4da..c9cbacafcd 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -162,6 +162,7 @@| `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) || `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) || `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) |+| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) || `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files || `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) || `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex b9dd5e8452..45e33042dd 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -186,14 +186,13 @@ struct clip_ctx {throw std::runtime_error("failed to initialize CPU backend");}if (ctx_params.use_gpu) {- auto * backend_name = std::getenv("MTMD_BACKEND_DEVICE");- if (backend_name != nullptr) {- backend = ggml_backend_init_by_name(backend_name, nullptr);+ if (ctx_params.device != nullptr) {+ backend = ggml_backend_dev_init(ctx_params.device, nullptr);if (!backend) {- LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name);+ throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",+ __func__, ggml_backend_dev_name(ctx_params.device)));}- }- if (!backend) {+ } else {backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);}diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.hindex a5b7137752..e07f258156 100644--- a/tools/mtmd/clip.h+++ b/tools/mtmd/clip.h@@ -48,6 +48,7 @@ enum clip_flash_attn_type {struct clip_context_params {bool use_gpu;+ ggml_backend_dev_t device;enum clip_flash_attn_type flash_attn_type;int image_min_tokens;int image_max_tokens;diff --git a/tools/mtmd/debug/mtmd-debug.cpp b/tools/mtmd/debug/mtmd-debug.cppindex b88a16f0f8..2719dae9b2 100644--- a/tools/mtmd/debug/mtmd-debug.cpp+++ b/tools/mtmd/debug/mtmd-debug.cpp@@ -84,6 +84,7 @@ int main(int argc, char ** argv) {const char * clip_path = params.mmproj.path.c_str();mtmd_context_params mparams = mtmd_context_params_default();mparams.use_gpu = params.mmproj_use_gpu;+ mparams.device = params.mmproj_device;mparams.print_timings = true;mparams.n_threads = params.cpuparams.n_threads;mparams.flash_attn_type = params.flash_attn_type;diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cppindex 07b45b6440..f6c787fdb6 100644--- a/tools/mtmd/mtmd-cli.cpp+++ b/tools/mtmd/mtmd-cli.cpp@@ -154,6 +154,7 @@ struct mtmd_cli_context {const char * clip_path = params.mmproj.path.c_str();mtmd_context_params mparams = mtmd_context_params_default();mparams.use_gpu = params.mmproj_use_gpu;+ mparams.device = params.mmproj_device;mparams.print_timings = true;mparams.n_threads = params.cpuparams.n_threads;mparams.flash_attn_type = params.flash_attn_type;diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cppindex 0f5cb8c7ae..95f17f7af8 100644--- a/tools/mtmd/mtmd.cpp+++ b/tools/mtmd/mtmd.cpp@@ -456,6 +456,7 @@ static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_mtmd_context_params mtmd_context_params_default() {mtmd_context_params params {/* use_gpu */ true,+ /* device */ nullptr,/* print_timings */ true,/* n_threads */ 4,/* image_marker */ nullptr,@@ -564,6 +565,7 @@ struct mtmd_context {clip_context_params ctx_clip_params {/* use_gpu */ ctx_params.use_gpu,+ /* device */ ctx_params.device,/* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),/* image_min_tokens */ ctx_params.image_min_tokens,/* image_max_tokens */ ctx_params.image_max_tokens,diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.hindex ef4f99c0b6..ef88efd316 100644--- a/tools/mtmd/mtmd.h+++ b/tools/mtmd/mtmd.h@@ -89,6 +89,7 @@ typedef bool (*mtmd_progress_callback)(float progress, void * user_data);struct mtmd_context_params {bool use_gpu;+ ggml_backend_dev_t device;bool print_timings;int n_threads;const char * image_marker; // deprecated, use media_marker insteaddiff --git a/tools/server/README.md b/tools/server/README.mdindex f5d747eee8..93736c3edf 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -178,6 +178,7 @@ For the full list of features, please refer to [server's changelog](https://gith| `-mmu, --mmproj-url URL` | URL to a multimodal projector file. see tools/mtmd/README.md<br/>(env: LLAMA_ARG_MMPROJ_URL) || `--mmproj-auto, --no-mmproj, --no-mmproj-auto` | whether to use multimodal projector file (if available), useful when using -hf (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_AUTO) || `--mmproj-offload, --no-mmproj-offload` | whether to enable GPU offloading for multimodal projector (default: enabled)<br/>(env: LLAMA_ARG_MMPROJ_OFFLOAD) |+| `-mmdev, --mmproj-device DEVICE` | device to use for multimodal projector (none = don't offload, default: auto)<br/>use --list-devices to see a list of available devices<br/>(env: MTMD_BACKEND_DEVICE) || `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) || `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) || `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) |@@ -196,11 +197,11 @@ For the full list of features, please refer to [server's changelog](https://gith| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) || `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) || `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable server tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) || `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/> 'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/> 'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/> 'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) || `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) || `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |-| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all server tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) |+| `-ag, --agent, -no-ag, --no-agent` | whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_AGENT) || `--ui, --webui, --no-ui, --no-webui` | whether to enable the Web UI (default: enabled)<br/>(env: LLAMA_ARG_UI) || `--embedding, --embeddings` | restrict to only support embedding use case; use only with dedicated embedding models (default: disabled)<br/>(env: LLAMA_ARG_EMBEDDINGS) || `--rerank, --reranking` | enable reranking endpoint on server (default: disabled)<br/>(env: LLAMA_ARG_RERANKING) |diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 21ff783941..1293c86402 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -998,6 +998,7 @@ private:mtmd_context_params mparams = mtmd_context_params_default();if (has_mmproj) {mparams.use_gpu = params_base.mmproj_use_gpu;+ mparams.device = params_base.mmproj_device;mparams.print_timings = false;mparams.n_threads = params_base.cpuparams.n_threads;mparams.flash_attn_type = params_base.flash_attn_type;diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cppindex fd7522f8de..368123baf5 100644--- a/tools/tts/tts.cpp+++ b/tools/tts/tts.cpp@@ -86,6 +86,7 @@ int main(int argc, char ** argv) {mtmd_context_params mtmd_params = mtmd_context_params_default();mtmd_params.use_gpu = params.mmproj_use_gpu;+ mtmd_params.device = params.mmproj_device;mtmd::context_ptr mctx(mtmd_init_from_file(params.mmproj.path.c_str(), model, mtmd_params));if (!mctx) {LOG_ERR("failed to load mmproj %s\n", params.mmproj.path.c_str());