LLM inference in C/C++
C++ ggml
args: add --video-* CLI arguments (#24318)
Xuan-Son Nguyenf29551215b71
diff --git a/common/arg.cpp b/common/arg.cppindex aad8266ed7..dd92e4720b 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2644,6 +2644,27 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.mtmd_batch_max_tokens = value;}).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));+ add_opt(common_arg(+ {"--video-fps"}, "N",+ string_format("target video frame rate (default: %.1f)", params.video_fps),+ [](common_params & params, const std::string & value) {+ params.video_fps = std::stof(value);+ }+ ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FPS"));+ add_opt(common_arg(+ {"--video-timestamp-interval"}, "N",+ string_format("interval in milliseconds between text timestamps (default: %" PRId64 ")", params.video_timestamp_interval_ms),+ [](common_params & params, int value) {+ params.video_timestamp_interval_ms = value;+ }+ ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"));+ add_opt(common_arg(+ {"--video-ffmpeg-dir"}, "DIR",+ "path to the directory containing ffmpeg and ffprobe (default: search in PATH)",+ [](common_params & params, const std::string & value) {+ params.video_ffmpeg_bin_dir = value;+ }+ ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));if (params.is_gen_docs || llama_supports_rpc()) {add_opt(common_arg({"--rpc"}, "SERVERS",diff --git a/common/common.h b/common/common.hindex 51518f343a..9593e10d9a 100644--- a/common/common.h+++ b/common/common.h@@ -590,6 +590,11 @@ struct common_params {int image_max_tokens = -1;int mtmd_batch_max_tokens = 1024;+ // for video input+ float video_fps = 4.0f;+ int64_t video_timestamp_interval_ms = 5000;+ std::string video_ffmpeg_bin_dir = "";+// finetunestruct lr_opt lr;enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;diff --git a/tools/cli/README.md b/tools/cli/README.mdindex c9cbacafcd..0fba70a90e 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -166,6 +166,9 @@| `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files || `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) || `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |+| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |+| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |+| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) || `-o, --output, --output-file FNAME` | output file (default: '') || `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) || `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cppindex f6c787fdb6..97678c6b2e 100644--- a/tools/mtmd/mtmd-cli.cpp+++ b/tools/mtmd/mtmd-cli.cpp@@ -87,6 +87,9 @@ struct mtmd_cli_context {mtmd::bitmaps bitmaps;std::vector<mtmd_helper::video_ptr> videos;+ mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();+ std::string video_ffmpeg_bin_dir;+mtmd::batch_ptr mbatch;// chat template@@ -170,6 +173,12 @@ struct mtmd_cli_context {LOG_ERR("Failed to load vision model from %s\n", clip_path);exit(1);}++ video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;+ init_opt.video_params.fps_target = params.video_fps;+ init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;+ init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()+ ? nullptr : video_ffmpeg_bin_dir.c_str();}bool check_antiprompt(const llama_tokens & generated_tokens) {@@ -184,7 +193,7 @@ struct mtmd_cli_context {}bool load_media(const std::string & fname) {- auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false);+ auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);if (!res.bitmap) {return false;}diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cppindex f1defb6477..77f9d58fe2 100644--- a/tools/mtmd/mtmd-helper.cpp+++ b/tools/mtmd/mtmd-helper.cpp@@ -369,14 +369,18 @@ static bool is_webp_file(const unsigned char * buf, size_t len) {}#ifdef MTMD_VIDEO-static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,+ const mtmd_helper_video_init_params & params);#endif-mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {+mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,+ mtmd_helper_init_opt opt) {// calculate the hash if neededstd::string id;mtmd_bitmap * result = nullptr;+ GGML_UNUSED(opt); // only used by video code paths+if (!placeholder) {// use sha256 to prevent cache poisoningid = hash_sha256_hex(buf, len);@@ -414,7 +418,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,#ifdef MTMD_VIDEO// stb_image does not support webp; decode it with ffmpeg as a single frameif (!result && is_webp_file(buf, len)) {- result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);+ result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params);if (!result) {LOG_ERR("%s: failed to decode webp buffer\n", __func__);return {nullptr, nullptr};@@ -427,8 +431,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,// last try: load as video#ifdef MTMD_VIDEOif (!result) {- auto params = mtmd_helper_video_init_params_default();- auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, params);+ auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params);if (!video_ctx) {LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__);return {nullptr, nullptr};@@ -456,7 +459,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,return {nullptr, nullptr};}-mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {+mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder,+ mtmd_helper_init_opt opt) {#ifdef _WIN32int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);if (!wlen) {@@ -497,7 +501,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx,return {nullptr, nullptr};}- return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder);+ return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);}bool mtmd_helper_support_video(mtmd_context * ctx) {@@ -855,6 +859,12 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {};}+mtmd_helper_init_opt mtmd_helper_init_opt_default() {+ return {+ /* video_params */ mtmd_helper_video_init_params_default(),+ };+}+static std::string video_resolve_bin(const char * bin_dir, const char * name) {if (!bin_dir || bin_dir[0] == '\0') {return name; // rely on PATH@@ -876,8 +886,8 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {}#ifdef MTMD_VIDEO-static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {- auto params = mtmd_helper_video_init_params_default();+static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,+ const mtmd_helper_video_init_params & params) {mtmd_helper_video vctx;vctx.mctx = mctx;vctx.input_buf.assign(buf, buf + len);diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.hindex 58dfb15250..772e0f091b 100644--- a/tools/mtmd/mtmd-helper.h+++ b/tools/mtmd/mtmd-helper.h@@ -23,6 +23,23 @@ extern "C" {struct mtmd_helper_video;typedef struct mtmd_helper_video mtmd_helper_video;+struct mtmd_helper_video_init_params {+ float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f+ const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH+ int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms+ // TODO @ngxson : allow "placeholder" bitmap output for counting tokens+};++MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);++// opt for mtmd_helper_bitmap_init_from_*()+struct mtmd_helper_init_opt {+ struct mtmd_helper_video_init_params video_params;+};+typedef struct mtmd_helper_init_opt mtmd_helper_init_opt;++MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);+// Set callback for all future logging events.// If this is not called, or NULL is supplied, everything is output on stderr.// Note: this also call mtmd_log_set() internally@@ -40,7 +57,11 @@ struct mtmd_helper_bitmap_wrapper {// it calls mtmd_helper_bitmap_init_from_buf() internally// returns nullptr on failure// this function is thread-safe-MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder);+MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(+ mtmd_context * ctx,+ const char * fname,+ bool placeholder,+ struct mtmd_helper_init_opt opt);// helper function to construct a mtmd_bitmap from a buffer containing a file// supported formats:@@ -53,7 +74,11 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm// - output bitmap will have SHA-256 hash (hex string) as the ID// returns nullptr on failure// this function is thread-safe-MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);+MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(+ mtmd_context * ctx,+ const unsigned char * buf, size_t len,+ bool placeholder,+ struct mtmd_helper_init_opt opt);// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cacheMTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);@@ -124,14 +149,7 @@ struct mtmd_helper_video_info {int32_t n_frames; // estimated total frames at effective fps (-1 if unknown)};-struct mtmd_helper_video_init_params {- float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f- const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH- int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms- // TODO @ngxson : allow "placeholder" bitmap output for counting tokens-};--MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);+// note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt// returns NULL on failure (ffprobe not found, file unreadable, etc.)MTMD_API mtmd_helper_video * mtmd_helper_video_init(diff --git a/tools/server/README.md b/tools/server/README.mdindex 93736c3edf..f49cdb2729 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -182,6 +182,9 @@ For the full list of features, please refer to [server's changelog](https://gith| `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) || `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) || `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) |+| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |+| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |+| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) || `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) || `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) || `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) |diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 7997d4016a..c30955e89f 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -910,12 +910,17 @@ size_t validate_utf8(const std::string& text) {return len;}-server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {+server_tokens process_mtmd_prompt(+ mtmd_context * mctx,+ const std::string & prompt,+ const std::vector<raw_buffer> & files,+ const mtmd_helper_init_opt & init_opt,+ bool is_placeholder) {// these will be freed upon going out of scopemtmd::bitmaps bitmaps;std::vector<mtmd_helper::video_ptr> videos;for (auto & file : files) {- auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder);+ auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder, init_opt);if (!out.bitmap) {throw std::runtime_error("Failed to load image or audio file");}@@ -956,7 +961,7 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp* - "prompt": [12, 34, "string", 56, 78]* - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }*/-static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {+static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";const bool has_mtmd = mctx != nullptr;@@ -979,7 +984,7 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_cofor (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {files.push_back(base64_decode(entry));}- return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files);+ return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files, init_opt);} else {// Not multimodal, but contains a subobject.llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);@@ -990,15 +995,15 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co}}-std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {+std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {std::vector<server_tokens> result;if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {result.reserve(json_prompt.size());for (const auto & p : json_prompt) {- result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special));+ result.push_back(tokenize_input_subprompt(vocab, mctx, p, add_special, parse_special, init_opt));}} else {- result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special));+ result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special, init_opt));}if (result.empty()) {throw std::runtime_error("\"prompt\" must not be empty");@@ -1787,7 +1792,8 @@ server_tokens format_prompt_rerank(const struct llama_vocab * vocab,mtmd_context * mctx,const std::string & query,- const std::string & doc) {+ const std::string & doc,+ const mtmd_helper_init_opt & init_opt) {server_tokens result = {};const char * rerank_prompt = llama_model_chat_template(model, "rerank");@@ -1796,12 +1802,12 @@ server_tokens format_prompt_rerank(std::string prompt = rerank_prompt;string_replace_all(prompt, "{query}" , query);string_replace_all(prompt, "{document}", doc );- server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true);+ server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);result.push_back(tokens);} else {// Get EOS token - use SEP token as fallback if EOS is not available- server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false);- server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false);+ server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);+ server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false, init_opt);llama_token eos_token = llama_vocab_eos(vocab);if (eos_token == LLAMA_TOKEN_NULL) {eos_token = llama_vocab_sep(vocab);diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex f8ea82ef4c..6c681a2cf5 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -5,6 +5,7 @@#include "llama.h"#include "chat.h"#include "mtmd.h"+#include "mtmd-helper.h"#include "json.h"@@ -269,7 +270,12 @@ size_t validate_utf8(const std::string& text);// process mtmd prompt, return the server_tokens containing both text tokens and media chunks// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)-server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder = false);+server_tokens process_mtmd_prompt(+ mtmd_context * mctx,+ const std::string & prompt,+ const std::vector<raw_buffer> & files,+ const mtmd_helper_init_opt & init_opt,+ bool is_placeholder = false);/*** break the input "prompt" object into multiple prompt if needed, then tokenize them@@ -289,7 +295,8 @@ std::vector<server_tokens> tokenize_input_prompts(mtmd_context * mctx,const json & json_prompt,bool add_special,- bool parse_special);+ bool parse_special,+ const mtmd_helper_init_opt & init_opt);//// OAI utils@@ -538,7 +545,8 @@ server_tokens format_prompt_rerank(const struct llama_vocab * vocab,mtmd_context * mctx,const std::string & query,- const std::string & doc);+ const std::string & doc,+ const mtmd_helper_init_opt & init_opt);// simple implementation of a pipe// used for streaming data between threadsdiff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex a9edbd7be8..9fdfcae563 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -794,6 +794,8 @@ public:llama_model * model_tgt = nullptr;mtmd_context * mctx = nullptr;+ // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct+ mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();const llama_vocab * vocab = nullptr;server_queue queue_tasks;@@ -1118,6 +1120,11 @@ private:}SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());+ init_opt.video_params.fps_target = params_base.video_fps;+ init_opt.video_params.timestamp_interval_ms = params_base.video_timestamp_interval_ms;+ init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()+ ? nullptr : params_base.video_ffmpeg_bin_dir.c_str();+if (params_base.ctx_shift) {params_base.ctx_shift = false;SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");@@ -2134,9 +2141,9 @@ private:try {auto & prompt = task.cli_prompt;if (mctx != nullptr) {- task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files);+ task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files, init_opt);} else {- task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true)[0]);+ task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true, init_opt)[0]);}task.cli_prompt.clear();task.cli_files.clear();@@ -4165,10 +4172,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) {// This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below.- inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files));+ inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files, ctx_server.init_opt));} else {// Everything else, including multimodal completions.- inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true);+ inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);}// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks@@ -4752,7 +4759,7 @@ void server_routes::init_routes() {data["input_extra"] = input_extra; // default to empty array if it's not existstd::string prompt = json_value(data, "prompt", std::string());- std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true);+ std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true, ctx_server.init_opt);SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size());data["prompt"] = format_prompt_infill(ctx_server.vocab,@@ -4816,7 +4823,7 @@ void server_routes::init_routes() {};this->post_chat_completions_tok = [this](const server_http_req & req) {- return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_CHAT);+ return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_CHAT);};this->post_control = [this](const server_http_req & req) {@@ -4875,7 +4882,7 @@ void server_routes::init_routes() {};this->post_responses_tok_oai = [this](const server_http_req & req) {- return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_RESP);+ return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_RESP);};this->post_transcriptions_oai = [this](const server_http_req & req) {@@ -4925,7 +4932,7 @@ void server_routes::init_routes() {};this->post_anthropic_count_tokens = [this](const server_http_req & req) {- return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_ANTHROPIC);+ return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_ANTHROPIC);};// same with handle_chat_completions, but without inference part@@ -5058,7 +5065,7 @@ void server_routes::init_routes() {std::vector<server_task> tasks;tasks.reserve(documents.size());for (size_t i = 0; i < documents.size(); i++) {- auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]);+ auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);server_task task = server_task(SERVER_TASK_TYPE_RERANK);task.id = rd.get_new_id();task.tokens = std::move(tmp);@@ -5296,7 +5303,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons}}- auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true);+ auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);for (const auto & tokens : tokenized_prompts) {// this check is necessary for models that do not add BOS token to the inputif (tokens.empty()) {@@ -5357,7 +5364,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(consreturn res;}-std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type) {+std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type) {auto res = create_response();std::vector<raw_buffer> files;json body = json::parse(req.body);@@ -5395,7 +5402,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const lif (!prompt.is_string()) {throw std::runtime_error("for mtmd, input prompt must be a string.");}- n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, true).size();+ n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, init_opt, true).size();} else {n_tokens = tokenize_mixed(vocab, prompt, true, true).size();}diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex 5d464b8e8c..0acbbffa9e 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -169,7 +169,7 @@ private:std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);- std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type);+ std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type);// using unique_ptr to allow late initialization of conststd::unique_ptr<const server_context_meta> meta;diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cppindex 368123baf5..6fd1936324 100644--- a/tools/tts/tts.cpp+++ b/tools/tts/tts.cpp@@ -103,7 +103,7 @@ int main(int argc, char ** argv) {mtmd::bitmap_ptr speaker_bitmap;if (!params.tts_speaker_file.empty()) {- auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false);+ auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false, mtmd_helper_init_opt_default());if (!wrapper.bitmap) {LOG_ERR("failed to load speaker file %s\n", params.tts_speaker_file.c_str());return 1;