LLM inference in C/C++
C++ ggml
mtmd, arg: fix utf8 handling on windows (#24779)
Xuan-Son Nguyene475fa2b5f9f
diff --git a/common/arg.cpp b/common/arg.cppindex 6fd366d33b..8f4f7d0763 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -17,6 +17,7 @@# define NOMINMAX#endif#include <windows.h>+#include <shellapi.h>#endif#define JSON_ASSERT GGML_ASSERT@@ -893,7 +894,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<comreturn true;}+#ifdef _WIN32+struct utf8_argv {+ std::vector<std::string> buf;+ std::vector<char*> ptrs;+};++static utf8_argv make_utf8_argv() {+ utf8_argv out;+ int wargc = 0;+ LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);+ if (!wargv) return out;++ out.buf.reserve(wargc);+ for (int i = 0; i < wargc; ++i) {+ int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);+ if (n <= 0) { out.buf.emplace_back(); continue; }+ auto& s = out.buf.emplace_back();+ s.resize(static_cast<size_t>(n - 1));+ (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);+ }+ LocalFree(wargv);++ out.ptrs.reserve(out.buf.size() + 1);+ for (auto& s : out.buf) out.ptrs.push_back(s.data());+ out.ptrs.push_back(nullptr);+ return out;+}+#endif+bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {+#ifdef _WIN32+ auto utf8 = make_utf8_argv();+ if (!utf8.ptrs.empty()) {+ argc = static_cast<int>(utf8.buf.size());+ argv = utf8.ptrs.data();+ }+#endif+auto ctx_arg = common_params_parser_init(params, ex, print_usage);const common_params params_org = ctx_arg.params; // the example can modify the default paramsdiff --git a/common/common.cpp b/common/common.cppindex f3f114f682..a14e7bbed9 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1074,6 +1074,18 @@ std::vector<common_file_info> fs_list(const std::string & path, bool include_dirreturn files;}+std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) {+#ifdef _WIN32+ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);+ if (!wlen) { return std::ifstream(); }+ std::vector<wchar_t> wfname(wlen);+ (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);+ return std::ifstream(wfname.data(), mode);+#else+ return std::ifstream(fname, mode);+#endif+}+//// TTY utils//diff --git a/common/common.h b/common/common.hindex 44c605189c..254454dcb1 100644--- a/common/common.h+++ b/common/common.h@@ -842,6 +842,9 @@ struct common_file_info {};std::vector<common_file_info> fs_list(const std::string & path, bool include_directories);+// fs open, also handle UTF8 on Windows+std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode);+//// TTY utils//diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex b43016c87d..0f682fd185 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -600,18 +600,15 @@ FILE * ggml_fopen(const char * fname, const char * mode) {// convert fname (UTF-8)wchar_t * wfname = ggml_mbstowcs(fname);if (wfname) {- // convert mode (ANSI)- wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t));- wchar_t * wmode_p = wmode;- do {- *wmode_p++ = (wchar_t)*mode;- } while (*mode++);-- // open file- file = _wfopen(wfname, wmode);+ // convert mode (UTF-8)+ wchar_t * wmode = ggml_mbstowcs(mode);+ if (wmode) {+ // open file+ file = _wfopen(wfname, wmode);+ GGML_FREE(wmode);+ }GGML_FREE(wfname);- GGML_FREE(wmode);}return file;diff --git a/tools/cli/cli.cpp b/tools/cli/cli.cppindex c03894b4b1..8b7b58693f 100644--- a/tools/cli/cli.cpp+++ b/tools/cli/cli.cpp@@ -202,7 +202,7 @@ struct cli_context {// TODO: support remote files in the future (http, https, etc)std::string load_input_file(const std::string & fname, bool is_media) {- std::ifstream file(fname, std::ios::binary);+ std::ifstream file = fs_open_ifstream(fname, std::ios::binary);if (!file) {return "";}diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.hindex f232b68e5a..e7b5301445 100644--- a/tools/mtmd/clip-impl.h+++ b/tools/mtmd/clip-impl.h@@ -13,6 +13,14 @@#include <sstream>#include <vector>#include <memory>+#include <fstream>++#ifdef _WIN32+#ifndef NOMINMAX+#define NOMINMAX+#endif+#include <windows.h>+#endif// Internal header for clip.cpp@@ -661,6 +669,22 @@ struct clip_image_f32_batch {// common utils//+#ifdef _WIN32+static std::ifstream open_ifstream_binary(const std::string & fname) {+ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);+ if (!wlen) {+ throw std::runtime_error("failed to convert filename to UTF-16: " + fname);+ }+ std::vector<wchar_t> wfname(wlen);+ (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);+ return std::ifstream(wfname.data(), std::ios::binary);+}+#else+static std::ifstream open_ifstream_binary(const std::string & fname) {+ return std::ifstream(fname, std::ios::binary);+}+#endif+static std::string string_format(const char * fmt, ...) {va_list ap;va_list ap2;diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 10840a851f..c713703e01 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -1752,7 +1752,7 @@ struct clip_model_loader {std::map<std::string, size_t> tensor_offset;std::vector<ggml_tensor *> tensors_to_load;- auto fin = std::ifstream(fname, std::ios::binary);+ auto fin = open_ifstream_binary(fname);if (!fin) {throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str()));}diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cppindex 0ad000ef01..8704ea79d7 100644--- a/tools/mtmd/mtmd-cli.cpp+++ b/tools/mtmd/mtmd-cli.cpp@@ -396,6 +396,9 @@ int main(int argc, char ** argv) {int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict;+ console::init(params.simple_io, params.use_color);+ atexit([]() { console::cleanup(); });+// Ctrl+C handling{#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cppindex b5c4089232..3c73db4431 100644--- a/tools/mtmd/mtmd-helper.cpp+++ b/tools/mtmd/mtmd-helper.cpp@@ -582,13 +582,29 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,}mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {- std::vector<unsigned char> buf;+#ifdef _WIN32+ int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);+ if (!wlen) {+ LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);+ return {nullptr, nullptr};+ }+ std::vector<wchar_t> wfname(wlen);+ wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wlen);+ if (!wlen) {+ LOG_ERR("Unable to convert filename to UTF-16: %s\n", fname);+ return {nullptr, nullptr};+ }+ FILE * f = _wfopen(wfname.data(), L"rb");+#elseFILE * f = fopen(fname, "rb");+#endifif (!f) {LOG_ERR("Unable to open file %s: %s\n", fname, strerror(errno));return {nullptr, nullptr};}+ std::vector<unsigned char> buf;+fseek(f, 0, SEEK_END);long file_size = ftell(f);fseek(f, 0, SEEK_SET);