LLM inference in C/C++
C++ ggml
common : extract shared unicode path/string helpers (#29415)
Adrien Gallouët27b20ba8b1b8
diff --git a/common/common.cpp b/common/common.cppindex a316b2ae17..364688ef46 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -900,7 +900,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {#ifdef _WIN32-static std::wstring utf8_to_wstring(const std::string & str) {+std::wstring utf8_to_wstring(const std::string & str) {if (str.empty()) {return std::wstring();}@@ -916,8 +916,31 @@ static std::wstring utf8_to_wstring(const std::string & str) {return wstr;}++std::string wstring_to_utf8(const std::wstring & str) {+ if (str.empty()) {+ return std::string();+ }++ int size = WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0, NULL, NULL);++ if (size <= 0) {+ return std::string();+ }++ std::string utf8(size, 0);+ WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), &utf8[0], size, NULL, NULL);++ return utf8;+}#endif+// returns the path as a UTF-8 string, preserving its separators+std::string fs_path_to_utf8(const std::filesystem::path & path) {+ const auto value = path.u8string();+ return std::string(value.begin(), value.end());+}+// returns true if successful, false otherwisebool fs_create_directory_with_parents(const std::string & path) {#ifdef _WIN32diff --git a/common/common.h b/common/common.hindex 39f855c4f2..9194d3dcb6 100644--- a/common/common.h+++ b/common/common.h@@ -16,6 +16,7 @@#include <vector>#include <map>#include <algorithm>+#include <filesystem>#include <fstream>#if defined(_WIN32) && !defined(_WIN32_WINNT)@@ -880,6 +881,18 @@ std::string string_from(const struct llama_context * ctx, const struct llama_batbool glob_match(const std::string & pattern, const std::string & str);+//+// Unicode utils+//++#ifdef _WIN32+std::wstring utf8_to_wstring(const std::string & str);+std::string wstring_to_utf8(const std::wstring & str);+#endif++// returns the path as a UTF-8 string, preserving its separators+std::string fs_path_to_utf8(const std::filesystem::path & path);+//// Environment utils//diff --git a/common/console.cpp b/common/console.cppindex 36f645f332..9020eb5131 100644--- a/common/console.cpp+++ b/common/console.cpp@@ -1,4 +1,5 @@#include "console.h"+#include "common.h"#include "log.h"#include <vector>#include <iostream>@@ -1053,9 +1054,7 @@ namespace console {return false;}- int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);- line.resize(size_needed);- WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);+ line = wstring_to_utf8(wline);#elseif (!std::getline(std::cin, line)) {// Input stream is bad or EOF receiveddiff --git a/common/hf-cache.cpp b/common/hf-cache.cppindex 12d4fcc804..65c9722f33 100644--- a/common/hf-cache.cpp+++ b/common/hf-cache.cpp@@ -63,12 +63,7 @@ static fs::path get_cache_directory() {}std::string get_cache_path() {-#if defined(__cpp_lib_char8_t)- const std::u8string u8str = get_cache_directory().u8string();- return std::string(reinterpret_cast<const char *>(u8str.data()), u8str.size());-#else- return get_cache_directory().u8string();-#endif+ return fs_path_to_utf8(get_cache_directory());}static std::string folder_name_to_repo(const std::string & folder) {diff --git a/tools/server/server-mcp.cpp b/tools/server/server-mcp.cppindex 93db6164d3..88b2c92399 100644--- a/tools/server/server-mcp.cpp+++ b/tools/server/server-mcp.cpp@@ -1,5 +1,6 @@#include "server-mcp.h"+#include "common.h"#include "subproc.h"#include <atomic>@@ -349,43 +350,21 @@ struct server_mcp_stdio::process_handle {#if defined(_WIN32)// config strings are UTF-8 (from JSON) and subprocess.h converts them with CP_UTF8, so inputs must be UTF-8, not the active code page-static std::wstring windows_utf8_to_wide(const std::string & s) {- if (s.empty()) {- return std::wstring();- }- int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), NULL, 0);- if (n <= 0) {- return std::wstring();- }- std::wstring w((size_t) n, L'\0');- MultiByteToWideChar(CP_UTF8, 0, s.data(), (int) s.size(), &w[0], n);- return w;-}--static std::string windows_wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) {- int n = WideCharToMultiByte(CP_UTF8, 0, s, len, NULL, 0, NULL, NULL);- if (n <= 0) {- return std::string();- }- std::string out((size_t) n, '\0');- WideCharToMultiByte(CP_UTF8, 0, s, len, &out[0], n, NULL, NULL);- if (len == -1 && !out.empty() && out.back() == '\0') {- out.pop_back(); // drop the terminator WideCharToMultiByte counts for -1- }- return out;+static std::string wide_to_utf8(const wchar_t * s, int len /* -1 for NUL-terminated */) {+ return wstring_to_utf8(len == -1 ? std::wstring(s) : std::wstring(s, s + len));}#endifstatic std::string mcp_resolve_command(const std::string & command) {#if defined(_WIN32)// For Windows: make sure we handle ".exe" correctly, as well as UTF-8- std::wstring wcmd = windows_utf8_to_wide(command);+ std::wstring wcmd = utf8_to_wstring(command);wchar_t buf[MAX_PATH * 4];const DWORD cap = (DWORD) (sizeof(buf) / sizeof(buf[0]));auto search = [&](const wchar_t * ext) -> std::string {DWORD n = SearchPathW(NULL, wcmd.c_str(), ext, cap, buf, NULL);- return (n > 0 && n < cap) ? windows_wide_to_utf8(buf, (int) n) : std::string();+ return (n > 0 && n < cap) ? wide_to_utf8(buf, (int) n) : std::string();};std::string found = search(NULL); // exact path / already-extensioned / .exe on PATH@@ -429,7 +408,7 @@ static std::vector<std::string> mcp_parent_env() {LPWCH block = GetEnvironmentStringsW();if (block) {for (LPWCH e = block; *e; e += wcslen(e) + 1) {- env.emplace_back(windows_wide_to_utf8(e, -1));+ env.emplace_back(wide_to_utf8(e, -1));}FreeEnvironmentStringsW(block);}diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppindex 841221d8c7..a8996b00c9 100644--- a/tools/server/server-models.cpp+++ b/tools/server/server-models.cpp@@ -483,25 +483,6 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {}}-#ifdef _WIN32-static std::string wide_to_utf8(const wchar_t * ws) {- if (!ws || !*ws) {- return {};- }-- const int len = static_cast<int>(std::wcslen(ws));- const int bytes = WideCharToMultiByte(CP_UTF8, 0, ws, len, nullptr, 0, nullptr, nullptr);- if (bytes == 0) {- return {};- }-- std::string utf8(bytes, '\0');- WideCharToMultiByte(CP_UTF8, 0, ws, len, utf8.data(), bytes, nullptr, nullptr);-- return utf8;-}-#endif-static std::vector<std::string> get_environment() {std::vector<std::string> env;@@ -511,7 +492,7 @@ static std::vector<std::string> get_environment() {return env;}for (LPWCH e = env_block; *e; e += wcslen(e) + 1) {- env.emplace_back(wide_to_utf8(e));+ env.emplace_back(wstring_to_utf8(e));}FreeEnvironmentStringsW(env_block);#else@@ -592,7 +573,7 @@ server_models::server_models(// set binary pathtry {- bin_path = get_server_exec_path().string();+ bin_path = fs_path_to_utf8(get_server_exec_path());} catch (const std::exception & e) {bin_path = argv[0];LOG_WRN("failed to get server executable path: %s\n", e.what());