LLM inference in C/C++
C++ ggml
common : use fs::path for cache dirs (#29595)
Adrien Gallouët76a5bc86d1bd
diff --git a/common/arg.cpp b/common/arg.cppindex 2acac3b5cf..b70da5e9e6 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -351,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {static std::string get_default_local_path(const std::string & url) {auto f = string_split<std::string>(url, '#').front();f = string_split<std::string>(f, '?').front();- return fs_get_cache_file(string_split<std::string>(f, '/').back());+ return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));}static bool spec_types_is_default(const common_params & params) {diff --git a/common/common.cpp b/common/common.cppindex 6f443f1bf0..6d57b54217 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -49,7 +49,7 @@#include <unistd.h>#endif-#if defined(__linux__)+#if !defined(_WIN32) && !defined(__APPLE__)#include <sys/types.h>#include <pwd.h>#endif@@ -912,13 +912,6 @@ std::string fs_path_to_utf8(const std::filesystem::path & path) {return std::string(value.begin(), value.end());}-// returns true if successful, false otherwise-bool fs_create_directory_with_parents(const std::string & path) {- std::error_code ec;- std::filesystem::create_directories(std::filesystem::u8path(path), ec);- return !ec;-}-bool fs_is_directory(const std::string & path) {std::filesystem::path dir(path);return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);@@ -952,58 +945,41 @@ std::filesystem::path common_get_path_from_env(const std::string & name) {#endif}-std::string fs_get_cache_directory() {- std::string cache_directory = "";- auto ensure_trailing_slash = [](std::string p) {- // Make sure to add trailing slash- if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {- p += DIRECTORY_SEPARATOR;- }- return p;- };- cache_directory = common_get_env("LLAMA_CACHE");+std::filesystem::path fs_get_cache_directory() {+ std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");+ if (!cache_directory.empty()) {+ return cache_directory;+ }++#if defined(_WIN32)+ cache_directory = common_get_path_from_env("LOCALAPPDATA");if (cache_directory.empty()) {-#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \- defined(__OpenBSD__) || defined(__NetBSD__)- const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");- const std::string home = common_get_env("HOME");- if (!xdg_cache_home.empty()) {- cache_directory = xdg_cache_home;- } else if (!home.empty()) {- cache_directory = home + "/.cache/";+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");+ }+#elif defined(__APPLE__)+ cache_directory = common_get_path_from_env("HOME");+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find $HOME directory");+ }+ cache_directory /= "Library/Caches";+#else+ cache_directory = common_get_path_from_env("XDG_CACHE_HOME");+ if (cache_directory.empty()) {+ cache_directory = common_get_path_from_env("HOME");+ if (!cache_directory.empty()) {+ cache_directory /= ".cache";} else {-#if defined(__linux__)/* no $HOME is defined, fallback to getpwuid */- struct passwd *pw = getpwuid(getuid());- if ((!pw) || (!pw->pw_dir)) {+ const struct passwd * pw = getpwuid(getuid());+ if (!pw || !pw->pw_dir || !*pw->pw_dir) {throw std::runtime_error("Failed to find $HOME directory");}-- cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");-#else /* defined(__linux__) */- throw std::runtime_error("Failed to find $HOME directory");-#endif /* defined(__linux__) */- }-#elif defined(__APPLE__)- cache_directory = common_get_env("HOME");- if (cache_directory.empty()) {- throw std::runtime_error("Failed to find $HOME directory");+ cache_directory = pw->pw_dir;+ cache_directory /= ".cache";}- cache_directory += "/Library/Caches/";-#elif defined(_WIN32)- cache_directory = common_get_env("LOCALAPPDATA");- if (cache_directory.empty()) {- throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");- }-#elif defined(__EMSCRIPTEN__)- GGML_ABORT("not implemented on this platform");-#else-# error Unknown architecture-#endif- cache_directory = ensure_trailing_slash(cache_directory);- cache_directory += "llama.cpp";}- return ensure_trailing_slash(cache_directory);+#endif+ return cache_directory / "llama.cpp";}std::string fs_get_config_directory() {@@ -1051,14 +1027,15 @@ std::string fs_get_config_directory() {return ensure_trailing_slash(config_directory);}-std::string fs_get_cache_file(const std::string & filename) {+std::filesystem::path fs_get_cache_file(const std::string & filename) {GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);- std::string cache_directory = fs_get_cache_directory();- const bool success = fs_create_directory_with_parents(cache_directory);- if (!success) {- throw std::runtime_error("failed to create cache directory: " + cache_directory);+ const std::filesystem::path cache_directory = fs_get_cache_directory();+ std::error_code ec;+ std::filesystem::create_directories(cache_directory, ec);+ if (ec) {+ throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));}- return cache_directory + filename;+ return cache_directory / std::filesystem::u8path(filename);}std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {diff --git a/common/common.h b/common/common.hindex 45b15def71..0a09fed7f6 100644--- a/common/common.h+++ b/common/common.h@@ -912,11 +912,10 @@ std::filesystem::path common_get_path_from_env(const std::string & name);//bool fs_validate_filename(const std::string & filename, bool allow_subdirs = false);-bool fs_create_directory_with_parents(const std::string & path);bool fs_is_directory(const std::string & path);-std::string fs_get_cache_directory();-std::string fs_get_cache_file(const std::string & filename);+std::filesystem::path fs_get_cache_directory();+std::filesystem::path fs_get_cache_file(const std::string & filename);std::string fs_get_config_directory();struct common_file_info {diff --git a/common/download.cpp b/common/download.cppindex 2edd2a11ca..d7875078c5 100644--- a/common/download.cpp+++ b/common/download.cpp@@ -943,7 +943,7 @@ std::string common_docker_resolve_model(const std::string & docker) {std::string model_filename = repo;std::replace(model_filename.begin(), model_filename.end(), '/', '_');model_filename += "_" + tag + ".gguf";- std::string local_path = fs_get_cache_file(model_filename);+ std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;common_download_opts opts;diff --git a/tests/gguf-model-data.cpp b/tests/gguf-model-data.cppindex fe8b4ca76e..3e2147dd2b 100644--- a/tests/gguf-model-data.cpp+++ b/tests/gguf-model-data.cpp@@ -371,8 +371,8 @@ static std::optional<gguf_remote_model> gguf_parse_meta(const std::vector<char>}// cache handling for local download-static std::string get_default_cache_dir() {- return fs_get_cache_directory() + "gguf-headers/";+static std::filesystem::path get_default_cache_dir() {+ return fs_get_cache_directory() / "gguf-headers";}static std::string sanitize_for_path(const std::string & s) {@@ -385,7 +385,7 @@ static std::string sanitize_for_path(const std::string & s) {return out;}-static bool read_file(const std::string & path, std::vector<char> & out) {+static bool read_file(const std::filesystem::path & path, std::vector<char> & out) {std::ifstream f(path, std::ios::binary | std::ios::ate);if (!f.good()) {return false;@@ -400,7 +400,7 @@ static bool read_file(const std::string & path, std::vector<char> & out) {return f.good();}-static bool write_file(const std::string & path, const std::vector<char> & data) {+static bool write_file(const std::filesystem::path & path, const std::vector<char> & data) {std::ofstream f(path, std::ios::binary | std::ios::trunc);if (!f.good()) {return false;@@ -516,7 +516,7 @@ static std::string detect_gguf_filename(const std::string & repo, const std::strstatic std::optional<gguf_remote_model> fetch_and_parse(const std::string & repo,const std::string & filename,- const std::string & cache_path,+ const std::filesystem::path & cache_path,bool verbose) {std::string url = "https://huggingface.co/" + repo + "/resolve/main/" + filename;@@ -564,19 +564,19 @@ static std::optional<gguf_remote_model> fetch_and_parse(return std::nullopt;}-static std::string get_cache_file_path(const std::string& cdir, const std::string& repo_part, const std::string& filename) {+static std::filesystem::path get_cache_file_path(const std::filesystem::path & cdir, const std::string & repo_part, const std::string & filename) {std::string fname_part = sanitize_for_path(filename);- return cdir + "/" + repo_part + "--" + fname_part + ".partial";+ return cdir / std::filesystem::u8path(repo_part + "--" + fname_part + ".partial");}// Try cache first, then fetch and parse a single GGUF shard.static std::optional<gguf_remote_model> fetch_or_cached(const std::string & repo,const std::string & filename,- const std::string & cdir,+ const std::filesystem::path & cdir,const std::string & repo_part,bool verbose) {- std::string cache_path = get_cache_file_path(cdir, repo_part, filename);+ std::filesystem::path cache_path = get_cache_file_path(cdir, repo_part, filename);{std::vector<char> cached;@@ -584,14 +584,15 @@ static std::optional<gguf_remote_model> fetch_or_cached(auto result = gguf_parse_meta(cached);if (result.has_value()) {if (verbose) {- fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", cache_path.c_str());+ fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", fs_path_to_utf8(cache_path).c_str());}return result;}}}- fs_create_directory_with_parents(cdir);+ std::error_code ec;+ std::filesystem::create_directories(cdir, ec);return fetch_and_parse(repo, filename, cache_path, verbose);}@@ -600,7 +601,7 @@ std::optional<gguf_remote_model> gguf_fetch_model_meta(const std::string & quant,const std::string & cache_dir,bool verbose) {- std::string cdir = cache_dir.empty() ? get_default_cache_dir() : cache_dir;+ const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir);std::string repo_part = sanitize_for_path(repo);std::string split_prefix;@@ -661,7 +662,7 @@ gguf_context_ptr gguf_fetch_gguf_ctx(const std::string & quant,const std::string & cache_dir,bool verbose) {- std::string cdir = cache_dir.empty() ? get_default_cache_dir() : cache_dir;+ const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir);std::string repo_part = sanitize_for_path(repo);std::string split_prefix;@@ -679,12 +680,12 @@ gguf_context_ptr gguf_fetch_gguf_ctx(auto & model = model_opt.value();- const std::string cache_path = get_cache_file_path(cdir, repo_part, filename);+ const std::filesystem::path cache_path = get_cache_file_path(cdir, repo_part, filename);ggml_context_ptr ggml_ctx_ptr;ggml_context * ggml_ctx{};gguf_init_params params{true, &ggml_ctx};- gguf_context_ptr ctx{gguf_init_from_file(cache_path.c_str(), params)};+ gguf_context_ptr ctx{gguf_init_from_file(fs_path_to_utf8(cache_path).c_str(), params)};ggml_ctx_ptr.reset(ggml_ctx);if (ctx == nullptr) {@@ -718,11 +719,11 @@ gguf_context_ptr gguf_fetch_gguf_ctx(}// Load tensors from shard and add to main gguf_context- const std::string shard_path = get_cache_file_path(cdir, repo_part, shard_name);+ const std::filesystem::path shard_path = get_cache_file_path(cdir, repo_part, shard_name);ggml_context_ptr shard_ggml_ctx_ptr;ggml_context * shard_ggml_ctx{};gguf_init_params shard_params{true, &shard_ggml_ctx};- gguf_context_ptr shard_ctx{gguf_init_from_file(shard_path.c_str(), shard_params)};+ gguf_context_ptr shard_ctx{gguf_init_from_file(fs_path_to_utf8(shard_path).c_str(), shard_params)};shard_ggml_ctx_ptr.reset(shard_ggml_ctx);if (shard_ctx == nullptr) {diff --git a/tools/rpc/rpc-server.cpp b/tools/rpc/rpc-server.cppindex 100df3c409..5c30e6fe47 100644--- a/tools/rpc/rpc-server.cpp+++ b/tools/rpc/rpc-server.cpp@@ -2,12 +2,10 @@#include "ggml-rpc.h"#ifdef _WIN32# define NOMINMAX-# define DIRECTORY_SEPARATOR '\\'# include <windows.h># include <fcntl.h># include <io.h>#else-# define DIRECTORY_SEPARATOR '/'# include <unistd.h>#endif#include <algorithm>@@ -20,64 +18,68 @@#include <thread>#include <vector>-#if defined(__linux__)+#if !defined(_WIN32) && !defined(__APPLE__)#include <sys/types.h>#include <pwd.h>#endif-// NOTE: this is copied from common.cpp to avoid linking with libcommon-// returns true if successful, false otherwise-static bool fs_create_directory_with_parents(const std::string & path) {- std::error_code ec;- std::filesystem::create_directories(std::filesystem::u8path(path), ec);- return !ec;++static std::string fs_path_to_utf8(const std::filesystem::path & path) {+ const auto value = path.u8string();+ return std::string(value.begin(), value.end());+}++// common_get_path_from_env() is adapted to avoid utf8_to_wstring+static std::filesystem::path common_get_path_from_env(const std::string & name) {+#ifdef _WIN32+ std::wstring wname;+ for (const char * p = name.c_str(); *p; ++p) {+ wname.push_back((wchar_t)*p);+ }+ const wchar_t * wvalue = _wgetenv(wname.c_str());+ return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();+#else+ const char * value = std::getenv(name.c_str());+ return value ? std::filesystem::path(value) : std::filesystem::path();+#endif}// NOTE: this is copied from common.cpp to avoid linking with libcommon-static std::string fs_get_cache_directory() {- std::string cache_directory = "";- auto ensure_trailing_slash = [](std::string p) {- // Make sure to add trailing slash- if (p.back() != DIRECTORY_SEPARATOR) {- p += DIRECTORY_SEPARATOR;- }- return p;- };- if (getenv("LLAMA_CACHE")) {- cache_directory = std::getenv("LLAMA_CACHE");- } else {-#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \- defined(__OpenBSD__) || defined(__NetBSD__)- if (std::getenv("XDG_CACHE_HOME")) {- cache_directory = std::getenv("XDG_CACHE_HOME");- } else if (std::getenv("HOME")) {- cache_directory = std::getenv("HOME") + std::string("/.cache/");+static std::filesystem::path fs_get_cache_directory() {+ std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");+ if (!cache_directory.empty()) {+ return cache_directory;+ }++#if defined(_WIN32)+ cache_directory = common_get_path_from_env("LOCALAPPDATA");+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");+ }+#elif defined(__APPLE__)+ cache_directory = common_get_path_from_env("HOME");+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find $HOME directory");+ }+ cache_directory /= "Library/Caches";+#else+ cache_directory = common_get_path_from_env("XDG_CACHE_HOME");+ if (cache_directory.empty()) {+ cache_directory = common_get_path_from_env("HOME");+ if (!cache_directory.empty()) {+ cache_directory /= ".cache";} else {-#if defined(__linux__)/* no $HOME is defined, fallback to getpwuid */- struct passwd *pw = getpwuid(getuid());- if ((!pw) || (!pw->pw_dir)) {+ const struct passwd * pw = getpwuid(getuid());+ if (!pw || !pw->pw_dir || !*pw->pw_dir) {throw std::runtime_error("Failed to find $HOME directory");}-- cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");-#else /* defined(__linux__) */- throw std::runtime_error("Failed to find $HOME directory");-#endif /* defined(__linux__) */+ cache_directory = pw->pw_dir;+ cache_directory /= ".cache";}-#elif defined(__APPLE__)- cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");-#elif defined(_WIN32)- cache_directory = std::getenv("LOCALAPPDATA");-#elif defined(__EMSCRIPTEN__)- GGML_ABORT("not implemented on this platform");-#else-# error Unknown architecture-#endif- cache_directory = ensure_trailing_slash(cache_directory);- cache_directory += "llama.cpp";}- return ensure_trailing_slash(cache_directory);+#endif+ return cache_directory / "llama.cpp";}struct rpc_server_params {@@ -229,11 +231,14 @@ int main(int argc, char * argv[]) {const char * cache_dir = nullptr;std::string cache_dir_str;if (params.use_cache) {- cache_dir_str = fs_get_cache_directory() + "rpc" + DIRECTORY_SEPARATOR;- if (!fs_create_directory_with_parents(cache_dir_str)) {- fprintf(stderr, "Failed to create cache directory: %s\n", cache_dir_str.c_str());+ const std::filesystem::path cache_dir_path = fs_get_cache_directory() / "rpc";+ std::error_code ec;+ std::filesystem::create_directories(cache_dir_path, ec);+ if (ec) {+ fprintf(stderr, "Failed to create cache directory: %s\n", fs_path_to_utf8(cache_dir_path).c_str());return 1;}+ cache_dir_str = fs_path_to_utf8(cache_dir_path);cache_dir = cache_dir_str.c_str();}