Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Use hipcc on $PATH if it exists
Justine Tunney67d97b520278
diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 2021dd5b..c3c3baf8 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -187,7 +187,7 @@ static bool Compile(const char *src,}}if (ws) {- tinylog(args[0], ": returned nonzero exit status\n", NULL);+ tinylog(__func__, ": warning: ", args[0], " returned nonzero exit status\n", NULL);unlink(tmp);return false;}@@ -199,29 +199,51 @@ static bool Compile(const char *src,return true;}-static bool GetRocmBinPath(char path[static PATH_MAX], const char *bin) {+static bool get_rocm_bin_path(char path[static PATH_MAX], const char *bin) {++ // create filename of executable+ char name[NAME_MAX];+ strlcpy(name, bin, PATH_MAX);+ if (IsWindows()) {+ strlcat(name, ".exe", PATH_MAX);+ }++ // search for it on $PATH+ if (commandv(name, path, PATH_MAX)) {+ return path;+ } else {+ tinylog(__func__, ": note: ", name, " not found on $PATH\n", NULL);+ }++ // 1. use $HIP_PATH/bin/$name if it exists+ // 2. use /opt/rocm/bin/$name if it existsconst char *hip_path = getenv("HIP_PATH");- if (hip_path || FileExists((hip_path = "/opt/rocm"))) {- strlcpy(path, hip_path, PATH_MAX);- strlcat(path, "/bin/", PATH_MAX);- strlcat(path, bin, PATH_MAX);- if (IsWindows()) {- strlcat(path, ".exe", PATH_MAX);+ if (!hip_path) {+ tinylog(__func__, ": note: $HIP_PATH/bin/", name, " does not exist\n", NULL);+ if (!FileExists((hip_path = "/opt/rocm"))) {+ tinylog(__func__, ": note: /opt/rocm/bin/", name, " does not exist\n", NULL);+ return false;}- return FileExists(path);+ }+ strlcpy(path, hip_path, PATH_MAX);+ strlcat(path, "/bin/", PATH_MAX);+ strlcat(path, name, PATH_MAX);+ if (FileExists(path)) {+ return true;} else {+ tinylog(__func__, ": note: ", path, " does not exist\n", NULL);return false;}}// Returns word-encoded array of 16-bit gfxXXXX gcnArchName numbers.-static bool GetAmdOffloadArchFlag(char out[static 64]) {+static bool get_amd_offload_arch_flag(char out[static 64]) {// Get hipInfo executable path.char hip_info_path[PATH_MAX];- if (!GetRocmBinPath(hip_info_path, "hipInfo") &&- !GetRocmBinPath(hip_info_path, "rocminfo")) {- tinylog("note: can't find hipInfo/rocminfo commands for AMD GPU detection\n", NULL);+ if (!get_rocm_bin_path(hip_info_path, "hipInfo") &&+ !get_rocm_bin_path(hip_info_path, "rocminfo")) {+ tinylog(__func__, ": warning: can't find hipInfo/rocminfo commands for AMD GPU detection\n", NULL);return false;}@@ -315,13 +337,13 @@ static bool GetAmdOffloadArchFlag(char out[static 64]) {}}if (ws) {- tinylog("error: hipInfo returned non-zero exit status\n", NULL);+ tinylog(__func__, ": error: hipInfo returned non-zero exit status\n", NULL);return false;}// Serialize value for --offload-arch=LIST flag.if (!archs) {- tinylog("warning: hipInfo output didn't list any graphics cards\n", NULL);+ tinylog(__func__, ": warning: hipInfo output didn't list any graphics cards\n", NULL);return false;}bool gotsome = false;@@ -344,27 +366,37 @@ static bool GetAmdOffloadArchFlag(char out[static 64]) {// 4. /usr/local/cuda/bin/nvcc//// set $CUDA_PATH to empty string to disable cuda-static bool GetNvccPath(char path[static PATH_MAX]) {- const char *cuda_path;- if (commandv(IsWindows() ? "nvcc.exe" : "nvcc", path, PATH_MAX)) {+static bool get_nvcc_path(char path[static PATH_MAX]) {+ const char *name = IsWindows() ? "nvcc.exe" : "nvcc";+ if (commandv(name, path, PATH_MAX)) {return true;- } else if ((cuda_path = getenv("CUDA_PATH"))) {+ } else {+ tinylog(__func__, ": note: ", name, " not found on $PATH\n", NULL);+ }+ const char *cuda_path;+ if ((cuda_path = getenv("CUDA_PATH"))) {if (!*cuda_path) return false;strlcpy(path, cuda_path, PATH_MAX);strlcat(path, "/bin/", PATH_MAX);- } else if (FileExists("/opt/cuda")) {- strlcpy(path, "/opt/cuda/bin/", PATH_MAX);} else {- strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);+ tinylog(__func__, ": note: $CUDA_PATH/bin/", name, " does not exist\n", NULL);+ if (FileExists("/opt/cuda")) {+ strlcpy(path, "/opt/cuda/bin/", PATH_MAX);+ } else {+ tinylog(__func__, ": note: /opt/cuda/bin/", name, " does not exist\n", NULL);+ strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);+ }}- strlcat(path, "nvcc", PATH_MAX);- if (IsWindows()) {- strlcat(path, ".exe", PATH_MAX);+ strlcat(path, name, PATH_MAX);+ if (IsExecutable(path)) {+ return true;+ } else {+ tinylog(__func__, ": note: ", path, " does not exist\n", NULL);+ return false;}- return IsExecutable(path);}-static dontinline bool GetNvccArchFlag(const char *nvcc, char flag[static 32]) {+static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]) {// create path of exechar exe[PATH_MAX];@@ -393,7 +425,7 @@ static dontinline bool GetNvccArchFlag(const char *nvcc, char flag[static 32]) {// nvidia-smi in cuda 11.5+ can do this but (1) it takes longer to// run than compiling / running this script and (2) the nvidia-smi// command isn't available on Jetson devices.- tinylog("building nvidia compute capability detector...\n", NULL);+ tinylog(__func__, ": note: building nvidia compute capability detector...\n", NULL);if (!Compile(src, tmp, exe, (char *[]){(char *)nvcc, "-o", tmp, src, 0})) {return false;}@@ -432,14 +464,14 @@ static dontinline bool GetNvccArchFlag(const char *nvcc, char flag[static 32]) {}}if (ws) {- tinylog("error: compute capability detector returned nonzero exit status\n", NULL);+ tinylog(__func__, ": error: compute capability detector returned nonzero exit status\n", NULL);return false;}// parse output of detectorchar *endptr;if (!*ibuf || !strtol(ibuf, &endptr, 10) || *endptr) {- tinylog("error: bad compute capability detector output\n", NULL);+ tinylog(__func__, ": error: bad compute capability detector output\n", NULL);return false;}@@ -448,13 +480,13 @@ static dontinline bool GetNvccArchFlag(const char *nvcc, char flag[static 32]) {return true;}-static bool CompileAmdWindows(const char *clangxx, const char *dso, const char *src, const char *tmpdso) {+static bool compile_amd_windows(const char *clangxx, const char *dso, const char *src, const char *tmpdso) {const char *lib = IsWindows() ? "lib" : GetDsoExtension();const char *hip_path = getenv("HIP_PATH");// get set of microarchitectures for all installed graphics cardschar offload_arch[64];- if (!GetAmdOffloadArchFlag(offload_arch)) {+ if (!get_amd_offload_arch_flag(offload_arch)) {unlink(tmpdso);return false;}@@ -497,13 +529,13 @@ static bool CompileAmdWindows(const char *clangxx, const char *dso, const char *return Compile(src, tmpdso, dso, args);}-static bool CompileAmdUnix(const char *dso, const char *src, const char *tmpdso) {+static bool compile_amd_unix(const char *dso, const char *src, const char *tmpdso) {// get set of microarchitectures for all installed graphics cards// it's possible to safe --offload-arch=native but we do it ourself// the amdgpu-arch that hipcc runs fails to link libhsa-runtime64.sochar offload_arch[64];- if (!GetAmdOffloadArchFlag(offload_arch)) {+ if (!get_amd_offload_arch_flag(offload_arch)) {strcpy(offload_arch, "--offload-arch=native");}@@ -537,7 +569,7 @@ static bool CompileAmdUnix(const char *dso, const char *src, const char *tmpdso)return Compile(src, tmpdso, dso, args);}-static bool CompileAmd(const char *clangxx, const char *dso, const char *src) {+static bool compile_amd(const char *clangxx, const char *dso, const char *src) {// create temporary output path for atomicitychar tmpdso[PATH_MAX];@@ -546,13 +578,13 @@ static bool CompileAmd(const char *clangxx, const char *dso, const char *src) {}if (!IsWindows()) {- return CompileAmdUnix(dso, src, tmpdso);+ return compile_amd_unix(dso, src, tmpdso);} else {- return CompileAmdWindows(clangxx, dso, src, tmpdso);+ return compile_amd_windows(clangxx, dso, src, tmpdso);}}-static bool CompileNvidia(const char *nvcc, const char *dso, const char *src) {+static bool compile_nvidia(const char *nvcc, const char *dso, const char *src) {// create temporary output path for atomicitychar tmpdso[PATH_MAX];@@ -561,7 +593,7 @@ static bool CompileNvidia(const char *nvcc, const char *dso, const char *src) {}// try building dso with host nvidia microarchitecture- tinylog("building ggml-cuda with nvcc -arch=native...\n", NULL);+ tinylog(__func__, ": note: building ggml-cuda with nvcc -arch=native...\n", NULL);if (Compile(src, tmpdso, dso, (char *[]){(char *)nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,(char *)src, NVCC_LIBS, NULL})) {@@ -570,10 +602,10 @@ static bool CompileNvidia(const char *nvcc, const char *dso, const char *src) {// try again with different arch flagchar archflag[32];- if (!GetNvccArchFlag(nvcc, archflag)) {+ if (!get_nvcc_arch_flag(nvcc, archflag)) {return false;}- tinylog("building ggml-cuda with nvcc ", archflag, "...\n", NULL);+ tinylog(__func__, ": note: building ggml-cuda with nvcc ", archflag, "...\n", NULL);if (Compile(src, tmpdso, dso, (char *[]){(char *)nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,(char *)src, NVCC_LIBS, NULL})) {@@ -584,7 +616,7 @@ static bool CompileNvidia(const char *nvcc, const char *dso, const char *src) {return false;}-static bool ExtractCudaDso(const char *dso, const char *name) {+static bool extract_cuda_dso(const char *dso, const char *name) {// see if prebuilt dso is bundled in zip assetschar zip[80];@@ -593,7 +625,7 @@ static bool ExtractCudaDso(const char *dso, const char *name) {strlcat(zip, ".", sizeof(zip));strlcat(zip, GetDsoExtension(), sizeof(zip));if (!FileExists(zip)) {- tinylog("prebuilt binary ", zip, " not found\n", NULL);+ tinylog(__func__, ": note: prebuilt binary ", zip, " not found\n", NULL);return false;}@@ -601,7 +633,7 @@ static bool ExtractCudaDso(const char *dso, const char *name) {return llamafile_extract(zip, dso);}-static bool LinkCudaDso(const char *dso, const char *dir) {+static bool link_cuda_dso(const char *dso, const char *dir) {// Change directory so BLAS library is more likely to be linked.char cwd[PATH_MAX];@@ -612,16 +644,16 @@ static bool LinkCudaDso(const char *dso, const char *dir) {// runtime link dynamic shared objectvoid *lib;- tinylog("dynamically linking ", dso, "\n", NULL);+ tinylog(__func__, ": note: dynamically linking ", dso, "\n", NULL);lib = cosmo_dlopen(dso, RTLD_LAZY);if (dir) {chdir(cwd);}if (!lib) {char cc[PATH_MAX];- tinylog(Dlerror(), ": failed to load library\n", NULL);+ tinylog(__func__, ": warning: ", Dlerror(), ": failed to load library\n", NULL);if ((IsLinux() || IsBsd()) && !commandv("cc", cc, PATH_MAX)) {- tinylog("you need to install cc for gpu support\n", NULL);+ tinylog(__func__, ": note: you need to install cc for gpu support\n", NULL);}return false;}@@ -654,7 +686,7 @@ static bool LinkCudaDso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.ggml_backend_cuda_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_buffer_type"));ok &= !!(ggml_cuda.ggml_backend_cuda_init = cosmo_dlsym(lib, "ggml_backend_cuda_init"));if (!ok) {- tinylog("error: not all cuda symbols could be imported\n", NULL);+ tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);return false;}@@ -663,7 +695,7 @@ static bool LinkCudaDso(const char *dso, const char *dir) {return true;}-static bool ImportCudaImpl(void) {+static bool import_cuda_impl(void) {assert(FLAG_gpu != LLAMAFILE_GPU_ERROR);// No dynamic linking support on OpenBSD yet.@@ -680,7 +712,7 @@ static bool ImportCudaImpl(void) {default:return false;}- tinylog("initializing gpu module...\n", NULL);+ tinylog(__func__, ": initializing gpu module...\n", NULL);// extract source codechar src[PATH_MAX];@@ -723,8 +755,8 @@ static bool ImportCudaImpl(void) {// Get some essential paths.// ROCm SDK puts BLAS DLLs in same folder as clang++- if (GetRocmBinPath(compiler_path_buf, "amdclang++") ||- GetRocmBinPath(compiler_path_buf, "clang++")) {+ if (get_rocm_bin_path(compiler_path_buf, "amdclang++") ||+ get_rocm_bin_path(compiler_path_buf, "clang++")) {strcpy(library_path_buf, compiler_path_buf);dirname(library_path_buf);compiler_path = compiler_path_buf;@@ -740,8 +772,8 @@ static bool ImportCudaImpl(void) {strlcat(dso, GetDsoExtension(), PATH_MAX);if (FLAG_nocompile) {if ((FileExists(dso) ||- ExtractCudaDso(dso, "ggml-rocm")) &&- LinkCudaDso(dso, library_path)) {+ extract_cuda_dso(dso, "ggml-rocm")) &&+ link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;@@ -754,7 +786,7 @@ static bool ImportCudaImpl(void) {case -1:return false;case false:- if (LinkCudaDso(dso, library_path)) {+ if (link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;@@ -768,20 +800,20 @@ static bool ImportCudaImpl(void) {// Try building CUDA with ROCm SDK.if (compiler_path) {- if (CompileAmd(compiler_path, dso, src)) {- if (LinkCudaDso(dso, library_path)) {+ if (compile_amd(compiler_path, dso, src)) {+ if (link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;}}} else {- tinylog("note: won't compile AMD GPU support because $HIP_PATH/bin/clang++ is missing\n", NULL);+ tinylog(__func__, ": won't compile AMD GPU support because $HIP_PATH/bin/clang++ is missing\n", NULL);}// Try extracting prebuilt tinyBLAS DSO from PKZIP.- if (ExtractCudaDso(dso, "ggml-rocm")) {- if (LinkCudaDso(dso, library_path)) {+ if (extract_cuda_dso(dso, "ggml-rocm")) {+ if (link_cuda_dso(dso, library_path)) {return true;} else {goto TryNvidia;@@ -801,7 +833,7 @@ TryNvidia:// Get some essential paths.// CUDA SDK puts cuBLAS DLL in same folder as NVCC- if (GetNvccPath(compiler_path_buf)) {+ if (get_nvcc_path(compiler_path_buf)) {strcpy(library_path_buf, compiler_path_buf);dirname(library_path_buf);compiler_path = compiler_path_buf;@@ -817,8 +849,8 @@ TryNvidia:strlcat(dso, GetDsoExtension(), PATH_MAX);if (FLAG_nocompile) {return ((FileExists(dso) ||- ExtractCudaDso(dso, "ggml-cuda")) &&- LinkCudaDso(dso, library_path));+ extract_cuda_dso(dso, "ggml-cuda")) &&+ link_cuda_dso(dso, library_path));}// Check if DSO is already compiled.@@ -827,7 +859,7 @@ TryNvidia:case -1:return false;case false:- return LinkCudaDso(dso, library_path);+ return link_cuda_dso(dso, library_path);case true:break;default:@@ -836,13 +868,13 @@ TryNvidia:}// Try building CUDA from source with mighty cuBLAS.- if (compiler_path && CompileNvidia(compiler_path, dso, src)) {- return LinkCudaDso(dso, library_path);+ if (compiler_path && compile_nvidia(compiler_path, dso, src)) {+ return link_cuda_dso(dso, library_path);}// Try extracting prebuilt tinyBLAS DSO from PKZIP.- if (ExtractCudaDso(dso, "ggml-cuda")) {- return LinkCudaDso(dso, library_path);+ if (extract_cuda_dso(dso, "ggml-cuda")) {+ return link_cuda_dso(dso, library_path);}break;@@ -854,12 +886,12 @@ TryNvidia:return false;}-static void ImportCuda(void) {+static void import_cuda(void) {if (ggml_metal_supported()) {return;}- if (ImportCudaImpl()) {- tinylog("GPU support successfully linked and loaded\n", NULL);+ if (import_cuda_impl()) {+ tinylog(__func__, ": GPU support successfully linked and loaded\n", NULL);ggml_cuda.supported = true;} else if (FLAG_gpu == LLAMAFILE_GPU_AMD ||FLAG_gpu == LLAMAFILE_GPU_NVIDIA) {@@ -871,7 +903,7 @@ static void ImportCuda(void) {}bool ggml_cuda_supported(void) {- cosmo_once(&ggml_cuda.once, ImportCuda);+ cosmo_once(&ggml_cuda.once, import_cuda);return ggml_cuda.supported;}diff --git a/llamafile/log.c b/llamafile/log.cindex 03ca3304..7d0b281e 100644--- a/llamafile/log.c+++ b/llamafile/log.c@@ -44,6 +44,7 @@ void (tinylog)(const char *s, ...) {}void llamafile_log_command(char *args[]) {+ tinylog(__func__, ": ", NULL);for (int i = 0; args[i]; ++i) {if (i) {tinylog(" ", NULL);