Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Always use tinyBLAS with AMD GPUs on Windows
Justine Tunney60404a831edd
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 2dff6302..7653f8c4 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -10884,8 +10884,8 @@ static ggml_cuda_device_info ggml_cuda_init() {// Workaround for a rocBLAS bug when using multiple graphics cards:// https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346#ifndef GGML_USE_TINYBLAS- rocblas_initialize();- CUDA_CHECK(cudaDeviceSynchronize());+ // rocblas_initialize(); // already called+ // CUDA_CHECK(cudaDeviceSynchronize());#endif#endif@@ -13507,7 +13507,9 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_tGGML_ASSERT(stat == cudaSuccess);}// Launch graph+ printf("cudaGraphLaunch begin\n");CUDA_CHECK(cudaGraphLaunch(cuda_ctx->cuda_graph->instance, cuda_ctx->stream()));+ printf("cudaGraphLaunch done\n");#elsegraph_evaluated_or_captured = true;#endif // USE_CUDA_GRAPHdiff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 4c22b4d3..0834795e 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -559,7 +559,14 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char(char *)offload_arch,"-Wno-ignored-attributes","-D_CRT_SECURE_NO_WARNINGS",- COMMON_FLAGS,+ "-DGGML_BUILD=1",+ "-DGGML_SHARED=1",+ "-DGGML_MULTIPLATFORM",+ "-DGGML_CUDA_DMMV_X=32",+ "-DK_QUANTS_PER_ITERATION=2",+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128",+ "-DGGML_CUDA_MMV_Y=1",+ "-DGGML_USE_TINYBLAS","-o",(char *)tmpdso,(char *)src,@@ -571,10 +578,10 @@ static bool compile_amd_windows(const char *clangxx, const char *dso, const char"-amdgpu-early-inline-all=true","-isystem",gc(xasprintf("%s/include", hip_path)),- BLAS_ONLY("-l"),- BLAS_ONLY(gc(xasprintf("%s/lib/hipblas.%s", hip_path, lib))),- BLAS_ONLY("-l"),- BLAS_ONLY(gc(xasprintf("%s/lib/rocblas.%s", hip_path, lib))),+ /* BLAS_ONLY("-l"), */+ /* BLAS_ONLY(gc(xasprintf("%s/lib/hipblas.%s", hip_path, lib))), */+ /* BLAS_ONLY("-l"), */+ /* BLAS_ONLY(gc(xasprintf("%s/lib/rocblas.%s", hip_path, lib))), */"-l",gc(xasprintf("%s/lib/amdhip64.%s", hip_path, lib)),"-lkernel32",