Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make portability fixes
Justine Tunney3913d8858f9d
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 57781c1a..9c710fdd 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -135,7 +135,6 @@#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS#elif defined(GGML_USE_HIPBLAS)-#define HIPBLAS_V2#include <hip/hip_runtime.h>#include <hipblas/hipblas.h>#include <hip/hip_fp16.h>@@ -143,8 +142,8 @@// for rocblas_initialize()#include "rocblas/rocblas.h"#endif // __HIP_PLATFORM_AMD__-#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F-#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F+#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F+#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT@@ -152,10 +151,10 @@#define CUBLAS_OP_T HIPBLAS_OP_T#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS#define CUBLAS_TF32_TENSOR_OP_MATH 0-#define CUDA_R_16F HIP_R_16F-#define CUDA_R_32F HIP_R_32F+#define CUDA_R_16F HIPBLAS_R_16F+#define CUDA_R_32F HIPBLAS_R_32F#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)-#define cublasComputeType_t hipblasComputeType_t+#define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6#define cublasCreate hipblasCreate#define cublasGemmEx hipblasGemmEx#define cublasGemmBatchedEx hipblasGemmBatchedEx@@ -166,7 +165,7 @@#define cublasSgemm hipblasSgemm#define cublasStatus_t hipblasStatus_t#define cublasGetStatusString hipblasStatusToString-#define cudaDataType_t hipDataType+#define cudaDataType_t hipblasDatatype_t //deprecated, new hipblasDatatype not in 5.6#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccessdiff --git a/llamafile/cuda.h b/llamafile/cuda.hindex 08ba0112..b9791b6e 100644--- a/llamafile/cuda.h+++ b/llamafile/cuda.h@@ -9,6 +9,7 @@#define __shfl(var, srcLane, warpSize) __shfl_sync(-1u, var, srcLane, warpSize)#define __shfl_down(var, srcLane, warpSize) __shfl_down_sync(-1u, var, srcLane, warpSize)#else+#define HIPBLAS_V2#include <hip/hip_fp16.h>#include <hip/hip_runtime.h>#include <hipblas/hipblas.h>diff --git a/llamafile/gemm.h b/llamafile/gemm.hindex a26a3d32..09473156 100644--- a/llamafile/gemm.h+++ b/llamafile/gemm.h@@ -53,7 +53,7 @@ void gemm(bool aT, bool bT, //}}-// multiplies matrix on cpu with column row ordering+// multiplies matrix on cpu with row major ordering//// k×m * n×k → n×m// m×k * n×k → n×m if aᵀdiff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex 6488781a..d44fb6b2 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -387,8 +387,7 @@ static tinyblasStatus_t tinyblasGE_launcher(tinyblasHandle_t handle, tinyblasOpetinyblasOperation_t bT, int m, int n, int k, WORD alpha,const SRC *A, int lda, const SRC *B, int ldb, WORD beta,DST *C, int ldc) {- constexpr int TT = 256, BM = 128, BN = 128, BK = 64, WM = 64, WN = 32, WNI = 1, TM = 8, TN = 4,- REGS = 21504, SHARED = 16384; // THE BEAST+ const int TT = 256, BM = 128, BN = 64, BK = 64, WM = 128, WN = 8, WNI = 1, TM = 8, TN = 4;dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN));if (!aT && bT && !(n % BN) && !(k % BK) && alpha == (WORD)1 && beta == (WORD)0) {dim3 blocks(CEIL_DIV(n, BN), CEIL_DIV(m, BM));diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cuindex a02b01f5..10c4e10b 100644--- a/llamafile/tinyblas_test.cu+++ b/llamafile/tinyblas_test.cu@@ -15,6 +15,7 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "gemm.h"#include "tester.h"void checkTinyblasWorksHHHS() {@@ -156,14 +157,16 @@ void try_size(int m, int n, int k) {cuda_memory<half> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(1, m, n, k, G.p, ldc, C.p, ldc);}int main(int argc, char *argv[]) {+ try_size(512, 512, 512);try_size(5760, 1, 128); // mistral 7btry_size(128, 1, 5696); // mistral 7btry_size(14336, 512, 4096); // mistral 7b