Skip to content

Distribute and run LLMs with a single file.

Make portability fixes

Justine Tunney3913d8858f9d
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index 57781c1a..9c710fdd 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -135,7 +135,6 @@
#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
#elif defined(GGML_USE_HIPBLAS)
-#define HIPBLAS_V2
#include <hip/hip_runtime.h>
#include <hipblas/hipblas.h>
#include <hip/hip_fp16.h>
@@ -143,8 +142,8 @@
// for rocblas_initialize()
#include "rocblas/rocblas.h"
#endif // __HIP_PLATFORM_AMD__
-#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
-#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F
+#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F
#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F
#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT
@@ -152,10 +151,10 @@
#define CUBLAS_OP_T HIPBLAS_OP_T
#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
#define CUBLAS_TF32_TENSOR_OP_MATH 0
-#define CUDA_R_16F HIP_R_16F
-#define CUDA_R_32F HIP_R_32F
+#define CUDA_R_16F HIPBLAS_R_16F
+#define CUDA_R_32F HIPBLAS_R_32F
#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
-#define cublasComputeType_t hipblasComputeType_t
+#define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6
#define cublasCreate hipblasCreate
#define cublasGemmEx hipblasGemmEx
#define cublasGemmBatchedEx hipblasGemmBatchedEx
@@ -166,7 +165,7 @@
#define cublasSgemm hipblasSgemm
#define cublasStatus_t hipblasStatus_t
#define cublasGetStatusString hipblasStatusToString
-#define cudaDataType_t hipDataType
+#define cudaDataType_t hipblasDatatype_t //deprecated, new hipblasDatatype not in 5.6
#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess
#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
diff --git a/llamafile/cuda.h b/llamafile/cuda.h
index 08ba0112..b9791b6e 100644
--- a/llamafile/cuda.h
+++ b/llamafile/cuda.h
@@ -9,6 +9,7 @@
#define __shfl(var, srcLane, warpSize) __shfl_sync(-1u, var, srcLane, warpSize)
#define __shfl_down(var, srcLane, warpSize) __shfl_down_sync(-1u, var, srcLane, warpSize)
#else
+#define HIPBLAS_V2
#include <hip/hip_fp16.h>
#include <hip/hip_runtime.h>
#include <hipblas/hipblas.h>
diff --git a/llamafile/gemm.h b/llamafile/gemm.h
index a26a3d32..09473156 100644
--- a/llamafile/gemm.h
+++ b/llamafile/gemm.h
@@ -53,7 +53,7 @@ void gemm(bool aT, bool bT, //
}
}
-// multiplies matrix on cpu with column row ordering
+// multiplies matrix on cpu with row major ordering
//
// k×m * n×k → n×m
// m×k * n×k → n×m if aᵀ
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 6488781a..d44fb6b2 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -387,8 +387,7 @@ static tinyblasStatus_t tinyblasGE_launcher(tinyblasHandle_t handle, tinyblasOpe
tinyblasOperation_t bT, int m, int n, int k, WORD alpha,
const SRC *A, int lda, const SRC *B, int ldb, WORD beta,
DST *C, int ldc) {
- constexpr int TT = 256, BM = 128, BN = 128, BK = 64, WM = 64, WN = 32, WNI = 1, TM = 8, TN = 4,
- REGS = 21504, SHARED = 16384; // THE BEAST
+ const int TT = 256, BM = 128, BN = 64, BK = 64, WM = 128, WN = 8, WNI = 1, TM = 8, TN = 4;
dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN));
if (!aT && bT && !(n % BN) && !(k % BK) && alpha == (WORD)1 && beta == (WORD)0) {
dim3 blocks(CEIL_DIV(n, BN), CEIL_DIV(m, BM));
diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cu
index a02b01f5..10c4e10b 100644
--- a/llamafile/tinyblas_test.cu
+++ b/llamafile/tinyblas_test.cu
@@ -15,6 +15,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "gemm.h"
#include "tester.h"
void checkTinyblasWorksHHHS() {
@@ -156,14 +157,16 @@ void try_size(int m, int n, int k) {
cuda_memory<half> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(1, m, n, k, G.p, ldc, C.p, ldc);
}
int main(int argc, char *argv[]) {
+ try_size(512, 512, 512);
try_size(5760, 1, 128); // mistral 7b
try_size(128, 1, 5696); // mistral 7b
try_size(14336, 512, 4096); // mistral 7b