Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Run clang-format on llamafile directory
Justine Tunneyc908b0fc6302
diff --git a/README.md b/README.mdindex a8427315..4b443b68 100644--- a/README.md+++ b/README.md@@ -42,7 +42,7 @@ chmod +x llava-v1.5-7b-q4.llamafile```6. Your browser should open automatically and display a chat interface.-(If it doesn't, just open your browser and point it at http://localhost:8080.)+(If it doesn't, just open your browser and point it at http://localhost:8080)7. When you're done chatting, return to your terminal and hit`Control-C` to shut down llamafile.diff --git a/llamafile/.clang-format b/llamafile/.clang-formatindex 86aa09f2..bd9a1fd1 100644--- a/llamafile/.clang-format+++ b/llamafile/.clang-format@@ -1,8 +1,11 @@---BasedOnStyle: LLVMIndentWidth: 4-ColumnLimit: 80+ColumnLimit: 100---Language: CppAllowShortFunctionsOnASingleLine: false+AlignTrailingComments: false+AlignEscapedNewlines: DontAlign+BreakAfterAttributes: Always---diff --git a/llamafile/addnl.c b/llamafile/addnl.cindex 8830eeac..b82842e1 100644--- a/llamafile/addnl.c+++ b/llamafile/addnl.c@@ -15,9 +15,9 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <time.h>#include <stdio.h>#include <string.h>+#include <time.h>#include <unistd.h>// example usage@@ -40,7 +40,8 @@ int main(int argc, char *argv[]) {struct timespec t = timespec_real();for (;;) {ssize_t rc = read(0, buf, sizeof(buf));- if (!rc) break;+ if (!rc)+ break;struct timespec u = timespec_real();struct timespec d = timespec_sub(u, t);t = u;diff --git a/llamafile/check_cpu.c b/llamafile/check_cpu.cindex f79b9bed..9631bd4f 100644--- a/llamafile/check_cpu.c+++ b/llamafile/check_cpu.c@@ -32,10 +32,8 @@ void llamafile_check_cpu(void) {}static int on_missing_feature(const char *name) {- tinyprint(- 2, program_invocation_name, ": fatal error: the cpu feature ", name,- " was required at build time but isn't available on this system\n",- NULL);+ tinyprint(2, program_invocation_name, ": fatal error: the cpu feature ", name,+ " was required at build time but isn't available on this system\n", NULL);#if defined(__AVX2__) && !defined(__AVX512F__)tinyprint(2,"note: amd microprocessors made after 2017 usually work\n"@@ -49,8 +47,7 @@ static int on_missing_feature(const char *name) {// We need to perform this early in the initialization process, before// C++ codes built with -mavx has a chance to start allocating dynamic// memory that would otherwise crash the cpu before this could be done-__attribute__((__constructor__(101))) static void-llamafile_actually_check_cpu(void) {+__attribute__((__constructor__(101))) static void llamafile_actually_check_cpu(void) {if (X86_NEED(SSE3) && !X86_CHECK(SSE3)) {on_missing_feature("SSE3");}diff --git a/llamafile/compcap.cu b/llamafile/compcap.cuindex 5fe88ad3..c7d0cc0f 100644--- a/llamafile/compcap.cu+++ b/llamafile/compcap.cu@@ -4,27 +4,30 @@#include <cuda_runtime_api.h>int main(int argc, char *argv[]) {- cudaDeviceProp prop;- cudaError_t status;- int device_count;- int device_index = 0;- if (argc > 1) {- device_index = atoi(argv[1]);- }- status = cudaGetDeviceCount(&device_count);- if (status != cudaSuccess) {- fprintf(stderr, "cudaGetDeviceCount() failed: %s\n", cudaGetErrorString(status));- return -1;- }- if (device_index >= device_count) {- fprintf(stderr, "Specified device index %d exceeds the maximum (the device count on this system is %d)\n", device_index, device_count);- return -1;- }- status = cudaGetDeviceProperties(&prop, device_index);- if (status != cudaSuccess) {- fprintf(stderr, "cudaGetDeviceProperties() for device device_index failed: %s\n", cudaGetErrorString(status));- return -1;- }- int v = prop.major * 10 + prop.minor;- printf("%d", v);+ cudaDeviceProp prop;+ cudaError_t status;+ int device_count;+ int device_index = 0;+ if (argc > 1)+ device_index = atoi(argv[1]);+ status = cudaGetDeviceCount(&device_count);+ if (status != cudaSuccess) {+ fprintf(stderr, "cudaGetDeviceCount() failed: %s\n", cudaGetErrorString(status));+ return -1;+ }+ if (device_index >= device_count) {+ fprintf(stderr,+ "Specified device index %d exceeds the maximum (the device "+ "count on this system is %d)\n",+ device_index, device_count);+ return -1;+ }+ status = cudaGetDeviceProperties(&prop, device_index);+ if (status != cudaSuccess) {+ fprintf(stderr, "cudaGetDeviceProperties() for device device_index failed: %s\n",+ cudaGetErrorString(status));+ return -1;+ }+ int v = prop.major * 10 + prop.minor;+ printf("%d", v);}diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 8aed849d..e49dbf57 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -55,18 +55,14 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");#define NVCC_LIBS BLAS_ONLY("-lcublas"), "-lcuda"-#define NVCC_FLAGS \- "--shared", "--forward-unknown-to-host-compiler", "-use_fast_math", \- "--compiler-options", \- (!IsWindows() \- ? (!IsAarch64() \- ? "-fPIC -O3 -march=native -mtune=native" \- : "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \- : "/nologo /EHsc /O2 /GR /MT"), \- "-DNDEBUG", "-DGGML_BUILD=1", "-DGGML_SHARED=1", \- "-DGGML_CUDA_MMV_Y=1", "-DGGML_MULTIPLATFORM", \- "-DGGML_CUDA_DMMV_X=32", "-DK_QUANTS_PER_ITERATION=2", \- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+#define NVCC_FLAGS \+ "--shared", "--forward-unknown-to-host-compiler", "-use_fast_math", "--compiler-options", \+ (!IsWindows() ? (!IsAarch64() ? "-fPIC -O3 -march=native -mtune=native" \+ : "-fPIC -O3 -march=native -mtune=native -ffixed-x28") \+ : "/nologo /EHsc /O2 /GR /MT"), \+ "-DNDEBUG", "-DGGML_BUILD=1", "-DGGML_SHARED=1", "-DGGML_CUDA_MMV_Y=1", \+ "-DGGML_MULTIPLATFORM", "-DGGML_CUDA_DMMV_X=32", "-DK_QUANTS_PER_ITERATION=2", \+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \(FLAG_tinyblas ? "-DGGML_USE_TINYBLAS" : "-DGGML_USE_CUBLAS")static const struct Source {@@ -118,13 +114,12 @@ static const char *Dlerror(void) {}static const char *GetDsoExtension(void) {- if (IsWindows()) {+ if (IsWindows())return "dll";- } else if (IsXnu()) {+ else if (IsXnu())return "dylib";- } else {+ elsereturn "so";- }}static bool FileExists(const char *path) {@@ -150,8 +145,7 @@ static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {}}-static bool Compile(const char *src, const char *tmp, const char *out,- char *args[]) {+static bool Compile(const char *src, const char *tmp, const char *out, char *args[]) {int pid, ws;llamafile_log_command(args);errno_t err = posix_spawnp(&pid, args[0], NULL, NULL, args, environ);@@ -168,8 +162,7 @@ static bool Compile(const char *src, const char *tmp, const char *out,}}if (ws) {- tinylog(__func__, ": warning: ", args[0],- " returned nonzero exit status\n", NULL);+ tinylog(__func__, ": warning: ", args[0], " returned nonzero exit status\n", NULL);unlink(tmp);return false;}@@ -186,26 +179,22 @@ static bool get_rocm_bin_path(char path[static PATH_MAX], const char *bin) {// create filename of executablechar name[NAME_MAX];strlcpy(name, bin, PATH_MAX);- if (IsWindows()) {+ if (IsWindows())strlcat(name, ".exe", PATH_MAX);- }// search for it on $PATH- if (commandv(name, path, PATH_MAX)) {+ if (commandv(name, path, PATH_MAX))return path;- } else {+ elsetinylog(__func__, ": note: ", name, " not found on $PATH\n", NULL);- }// 1. use $HIP_PATH/bin/$name if it exists// 2. use /opt/rocm/bin/$name if it existsconst char *hip_path = getenv("HIP_PATH");if (!hip_path) {- tinylog(__func__, ": note: $HIP_PATH/bin/", name, " does not exist\n",- NULL);+ tinylog(__func__, ": note: $HIP_PATH/bin/", name, " does not exist\n", NULL);if (!FileExists((hip_path = "/opt/rocm"))) {- tinylog(__func__, ": note: /opt/rocm/bin/", name,- " does not exist\n", NULL);+ tinylog(__func__, ": note: /opt/rocm/bin/", name, " does not exist\n", NULL);return false;}}@@ -268,16 +257,14 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {for (int i = 0; i < rc; ++i) {switch (t) {case 0:- if (buf[i] == 'g') {+ if (buf[i] == 'g')t = 1;- }break;case 1:- if (buf[i] == 'f') {+ if (buf[i] == 'f')t = 2;- } else {+ elset = 0;- }break;case 2:if (buf[i] == 'x') {@@ -324,16 +311,13 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {}}if (ws) {- tinylog(__func__, ": error: hipInfo returned non-zero exit status\n",- NULL);+ tinylog(__func__, ": error: hipInfo returned non-zero exit status\n", NULL);return false;}// Serialize value for --offload-arch=LIST flag.if (!archs) {- tinylog(__func__,- ": warning: hipInfo output didn't list any graphics cards\n",- NULL);+ tinylog(__func__, ": warning: hipInfo output didn't list any graphics cards\n", NULL);return false;}bool gotsome = false;@@ -359,11 +343,10 @@ static bool get_amd_offload_arch_flag(char out[static 64]) {// set $CUDA_PATH to empty string to disable cudastatic bool get_nvcc_path(char path[static PATH_MAX]) {const char *name = IsWindows() ? "nvcc.exe" : "nvcc";- if (commandv(name, path, PATH_MAX)) {+ if (commandv(name, path, PATH_MAX))return true;- } else {+ elsetinylog(__func__, ": note: ", name, " not found on $PATH\n", NULL);- }const char *cuda_path;if ((cuda_path = getenv("CUDA_PATH"))) {if (!*cuda_path)@@ -371,13 +354,11 @@ static bool get_nvcc_path(char path[static PATH_MAX]) {strlcpy(path, cuda_path, PATH_MAX);strlcat(path, "/bin/", PATH_MAX);} else {- tinylog(__func__, ": note: $CUDA_PATH/bin/", name, " does not exist\n",- NULL);+ tinylog(__func__, ": note: $CUDA_PATH/bin/", name, " does not exist\n", NULL);if (FileExists("/opt/cuda")) {strlcpy(path, "/opt/cuda/bin/", PATH_MAX);} else {- tinylog(__func__, ": note: /opt/cuda/bin/", name,- " does not exist\n", NULL);+ tinylog(__func__, ": note: /opt/cuda/bin/", name, " does not exist\n", NULL);strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);}}@@ -390,8 +371,7 @@ static bool get_nvcc_path(char path[static PATH_MAX]) {}}-static dontinline bool get_nvcc_arch_flag(const char *nvcc,- char flag[static 32]) {+static dontinline bool get_nvcc_arch_flag(const char *nvcc, char flag[static 32]) {// create path of exechar exe[PATH_MAX];@@ -405,9 +385,8 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc,// create temporary path for outputchar tmp[PATH_MAX];- if (!CreateTempPath(exe, tmp)) {+ if (!CreateTempPath(exe, tmp))return false;- }// build nvidia compute capability detector// https://stackoverflow.com/a/40695640/1653720@@ -420,11 +399,9 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc,// nvidia-smi in cuda 11.5+ can do this but (1) it takes longer to// run than compiling / running this script and (2) the nvidia-smi// command isn't available on Jetson devices.- tinylog(__func__,- ": note: building nvidia compute capability detector...\n", NULL);- if (!Compile(src, tmp, exe, (char *[]){(char *)nvcc, "-o", tmp, src, 0})) {+ tinylog(__func__, ": note: building nvidia compute capability detector...\n", NULL);+ if (!Compile(src, tmp, exe, (char *[]){(char *)nvcc, "-o", tmp, src, 0}))return false;- }// create pipeint pipefds[2];@@ -470,8 +447,7 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc,// parse output of detectorchar *endptr;if (!*ibuf || !strtol(ibuf, &endptr, 10) || *endptr) {- tinylog(__func__, ": error: bad compute capability detector output\n",- NULL);+ tinylog(__func__, ": error: bad compute capability detector output\n", NULL);return false;}@@ -480,8 +456,8 @@ static dontinline bool get_nvcc_arch_flag(const char *nvcc,return true;}-static bool compile_amd_windows(const char *clangxx, const char *dso,- const char *src, const char *tmpdso) {+static bool compile_amd_windows(const char *clangxx, const char *dso, const char *src,+ const char *tmpdso) {const char *lib = IsWindows() ? "lib" : GetDsoExtension();const char *hip_path = getenv("HIP_PATH");@@ -540,16 +516,14 @@ static bool compile_amd_windows(const char *clangxx, const char *dso,return Compile(src, tmpdso, dso, args);}-static bool compile_amd_unix(const char *dso, const char *src,- const char *tmpdso) {+static bool compile_amd_unix(const char *dso, const char *src, const char *tmpdso) {// get set of microarchitectures for all installed graphics cards// it's possible to safe --offload-arch=native but we do it ourself// the amdgpu-arch that hipcc runs fails to link libhsa-runtime64.sochar offload_arch[64];- if (!get_amd_offload_arch_flag(offload_arch)) {+ if (!get_amd_offload_arch_flag(offload_arch))strcpy(offload_arch, "--offload-arch=native");- }char *args[] = {"hipcc",@@ -587,46 +561,38 @@ static bool compile_amd(const char *clangxx, const char *dso, const char *src) {// create temporary output path for atomicitychar tmpdso[PATH_MAX];- if (!CreateTempPath(dso, tmpdso)) {+ if (!CreateTempPath(dso, tmpdso))return false;- }- if (!IsWindows()) {+ if (!IsWindows())return compile_amd_unix(dso, src, tmpdso);- } else {+ elsereturn compile_amd_windows(clangxx, dso, src, tmpdso);- }}static bool compile_nvidia(const char *nvcc, const char *dso, const char *src) {// create temporary output path for atomicitychar tmpdso[PATH_MAX];- if (!CreateTempPath(dso, tmpdso)) {+ if (!CreateTempPath(dso, tmpdso))return false;- }// try building dso with host nvidia microarchitecture- tinylog(__func__, ": note: building ggml-cuda with nvcc -arch=native...\n",- NULL);+ tinylog(__func__, ": note: building ggml-cuda with nvcc -arch=native...\n", NULL);if (Compile(src, tmpdso, dso,- (char *[]){(char *)nvcc, "-arch=native", NVCC_FLAGS, "-o",- tmpdso, (char *)src, NVCC_LIBS, NULL})) {+ (char *[]){(char *)nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso, (char *)src,+ NVCC_LIBS, NULL}))return true;- }// try again with different arch flagchar archflag[32];- if (!get_nvcc_arch_flag(nvcc, archflag)) {+ if (!get_nvcc_arch_flag(nvcc, archflag))return false;- }- tinylog(__func__, ": note: building ggml-cuda with nvcc ", archflag,- "...\n", NULL);+ tinylog(__func__, ": note: building ggml-cuda with nvcc ", archflag, "...\n", NULL);if (Compile(src, tmpdso, dso,- (char *[]){(char *)nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,- (char *)src, NVCC_LIBS, NULL})) {+ (char *[]){(char *)nvcc, archflag, NVCC_FLAGS, "-o", tmpdso, (char *)src, NVCC_LIBS,+ NULL}))return true;- }// oh noreturn false;@@ -641,8 +607,7 @@ static bool extract_cuda_dso(const char *dso, const char *name) {strlcat(zip, ".", sizeof(zip));strlcat(zip, GetDsoExtension(), sizeof(zip));if (!FileExists(zip)) {- tinylog(__func__, ": note: prebuilt binary ", zip, " not found\n",- NULL);+ tinylog(__func__, ": note: prebuilt binary ", zip, " not found\n", NULL);return false;}@@ -652,10 +617,8 @@ static bool extract_cuda_dso(const char *dso, const char *name) {static void *imp(void *lib, const char *sym) {void *fun = cosmo_dlsym(lib, sym);- if (!fun) {- tinylog(__func__, ": error: failed to import symbol: ", sym, "\n",- NULL);- }+ if (!fun)+ tinylog(__func__, ": error: failed to import symbol: ", sym, "\n", NULL);return fun;}@@ -672,17 +635,13 @@ static bool link_cuda_dso(const char *dso, const char *dir) {void *lib;tinylog(__func__, ": note: dynamically linking ", dso, "\n", NULL);lib = cosmo_dlopen(dso, RTLD_LAZY);- if (dir) {+ if (dir)chdir(cwd);- }if (!lib) {char cc[PATH_MAX];- tinylog(__func__, ": warning: ", Dlerror(),- ": failed to load library\n", NULL);- if ((IsLinux() || IsBsd()) && !commandv("cc", cc, PATH_MAX)) {- tinylog(__func__,- ": note: you need to install cc for gpu support\n", NULL);- }+ tinylog(__func__, ": warning: ", Dlerror(), ": failed to load library\n", NULL);+ if ((IsLinux() || IsBsd()) && !commandv("cc", cc, PATH_MAX))+ tinylog(__func__, ": note: you need to install cc for gpu support\n", NULL);return false;}@@ -695,24 +654,17 @@ static bool link_cuda_dso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.host_malloc = imp(lib, "ggml_cuda_host_malloc"));ok &= !!(ggml_cuda.can_mul_mat = imp(lib, "ggml_cuda_can_mul_mat"));ok &= !!(ggml_cuda.compute_forward = imp(lib, "ggml_cuda_compute_forward"));- ok &=- !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));- ok &= !!(ggml_cuda.get_device_description =- imp(lib, "ggml_cuda_get_device_description"));- ok &= !!(ggml_cuda.host_buffer_type =- imp(lib, "ggml_backend_cuda_host_buffer_type"));+ ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_cuda_get_device_count"));+ ok &= !!(ggml_cuda.get_device_description = imp(lib, "ggml_cuda_get_device_description"));+ ok &= !!(ggml_cuda.host_buffer_type = imp(lib, "ggml_backend_cuda_host_buffer_type"));ok &= !!(ggml_cuda.buffer_type = imp(lib, "ggml_backend_cuda_buffer_type"));ok &= !!(ggml_cuda.backend_init = imp(lib, "ggml_backend_cuda_init"));- ok &= !!(ggml_cuda.get_device_count2 =- imp(lib, "ggml_backend_cuda_get_device_count"));- ok &= !!(ggml_cuda.split_buffer_type =- imp(lib, "ggml_backend_cuda_split_buffer_type"));+ ok &= !!(ggml_cuda.get_device_count2 = imp(lib, "ggml_backend_cuda_get_device_count"));+ ok &= !!(ggml_cuda.split_buffer_type = imp(lib, "ggml_backend_cuda_split_buffer_type"));ok &= !!(ggml_cuda.reg_devices = imp(lib, "ggml_backend_cuda_reg_devices"));- ok &= !!(ggml_cuda.get_device_memory =- imp(lib, "ggml_backend_cuda_get_device_memory"));+ ok &= !!(ggml_cuda.get_device_memory = imp(lib, "ggml_backend_cuda_get_device_memory"));if (!ok) {- tinylog(__func__, ": error: not all cuda symbols could be imported\n",- NULL);+ tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);cosmo_dlclose(lib);return false;}@@ -734,9 +686,8 @@ static bool link_cuda_dso(const char *dso, const char *dir) {static bool import_cuda_impl(void) {// No dynamic linking support on OpenBSD yet.- if (IsOpenbsd()) {+ if (IsOpenbsd())return false;- }// Check if we're allowed to even try.switch (FLAG_gpu) {@@ -766,9 +717,8 @@ static bool import_cuda_impl(void) {break;case true:needs_rebuild = true;- if (!llamafile_extract(srcs[i].zip, src)) {+ if (!llamafile_extract(srcs[i].zip, src))return false;- }break;default:__builtin_unreachable();@@ -820,11 +770,10 @@ static bool import_cuda_impl(void) {case -1:return false;case false:- if (link_cuda_dso(dso, library_path)) {+ if (link_cuda_dso(dso, library_path))return true;- } else {+ elsegoto TryNvidia;- }case true:break;default:@@ -835,11 +784,10 @@ static bool import_cuda_impl(void) {// Try building CUDA with ROCm SDK.if (compiler_path) {if (compile_amd(compiler_path, dso, src)) {- if (link_cuda_dso(dso, library_path)) {+ if (link_cuda_dso(dso, library_path))return true;- } else {+ elsegoto TryNvidia;- }}} else {tinylog(__func__,@@ -850,11 +798,10 @@ static bool import_cuda_impl(void) {// Try extracting prebuilt tinyBLAS DSO from PKZIP.if (extract_cuda_dso(dso, "ggml-rocm")) {- if (link_cuda_dso(dso, library_path)) {+ if (link_cuda_dso(dso, library_path))return true;- } else {+ elsegoto TryNvidia;- }}break;@@ -884,10 +831,9 @@ TryNvidia:llamafile_get_app_dir(dso, PATH_MAX);strlcat(dso, "ggml-cuda.", PATH_MAX);strlcat(dso, GetDsoExtension(), PATH_MAX);- if (FLAG_nocompile) {+ if (FLAG_nocompile)return ((FileExists(dso) || extract_cuda_dso(dso, "ggml-cuda")) &&link_cuda_dso(dso, library_path));- }// Check if DSO is already compiled.if (!needs_rebuild && !FLAG_recompile) {@@ -904,14 +850,12 @@ TryNvidia:}// Try building CUDA from source with mighty cuBLAS.- if (compiler_path && compile_nvidia(compiler_path, dso, src)) {+ if (compiler_path && compile_nvidia(compiler_path, dso, src))return link_cuda_dso(dso, library_path);- }// Try extracting prebuilt tinyBLAS DSO from PKZIP.- if (extract_cuda_dso(dso, "ggml-cuda")) {+ if (extract_cuda_dso(dso, "ggml-cuda"))return link_cuda_dso(dso, library_path);- }break;default:@@ -923,15 +867,13 @@ TryNvidia:}static void import_cuda(void) {- if (llamafile_has_metal()) {+ if (llamafile_has_metal())return;- }if (import_cuda_impl()) {ggml_cuda.supported = true;- } else if (FLAG_gpu == LLAMAFILE_GPU_AMD ||- FLAG_gpu == LLAMAFILE_GPU_NVIDIA) {- tinyprint(2, "fatal error: support for --gpu ",- llamafile_describe_gpu(), FLAG_tinyblas ? " --tinyblas" : "",+ } else if (FLAG_gpu == LLAMAFILE_GPU_AMD || FLAG_gpu == LLAMAFILE_GPU_NVIDIA) {+ tinyprint(2, "fatal error: support for --gpu ", llamafile_describe_gpu(),+ FLAG_tinyblas ? " --tinyblas" : ""," was explicitly requested, but it wasn't available\n", NULL);exit(1);}@@ -966,8 +908,7 @@ GGML_CALL void ggml_cuda_host_free(void *data) {return ggml_cuda.host_free(data);}-GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0,- const struct ggml_tensor *src1,+GGML_CALL bool ggml_cuda_can_mul_mat(const struct ggml_tensor *src0, const struct ggml_tensor *src1,struct ggml_tensor *dst) {if (!llamafile_has_cuda())return false;@@ -991,8 +932,7 @@ GGML_CALL void ggml_cuda_get_device_description(int device, char *description,size_t description_size) {if (!llamafile_has_cuda())return;- return ggml_cuda.get_device_description(device, description,- description_size);+ return ggml_cuda.get_device_description(device, description, description_size);}GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {@@ -1032,8 +972,7 @@ GGML_CALL int ggml_backend_cuda_reg_devices(void) {return ggml_cuda.reg_devices();}-GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *free,- size_t *total) {+GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t *free, size_t *total) {if (!llamafile_has_cuda())return;return ggml_cuda.get_device_memory(device, free, total);diff --git a/llamafile/get_tmp_dir.c b/llamafile/get_tmp_dir.cindex 22ce6ca1..709f893d 100644--- a/llamafile/get_tmp_dir.c+++ b/llamafile/get_tmp_dir.c@@ -20,10 +20,8 @@const char *llamafile_get_tmp_dir(void) {const char *tmpdir;- if (!(tmpdir = getenv("TMPDIR")) || !*tmpdir) {- if (!(tmpdir = getenv("HOME")) || !*tmpdir) {+ if (!(tmpdir = getenv("TMPDIR")) || !*tmpdir)+ if (!(tmpdir = getenv("HOME")) || !*tmpdir)tmpdir = ".";- }- }return tmpdir;}diff --git a/llamafile/gpu.c b/llamafile/gpu.cindex 03884533..dd2c7d67 100644--- a/llamafile/gpu.c+++ b/llamafile/gpu.c@@ -15,14 +15,14 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-metal.h"#include "llamafile.h"+#include "llamafile/log.h"+#include <limits.h>+#include <stdlib.h>#include <string.h>#include <unistd.h>-#include <stdlib.h>-#include <limits.h>-#include "llama.cpp/ggml-cuda.h"-#include "llamafile/log.h"-#include "llama.cpp/ggml-metal.h"int FLAG_gpu;bool FLAG_nogpu;@@ -32,18 +32,18 @@ bool FLAG_recompile;const char *llamafile_describe_gpu(void) {switch (FLAG_gpu) {- case LLAMAFILE_GPU_AUTO:- return "auto";- case LLAMAFILE_GPU_AMD:- return "amd";- case LLAMAFILE_GPU_APPLE:- return "apple";- case LLAMAFILE_GPU_NVIDIA:- return "nvidia";- case LLAMAFILE_GPU_DISABLE:- return "disabled";- default:- return "error";+ case LLAMAFILE_GPU_AUTO:+ return "auto";+ case LLAMAFILE_GPU_AMD:+ return "amd";+ case LLAMAFILE_GPU_APPLE:+ return "apple";+ case LLAMAFILE_GPU_NVIDIA:+ return "nvidia";+ case LLAMAFILE_GPU_DISABLE:+ return "disabled";+ default:+ return "error";}}@@ -61,29 +61,28 @@ int llamafile_gpu_layers(int n_gpu_layers) {// if user explicitly passed `--gpu KIND` but didn't specify `-ngl// LAYERS` then assume the user wants their model fully offloaded.- if (n_gpu_layers < 0 && FLAG_gpu > 0) {+ if (n_gpu_layers < 0 && FLAG_gpu > 0)n_gpu_layers = INT_MAX;- }// Apple Metal is safe enough to enable by default.- if (n_gpu_layers <= 0 && llamafile_has_metal()) {+ if (n_gpu_layers <= 0 && llamafile_has_metal())n_gpu_layers = INT_MAX;- }// make the -ngl flag not break example code when it's impossible// if you want it to be an error just pass --gpu apple/amd/nvidiaif (n_gpu_layers > 0 && !llamafile_has_gpu()) {tinylogf("warning: --n-gpu-layers %d was passed but no GPUs were found;"- " falling back to CPU inference\n", n_gpu_layers);+ " falling back to CPU inference\n",+ n_gpu_layers);FLAG_gpu = LLAMAFILE_GPU_DISABLE;n_gpu_layers = 0;}// don't bother linking gpu modules if zero layers are offloadedif (n_gpu_layers <= 0) {- if (n_gpu_layers == -1) {- tinylogf("note: if you have an AMD or NVIDIA GPU then you need to pass -ngl 9999 to enable GPU offloading\n");- }+ if (n_gpu_layers == -1)+ tinylogf("note: if you have an AMD or NVIDIA GPU then you need to pass -ngl 9999 to "+ "enable GPU offloading\n");FLAG_gpu = LLAMAFILE_GPU_DISABLE;}@@ -97,19 +96,30 @@ int llamafile_gpu_layers(int n_gpu_layers) {int llamafile_gpu_parse(const char *s) {// Parse canonical names for GPUs.- if (!strcasecmp(s, "disable")) return LLAMAFILE_GPU_DISABLE;- if (!strcasecmp(s, "auto")) return LLAMAFILE_GPU_AUTO;- if (!strcasecmp(s, "amd")) return LLAMAFILE_GPU_AMD;- if (!strcasecmp(s, "apple")) return LLAMAFILE_GPU_APPLE;- if (!strcasecmp(s, "nvidia")) return LLAMAFILE_GPU_NVIDIA;+ if (!strcasecmp(s, "disable"))+ return LLAMAFILE_GPU_DISABLE;+ if (!strcasecmp(s, "auto"))+ return LLAMAFILE_GPU_AUTO;+ if (!strcasecmp(s, "amd"))+ return LLAMAFILE_GPU_AMD;+ if (!strcasecmp(s, "apple"))+ return LLAMAFILE_GPU_APPLE;+ if (!strcasecmp(s, "nvidia"))+ return LLAMAFILE_GPU_NVIDIA;// Parse aliases.- if (!strcasecmp(s, "disabled")) return LLAMAFILE_GPU_DISABLE;- if (!strcasecmp(s, "metal")) return LLAMAFILE_GPU_APPLE;- if (!strcasecmp(s, "cublas")) return LLAMAFILE_GPU_NVIDIA;- if (!strcasecmp(s, "rocblas")) return LLAMAFILE_GPU_AMD;- if (!strcasecmp(s, "rocm")) return LLAMAFILE_GPU_AMD;- if (!strcasecmp(s, "hip")) return LLAMAFILE_GPU_AMD;+ if (!strcasecmp(s, "disabled"))+ return LLAMAFILE_GPU_DISABLE;+ if (!strcasecmp(s, "metal"))+ return LLAMAFILE_GPU_APPLE;+ if (!strcasecmp(s, "cublas"))+ return LLAMAFILE_GPU_NVIDIA;+ if (!strcasecmp(s, "rocblas"))+ return LLAMAFILE_GPU_AMD;+ if (!strcasecmp(s, "rocm"))+ return LLAMAFILE_GPU_AMD;+ if (!strcasecmp(s, "hip"))+ return LLAMAFILE_GPU_AMD;return LLAMAFILE_GPU_ERROR;}diff --git a/llamafile/has.c b/llamafile/has.cindex df882f0b..617ffed2 100644--- a/llamafile/has.c+++ b/llamafile/has.c@@ -19,10 +19,8 @@#include <string.h>bool llamafile_has(char **a, const char *x) {- for (int i = 0; a[i]; ++i) {- if (!strcmp(a[i], x)) {+ for (int i = 0; a[i]; ++i)+ if (!strcmp(a[i], x))return true;- }- }return false;}diff --git a/llamafile/init.c b/llamafile/init.cindex 6104df3b..626a9b74 100644--- a/llamafile/init.c+++ b/llamafile/init.c@@ -15,24 +15,5 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <cosmo.h>-#include <stdlib.h>-#include <string.h>-void llamafile_init(void) {-- // todo(jart): cosmo runtime should do this- if (IsWindows()) {- static char home[256];- if (!getenv("HOME")) {- const char *homepath = getenv("HOMEPATH");- const char *homedrive = getenv("HOMEDRIVE");- if (homedrive && homepath) {- strlcpy(home, "HOME=", sizeof(home));- strlcat(home, homedrive, sizeof(home));- strlcat(home, homepath, sizeof(home));- putenv(home);- }- }- }}diff --git a/llamafile/is_file_newer_than.c b/llamafile/is_file_newer_than.cindex e05b6019..e13789b6 100644--- a/llamafile/is_file_newer_than.c+++ b/llamafile/is_file_newer_than.c@@ -15,15 +15,15 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <time.h>+#include "llamafile.h"+#include <errno.h>#include <fcntl.h>#include <stdio.h>-#include <errno.h>#include <stdlib.h>#include <string.h>-#include <unistd.h>#include <sys/stat.h>-#include "llamafile.h"+#include <time.h>+#include <unistd.h>static int is_file_newer_than_time(const char *path, const char *other) {struct stat st1, st2;@@ -80,17 +80,15 @@ static int is_file_newer_than_bytes(const char *path, const char *other) {break;}if (!path_rc || !other_rc) {- if (!path_rc && !other_rc) {+ if (!path_rc && !other_rc)res = false;- } else {+ elseres = true;- }break;}size_t size = path_rc;- if (other_rc < path_rc) {+ if (other_rc < path_rc)size = other_rc;- }if (memcmp(path_buf, other_buf, size)) {res = true;break;@@ -117,11 +115,10 @@ static int is_file_newer_than_bytes(const char *path, const char *other) {* would be the generated artifact that's dependent on `path`.*/int llamafile_is_file_newer_than(const char *path, const char *other) {- if (startswith(path, "/zip/")) {+ if (startswith(path, "/zip/"))// to keep builds deterministic, embedded zip files always have// the same timestamp from back in 2022 when it was implementedreturn is_file_newer_than_bytes(path, other);- } else {+ elsereturn is_file_newer_than_time(path, other);- }}diff --git a/llamafile/launch_browser.c b/llamafile/launch_browser.cindex f9359af9..4af08d84 100644--- a/llamafile/launch_browser.c+++ b/llamafile/launch_browser.c@@ -16,34 +16,30 @@// limitations under the License.#include "llamafile.h"+#include "llamafile/log.h"#include <cosmo.h>-#include <spawn.h>#include <errno.h>+#include <signal.h>+#include <spawn.h>#include <stdio.h>#include <stdlib.h>#include <string.h>-#include <unistd.h>-#include <signal.h>#include <sys/wait.h>-#include "llamafile/log.h"+#include <unistd.h>static volatile bool g_timed_out;static void finish(void) {- if (!IsWindows()) {+ if (!IsWindows())_exit(0);- }}static void handle_timeout(int sig) {g_timed_out = true;}-static void report_failure(const char *url,- const char *cmd,- const char *reason) {- tinylog("failed to open ", url, " in a browser tab using ", cmd,- ": ", reason, "\n", NULL);+static void report_failure(const char *url, const char *cmd, const char *reason) {+ tinylog("failed to open ", url, " in a browser tab using ", cmd, ": ", reason, "\n", NULL);}/**@@ -55,13 +51,13 @@ void llamafile_launch_browser(const char *url) {tinylog("opening browser tab... (pass --nobrowser to disable)\n", NULL);if (!IsWindows()) {switch (fork()) {- case 0:- break;- default:- return;- case -1:- perror("fork");- return;+ case 0:+ break;+ default:+ return;+ case -1:+ perror("fork");+ return;}}@@ -113,9 +109,8 @@ void llamafile_launch_browser(const char *url) {return finish();}}- if (ws) {+ if (ws)report_failure(url, cmd, "process exited with non-zero status");- }// we're donereturn finish();diff --git a/llamafile/llamafile.c b/llamafile/llamafile.cindex 883dca6c..ad29b96a 100644--- a/llamafile/llamafile.c+++ b/llamafile/llamafile.c@@ -16,35 +16,34 @@// limitations under the License.#include "llamafile.h"+#include "zip.h"+#include <assert.h>#include <cosmo.h>#include <errno.h>#include <fcntl.h>+#include <limits.h>+#include <stdint.h>#include <stdio.h>-#include <assert.h>-#include <unistd.h>#include <stdlib.h>-#include <stdint.h>-#include <limits.h>-#include <unistd.h>#include <sys/mman.h>-#include "zip.h"+#include <unistd.h>#define Min(a, b) ((a) < (b) ? (a) : (b))-asm(".ident\t\"\\n\\n\-llamafile (Apache 2.0)\\n\-Copyright 2023 Mozilla Foundation\\n\-\\n\-Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\-you may not use this file except in compliance with the License.\\n\-You may obtain a copy of the License at\\n\-\\n\- http://www.apache.org/licenses/LICENSE-2.0\\n\-\\n\-Unless required by applicable law or agreed to in writing, software\\n\-distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\-See the License for the specific language governing permissions and\\n\+__notice(llamafile_notice, "\+llamafile (Apache 2.0)\n\+Copyright 2023 Mozilla Foundation\n\+\n\+Licensed under the Apache License, Version 2.0 (the \"License\");\n\+you may not use this file except in compliance with the License.\n\+You may obtain a copy of the License at\n\+\n\+ http://www.apache.org/licenses/LICENSE-2.0\n\+\n\+Unless required by applicable law or agreed to in writing, software\n\+distributed under the License is distributed on an \"AS IS\" BASIS,\n\+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\+See the License for the specific language governing permissions and\n\limitations under the License.\"");struct llamafile {@@ -64,9 +63,8 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,uint8_t *cdirdata = NULL;struct llamafile *file = NULL;- if (!(file = calloc(1, sizeof(struct llamafile)))) {+ if (!(file = calloc(1, sizeof(struct llamafile))))return 0;- }strlcpy(file->fname, prog, PATH_MAX);// try opening from this executable's zip store@@ -75,9 +73,8 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,return 0;}ssize_t rc;- if ((rc = lseek(fd, 0, SEEK_END)) == -1) {+ if ((rc = lseek(fd, 0, SEEK_END)) == -1)goto Failure;- }file->size = rc;// read the last 64kb of file@@ -91,11 +88,11 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,off = file->size - 65536;amt = file->size - off;}- if (!(bufdata = gc(malloc(65536)))) {+ if (!(bufdata = gc(malloc(65536))))goto Failure;- }if (pread(fd, bufdata, amt, off) != amt) {- fprintf(stderr, "%s: warning: failed to read last 64kb of file: %s\n", prog, strerror(errno));+ fprintf(stderr, "%s: warning: failed to read last 64kb of file: %s\n", prog,+ strerror(errno));goto Failure;}@@ -106,8 +103,8 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i >= 0; --i) {uint32_t magic = ZIP_READ32(bufdata + i);if (magic == kZipCdir64LocatorMagic && i + kZipCdir64LocatorSize <= amt &&- pread(fd, bufdata, kZipCdir64HdrMinSize,- ZIP_LOCATE64_OFFSET(bufdata + i)) == (long)kZipCdir64HdrMinSize &&+ pread(fd, bufdata, kZipCdir64HdrMinSize, ZIP_LOCATE64_OFFSET(bufdata + i)) ==+ (long)kZipCdir64HdrMinSize &&ZIP_READ32(bufdata) == kZipCdir64HdrMagic &&ZIP_CDIR64_RECORDS(bufdata) == ZIP_CDIR64_RECORDSONDISK(bufdata) &&ZIP_CDIR64_RECORDS(bufdata) && ZIP_CDIR64_SIZE(bufdata) <= INT_MAX) {@@ -134,9 +131,8 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,// read the central directorycdirsize = amt;- if (!(cdirdata = gc(malloc(cdirsize)))) {+ if (!(cdirdata = gc(malloc(cdirsize))))goto Failure;- }if (pread(fd, cdirdata, cdirsize, off) != (long)cdirsize) {fprintf(stderr, "%s: warning: failed to pread zip cdir: %s\n", prog, strerror(errno));goto Failure;@@ -154,7 +150,7 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,unsigned entry_index, entry_offset;for (entry_index = entry_offset = 0;entry_index < cnt && entry_offset + kZipCfileHdrMinSize <= cdirsize &&- entry_offset + ZIP_CFILE_HDRSIZE(cdirdata + entry_offset) <= cdirsize;+ entry_offset + ZIP_CFILE_HDRSIZE(cdirdata + entry_offset) <= cdirsize;++entry_index, entry_offset += ZIP_CFILE_HDRSIZE(cdirdata + entry_offset)) {if (ZIP_CFILE_MAGIC(cdirdata + entry_offset) != kZipCfileHdrMagic) {fprintf(stderr, "error: corrupted zip central directory entry magic: %s\n", prog);@@ -163,11 +159,9 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,}int entry_name_len = ZIP_CFILE_NAMESIZE(cdirdata + entry_offset);const char *entry_name_bytes = ZIP_CFILE_NAME(cdirdata + entry_offset);- if ((fname- ? (fname_len == entry_name_len &&- !memcmp(fname, entry_name_bytes, fname_len))- : (entry_name_len > 5 &&- !memcasecmp(entry_name_bytes + entry_name_len - 5, ".gguf", 5)))) {+ if ((fname ? (fname_len == entry_name_len && !memcmp(fname, entry_name_bytes, fname_len))+ : (entry_name_len > 5 &&+ !memcasecmp(entry_name_bytes + entry_name_len - 5, ".gguf", 5)))) {zip_name = gc(strndup(entry_name_bytes, entry_name_len));off = get_zip_cfile_offset(cdirdata + entry_offset);file->size = get_zip_cfile_compressed_size(cdirdata + entry_offset);@@ -176,21 +170,23 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,}}if (!found) {- fprintf(stderr, "%s: error: no %s file found in zip archive\n",- prog, fname ? fname : ".gguf");+ fprintf(stderr, "%s: error: no %s file found in zip archive\n", prog,+ fname ? fname : ".gguf");goto Invalid;}if (found != 1) {// TODO: Support opening LLaVA llamafiles.- fprintf(stderr, "%s: error: multiple %s files found in zip archive\n",- prog, fname ? fname : ".gguf");+ fprintf(stderr, "%s: error: multiple %s files found in zip archive\n", prog,+ fname ? fname : ".gguf");goto Invalid;}strlcat(file->fname, "@", PATH_MAX);strlcat(file->fname, zip_name, PATH_MAX);if (ZIP_CFILE_COMPRESSIONMETHOD(cdirdata + cdir_offset) != kZipCompressionNone) {- fprintf(stderr, "%s: error: weights stored in the zip executable can't be stored using compression\n",- file->fname);+ fprintf(+ stderr,+ "%s: error: weights stored in the zip executable can't be stored using compression\n",+ file->fname);goto Invalid;}@@ -209,10 +205,9 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,// perform sanity check// mapping weights for apple metal gpu requires 16kb alignment- if (off & 16383) {+ if (off & 16383)fprintf(stderr, "%s: warning: use zipalign (rather than zip) to create llamafiles\n",file->fname);- }// map the file into memorylong pagesz = sysconf(_SC_PAGESIZE);@@ -221,16 +216,14 @@ static struct llamafile *llamafile_open_zip(const char *prog, const char *fname,file->mapsize = skew + file->size;file->mapping = mmap(0, file->mapsize, PROT_READ, MAP_SHARED | MAP_POPULATE, fd, mapoff);if (file->mapping == MAP_FAILED) {- fprintf(stderr, "%s: warning: failed to map zip file: %s\n",- file->fname, strerror(errno));+ fprintf(stderr, "%s: warning: failed to map zip file: %s\n", file->fname, strerror(errno));goto Failure;}errno_t err;- if ((err = posix_fadvise(fd, mapoff, file->mapsize, POSIX_FADV_SEQUENTIAL)) && err != ENOSYS) {+ if ((err = posix_fadvise(fd, mapoff, file->mapsize, POSIX_FADV_SEQUENTIAL)) && err != ENOSYS)fprintf(stderr, "%s: warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",file->fname, strerror(err));- }// setup our synthetic filefile->position = 0;@@ -250,9 +243,8 @@ Failure:static struct llamafile *llamafile_open_file(const char *fname, const char *mode) {struct llamafile *file;- if (!(file = calloc(1, sizeof(struct llamafile)))) {+ if (!(file = calloc(1, sizeof(struct llamafile))))return 0;- }strlcpy(file->fname, fname, PATH_MAX);if ((file->fp = fopen(fname, mode))) {if (!llamafile_seek(file, 0, SEEK_END)) {@@ -271,9 +263,8 @@ struct llamafile *llamafile_open_gguf(const char *fname, const char *mode) {// support filenames like `[email protected]`const char *p;- if ((p = strchr(fname, '@'))) {+ if ((p = strchr(fname, '@')))return llamafile_open_zip(gc(strndup(fname, p - fname)), p + 1, mode);- }// open from file or from our own executable if it doesn't existstruct llamafile *file;@@ -324,40 +315,37 @@ void *llamafile_content(struct llamafile *file) {}size_t llamafile_tell(struct llamafile *file) {- if (!file->fp) {+ if (!file->fp)return file->position;- }long ret = ftell(file->fp);- unassert(ret != -1); // shouldn't fail because we seeked earlier- return (size_t) ret;+ unassert(ret != -1); // shouldn't fail because we seeked earlier+ return (size_t)ret;}bool llamafile_seek(struct llamafile *file, size_t offset, int whence) {if (!file->fp) {switch (whence) {- case SEEK_SET:- file->position = offset;- break;- case SEEK_CUR:- file->position += offset;- break;- case SEEK_END:- file->position = file->size + offset;- break;+ case SEEK_SET:+ file->position = offset;+ break;+ case SEEK_CUR:+ file->position += offset;+ break;+ case SEEK_END:+ file->position = file->size + offset;+ break;}return true;}- return !fseek(file->fp, (long) offset, whence);+ return !fseek(file->fp, (long)offset, whence);}long llamafile_read(struct llamafile *file, void *ptr, size_t len) {- if (len == 0) {+ if (len == 0)return 0;- }if (!file->fp) {- if (file->position > file->size) {+ if (file->position > file->size)return 0;- }size_t remain = file->size - file->position;size_t amt = Min(len, remain);memcpy(ptr, file->content + file->position, amt);@@ -366,38 +354,32 @@ long llamafile_read(struct llamafile *file, void *ptr, size_t len) {}errno = 0;size_t ret = fread(ptr, len, 1, file->fp);- if (ferror(file->fp)) {+ if (ferror(file->fp))return -1;- }- if (ret != 1) {+ if (ret != 1)return 0;- }return len;}long llamafile_write(struct llamafile *file, const void *ptr, size_t len) {- if (len == 0) {+ if (len == 0)return 0;- }if (!file->fp) {errno = EROFS;return -1;}errno = 0;size_t ret = fwrite(ptr, len, 1, file->fp);- if (ferror(file->fp)) {+ if (ferror(file->fp))return -1;- }- if (ret != 1) {+ if (ret != 1)return 0;- }return len;}void llamafile_close(struct llamafile *file) {- if (file->fp) {+ if (file->fp)fclose(file->fp);- }if (file->mapping && file->mapping != MAP_FAILED) {// TODO(jart): reference count this mapping w/ llama_mmap// munmap(file->mapping, file->mapsize);diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 71207b9c..d93e8e0a 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -48,8 +48,8 @@ bool llamafile_has_metal(void);int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);-void llamafile_sgemm(long, long, long, int, const void *, long, const void *,- long, float *, long, long, long);+void llamafile_sgemm(long, long, long, int, const void *, long, const void *, long, float *, long,+ long, long);#ifdef __cplusplus}diff --git a/llamafile/log.c b/llamafile/log.cindex 7d0b281e..b32f0431 100644--- a/llamafile/log.c+++ b/llamafile/log.c@@ -16,13 +16,13 @@// limitations under the License.#include "log.h"-#include <unistd.h>-#include <string.h>#include <pthread.h>+#include <string.h>+#include <unistd.h>bool FLAG_log_disable;-void (tinylog)(const char *s, ...) {+void(tinylog)(const char *s, ...) {size_t n;int c, cs;va_list va;@@ -46,19 +46,16 @@ void (tinylog)(const char *s, ...) {void llamafile_log_command(char *args[]) {tinylog(__func__, ": ", NULL);for (int i = 0; args[i]; ++i) {- if (i) {+ if (i)tinylog(" ", NULL);- }// this quoting should be close enough to correct to be// copy/pastable on both unix and windows command termsbool need_quotes = !!strchr(args[i], ' ');- if (need_quotes) {+ if (need_quotes)tinylog("\"", NULL);- }tinylog(args[i], NULL);- if (need_quotes) {+ if (need_quotes)tinylog("\"", NULL);- }}tinylog("\n", NULL);}diff --git a/llamafile/log.h b/llamafile/log.hindex e2f08312..a31003cf 100644--- a/llamafile/log.h+++ b/llamafile/log.h@@ -1,8 +1,8 @@#ifndef LLAMAFILE_LOG_H_#define LLAMAFILE_LOG_H_-#include <stdio.h>#include <stdarg.h>#include <stdbool.h>+#include <stdio.h>#ifdef __cplusplusextern "C" {#endifdiff --git a/llamafile/metal.c b/llamafile/metal.cindex dc8bd11d..82ed828c 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -15,22 +15,22 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <time.h>+#include "llama.cpp/ggml-metal.h"+#include "llamafile.h"+#include "log.h"+#include <assert.h>#include <cosmo.h>#include <dlfcn.h>#include <errno.h>-#include <spawn.h>-#include <assert.h>-#include <unistd.h>-#include <stdlib.h>#include <limits.h>#include <pthread.h>-#include <sys/wait.h>-#include <sys/stat.h>+#include <spawn.h>#include <stdatomic.h>-#include "llamafile/log.h"-#include "llamafile/llamafile.h"-#include "llama.cpp/ggml-metal.h"+#include <stdlib.h>+#include <sys/stat.h>+#include <sys/wait.h>+#include <time.h>+#include <unistd.h>__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/llamafile.h");@@ -77,7 +77,8 @@ static struct Metal {static const char *Dlerror(void) {const char *msg;msg = cosmo_dlerror();- if (!msg) msg = "null dlopen error";+ if (!msg)+ msg = "null dlopen error";return msg;}@@ -99,33 +100,33 @@ static bool BuildMetal(const char *dso) {}strlcat(src, srcs[i].name, sizeof(src));switch (llamafile_is_file_newer_than(srcs[i].zip, src)) {- case -1:+ case -1:+ return false;+ case 0:+ break;+ case 1:+ needs_rebuild = true;+ if (!llamafile_extract(srcs[i].zip, src)) {return false;- case 0:- break;- case 1:- needs_rebuild = true;- if (!llamafile_extract(srcs[i].zip, src)) {- return false;- }- break;- default:- __builtin_unreachable();+ }+ break;+ default:+ __builtin_unreachable();}}// determine if we need to buildif (!needs_rebuild) {switch (llamafile_is_file_newer_than(src, dso)) {- case -1:- return false;- case 0:- break;- case 1:- needs_rebuild = true;- break;- default:- __builtin_unreachable();+ case -1:+ return false;+ case 0:+ break;+ case 1:+ needs_rebuild = true;+ break;+ default:+ __builtin_unreachable();}}@@ -154,10 +155,14 @@ static bool BuildMetal(const char *dso) {"-DTARGET_OS_OSX","-DGGML_MULTIPLATFORM",src,- "-o", tmpdso,- "-framework", "Foundation",- "-framework", "Metal",- "-framework", "MetalKit",+ "-o",+ tmpdso,+ "-framework",+ "Foundation",+ "-framework",+ "Metal",+ "-framework",+ "MetalKit",NULL,};int pid, ws;@@ -204,7 +209,8 @@ static bool LinkMetal(const char *dso) {ok &= !!(ggml_metal.ggml_metal_link = cosmo_dlsym(lib, "ggml_metal_link"));ok &= !!(ggml_metal.backend_init = cosmo_dlsym(lib, "ggml_backend_metal_init"));ok &= !!(ggml_metal.backend_buffer_type = cosmo_dlsym(lib, "ggml_backend_metal_buffer_type"));- ok &= !!(ggml_metal.backend_buffer_from_ptr = cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));+ ok &= !!(ggml_metal.backend_buffer_from_ptr =+ cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));ok &= !!(ggml_metal.backend_is_metal = cosmo_dlsym(lib, "ggml_backend_is_metal"));ok &= !!(ggml_metal.backend_set_n_cb = cosmo_dlsym(lib, "ggml_backend_metal_set_n_cb"));ok &= !!(ggml_metal.log_set_callback = cosmo_dlsym(lib, "ggml_backend_metal_log_set_callback"));@@ -228,11 +234,11 @@ static bool ImportMetalImpl(void) {// Check if we're allowed to even try.switch (FLAG_gpu) {- case LLAMAFILE_GPU_AUTO:- case LLAMAFILE_GPU_APPLE:- break;- default:- return false;+ case LLAMAFILE_GPU_AUTO:+ case LLAMAFILE_GPU_APPLE:+ break;+ default:+ return false;}// Get path of DSO.@@ -256,8 +262,8 @@ static void ImportMetal(void) {ggml_metal.supported = true;tinylog("Apple Metal GPU support successfully loaded\n", NULL);} else if (FLAG_gpu == LLAMAFILE_GPU_APPLE) {- tinyprint(2, "fatal error: support for --gpu ",- llamafile_describe_gpu(), FLAG_tinyblas ? " --tinyblas" : "",+ tinyprint(2, "fatal error: support for --gpu ", llamafile_describe_gpu(),+ FLAG_tinyblas ? " --tinyblas" : ""," was explicitly requested, but it wasn't available\n", NULL);exit(1);}@@ -269,36 +275,44 @@ bool llamafile_has_metal(void) {}ggml_backend_t ggml_backend_metal_init(void) {- if (!llamafile_has_metal()) return 0;+ if (!llamafile_has_metal())+ return 0;return ggml_metal.backend_init();}GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {- if (!llamafile_has_metal()) return 0;+ if (!llamafile_has_metal())+ return 0;return ggml_metal.backend_buffer_type();}-GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {- if (!llamafile_has_metal()) return 0;+GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void *data, size_t size,+ size_t max_size) {+ if (!llamafile_has_metal())+ return 0;return ggml_metal.backend_buffer_from_ptr(data, size, max_size);}bool ggml_backend_is_metal(ggml_backend_t backend) {- if (!llamafile_has_metal()) return 0;+ if (!llamafile_has_metal())+ return 0;return ggml_metal.backend_is_metal(backend);}void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {- if (!llamafile_has_metal()) return;+ if (!llamafile_has_metal())+ return;return ggml_metal.backend_set_n_cb(backend, n_cb);}-void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {- if (!llamafile_has_metal()) return;+void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void *user_data) {+ if (!llamafile_has_metal())+ return;return ggml_metal.log_set_callback(log_callback, user_data);}-ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {- if (!llamafile_has_metal()) return 0;+ggml_backend_t ggml_backend_reg_metal_init(const char *params, void *user_data) {+ if (!llamafile_has_metal())+ return 0;return ggml_metal.reg_init(params, user_data);}diff --git a/llamafile/schlep.c b/llamafile/schlep.cindex c9791e66..79358379 100644--- a/llamafile/schlep.c+++ b/llamafile/schlep.c@@ -73,19 +73,16 @@ static void FormatPercent(char sbuf[static 8], double x) {void llamafile_schlep(const void *data, size_t size) {// don't bother if logging is disabled- if (FLAG_log_disable) {+ if (FLAG_log_disable)return;- }// don't bother if memory is small- if (size < 128 * 1024 * 1024) {+ if (size < 128 * 1024 * 1024)return;- }// don't bother if stderr isn't a terminal- if (!isatty(2)) {+ if (!isatty(2))return;- }// launch threadserrno_t err;@@ -126,7 +123,6 @@ void llamafile_schlep(const void *data, size_t size) {tinyprint(2, "\r\033[K", NULL);// wait for workers- for (int i = 0; i < THREADS; ++i) {+ for (int i = 0; i < THREADS; ++i)pthread_join(pf[i].th, 0);- }}diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 3b8f83d1..21a20d9e 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -22,45 +22,43 @@#include <stdio.h>//-// _ _ ___ _ _ ___-// | |_(_)_ _ _ _| _ ) | /_\ / __|-// | _| | ' \ || | _ \ |__ / _ \\__ \.-// \__|_|_||_\_, |___/____/_/ \_\___/-// |__/+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__///-// BASIC LINEAR ALGEBRA SUBPROGRAMS+// BASIC LINEAR ALGEBRA SUBPROGRAMS//-#define ASSERT(x) \- if (!(x)) { \- fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, \- #x); \- __builtin_trap(); \+#define ASSERT(x) \+ if (!(x)) { \+ fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, #x); \+ __builtin_trap(); \}#define END_KERNEL() }-#define BEGIN_KERNEL(RM, RN) \- long ytiles = (m - m0) / RM; \- long xtiles = (n - n0) / RN; \- long tiles = ytiles * xtiles; \- double duty = (double)tiles / nth; \- if (duty < 1) \- duty = 1; \- double spot = duty * ith + .5; \- long end = spot + duty; \- long start = spot; \- if (end > tiles) \- end = tiles; \- for (long job = start; job < end; ++job) { \- long i = m0 + job / xtiles * RM; \+#define BEGIN_KERNEL(RM, RN) \+ long ytiles = (m - m0) / RM; \+ long xtiles = (n - n0) / RN; \+ long tiles = ytiles * xtiles; \+ double duty = (double)tiles / nth; \+ if (duty < 1) \+ duty = 1; \+ double spot = duty * ith + .5; \+ long end = spot + duty; \+ long start = spot; \+ if (end > tiles) \+ end = tiles; \+ for (long job = start; job < end; ++job) { \+ long i = m0 + job / xtiles * RM; \long j = n0 + job % xtiles * RN;#ifdef __x86_64__#pragma GCC push_options#pragma GCC target("avx2,fma,f16c,avxvnni")-#define MM256_SET_M128I(a, b) \- _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)static inline float unhalf(ggml_fp16_t d) {return GGML_FP16_TO_FP32(d);@@ -99,10 +97,9 @@ template <> inline __m256i load32i(const block_q4_0 *b) {template <typename T> class tinyBLAS {public:- tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C,- long ldc, long ith, long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),- nth(nth) {+ tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C, long ldc, long ith,+ long nth)+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {ASSERT(A != nullptr);ASSERT(B != nullptr);ASSERT(C != nullptr);@@ -248,8 +245,7 @@ template <typename T> class tinyBLAS {BEGIN_KERNEL(1, 1)__m256 c = _mm256_setzero_ps();for (long l = 0; l < k; l += 8) {- c = _mm256_fmadd_ps(load8f(A + lda * i + l),- load8f(B + ldb * j + l), c);+ c = _mm256_fmadd_ps(load8f(A + lda * i + l), load8f(B + ldb * j + l), c);}C[ldc * j + i] = hsum8f(c);END_KERNEL()@@ -268,10 +264,9 @@ template <typename T> class tinyBLAS {template <typename T, typename U, int avxvnni> class tinyBLASq {public:- tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C,- long ldc, long ith, long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),- nth(nth) {+ tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C, long ldc, long ith,+ long nth)+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {ASSERT(A != nullptr);ASSERT(B != nullptr);ASSERT(C != nullptr);@@ -413,8 +408,7 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {if (avxvnni)res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);else- res = _mm256_madd_epi16(_mm256_set1_epi16(1),- _mm256_maddubs_epi16(u, s));+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));return _mm256_cvtepi32_ps(res);}@@ -432,18 +426,15 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {#endif // __x86_64__// computes m×k * n×k → n×m-void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,- const void *B, long ldb, float *C, long ldc, long ith,- long nth) {+void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda, const void *B,+ long ldb, float *C, long ldc, long ith, long nth) {ASSERT(X86_HAVE(FMA));ASSERT(X86_HAVE(AVX2));switch (dtype) {#ifdef __x86_64__case GGML_TYPE_F32: {- tinyBLAS<float> tb{- k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth,- };+ tinyBLAS<float> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.gemm(m, n);break;}@@ -451,11 +442,7 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,case GGML_TYPE_F16: {ASSERT(X86_HAVE(F16C));tinyBLAS<ggml_fp16_t> tb{- k, (const ggml_fp16_t *)A,- lda, (const ggml_fp16_t *)B,- ldb, C,- ldc, ith,- nth,+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth,};tb.gemm(m, n);break;@@ -464,20 +451,12 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,case GGML_TYPE_Q8_0:if (X86_HAVE(AVXVNNI)) {tinyBLASq<block_q8_0, block_q8_0, true> tb{- k, (const block_q8_0 *)A,- lda, (const block_q8_0 *)B,- ldb, C,- ldc, ith,- nth,+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,};tb.gemm(m, n);} else {tinyBLASq<block_q8_0, block_q8_0, false> tb{- k, (const block_q8_0 *)A,- lda, (const block_q8_0 *)B,- ldb, C,- ldc, ith,- nth,+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,};tb.gemm(m, n);}@@ -486,20 +465,12 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,case GGML_TYPE_Q4_0:if (X86_HAVE(AVXVNNI)) {tinyBLASq<block_q4_0, block_q8_0, true> tb{- k, (const block_q4_0 *)A,- lda, (const block_q8_0 *)B,- ldb, C,- ldc, ith,- nth,+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,};tb.gemm(m, n);} else {tinyBLASq<block_q4_0, block_q8_0, false> tb{- k, (const block_q4_0 *)A,- lda, (const block_q8_0 *)B,- ldb, C,- ldc, ith,- nth,+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,};tb.gemm(m, n);}diff --git a/llamafile/simple.cpp b/llamafile/simple.cppindex e505ad54..474e2436 100644--- a/llamafile/simple.cpp+++ b/llamafile/simple.cpp@@ -25,18 +25,15 @@#include "llama.cpp/llama.h"#include "llamafile/llamafile.h"-static bool eval_tokens(struct llama_context *ctx_llama,- std::vector<llama_token> tokens, int n_batch,- int *n_past) {+static bool eval_tokens(struct llama_context *ctx_llama, std::vector<llama_token> tokens,+ int n_batch, int *n_past) {int N = (int)tokens.size();for (int i = 0; i < N; i += n_batch) {int n_eval = (int)tokens.size() - i;if (n_eval > n_batch)n_eval = n_batch;- if (llama_decode(ctx_llama,- llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {+ if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0)))return false; // probably ran out of context- }*n_past += n_eval;}return true;@@ -48,11 +45,10 @@ static bool eval_id(struct llama_context *ctx_llama, int id, int *n_past) {return eval_tokens(ctx_llama, tokens, 1, n_past);}-static bool eval_string(struct llama_context *ctx_llama, const char *str,- int n_batch, int *n_past, bool add_bos) {+static bool eval_string(struct llama_context *ctx_llama, const char *str, int n_batch, int *n_past,+ bool add_bos) {std::string str2 = str;- std::vector<llama_token> embd_inp =- ::llama_tokenize(ctx_llama, str2, add_bos);+ std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, add_bos);return eval_tokens(ctx_llama, embd_inp, n_batch, n_past);}@@ -71,13 +67,11 @@ int main(int argc, char **argv) {llama_model_params model_params = llama_model_default_params();model_params.n_gpu_layers = llamafile_gpu_layers(35);- llama_model *model =- llama_load_model_from_file(params.model.c_str(), model_params);+ llama_model *model = llama_load_model_from_file(params.model.c_str(), model_params);if (model == NULL)return 2;- llama_context_params ctx_params =- llama_context_params_from_gpt_params(params);+ llama_context_params ctx_params = llama_context_params_from_gpt_params(params);llama_context *ctx = llama_new_context_with_model(model, ctx_params);if (ctx == NULL)return 3;@@ -86,8 +80,7 @@ int main(int argc, char **argv) {int n_past = 0;bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));eval_string(ctx, params.prompt.c_str(), params.n_batch, &n_past, add_bos);- struct llama_sampling_context *ctx_sampling =- llama_sampling_init(params.sparams);+ struct llama_sampling_context *ctx_sampling = llama_sampling_init(params.sparams);for (;;) {llama_token id = llama_sampling_sample(ctx_sampling, ctx, NULL);llama_sampling_accept(ctx_sampling, ctx, id, true);diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex 336e9ccd..48b6102d 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -17,20 +17,24 @@#include "tinyblas.h"-#define READ(A, trans, ld, i, j) \+//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS+//++#define READ(A, trans, ld, i, j) \(((trans) == TINYBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))+#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))-#define CEIL_DIV(M, N) (((M) + (N)-1) / (N))--template <typename SRC, typename DST>-__device__ __forceinline__ DST typechange(SRC x) {+template <typename SRC, typename DST> __device__ __forceinline__ DST typechange(SRC x) {static_assert(std::is_same<DST, SRC>::value, "write a specialization");- if (std::is_same<DST, SRC>::value) {- return x;- } else {- return (DST)(0);- }+ return x;}template <> __device__ __forceinline__ float typechange(half x) {@@ -44,21 +48,20 @@ template <> __device__ __forceinline__ half typechange(float x) {template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>static __device__ void matmul_block2d(tinyblasOperation_t transa, //tinyblasOperation_t transb, //- int m, int n, int k, //- int x, int y, //- const DST alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const DST beta, //+ int m, int n, int k, //+ int x, int y, //+ const DST alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const DST beta, //DST *C, int ldc) {- static_assert((BM % TM == 0) && (BN % TN == 0),- "can't divide work for threads");+ static_assert((BM % TM == 0) && (BN % TN == 0), "can't divide work for threads");static_assert(BK == ((BM * BN) / (TM * TN)), // optional"threads can't load memory properly");static_assert((BM * BN) <= (BM * BK) + (BK * BN),"didn't allocate enough shared mem for threads");- const int ii0 = threadIdx.x / (BN / TN); /* {0, ..., (BM/TM) - 1} */- const int ii1 = threadIdx.x % (BN / TN); /* {0, ..., (BN/TN) - 1} */+ const int ii0 = threadIdx.x / (BN / TN); // {0, ..., (BM/TM) - 1}+ const int ii1 = threadIdx.x % (BN / TN); // {0, ..., (BN/TN) - 1}extern __shared__ float svals[]; // shared across all threads in a blockSRC *As = (SRC *)svals;SRC *Bs = (SRC *)svals + BM * BK;@@ -67,31 +70,26 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //SRC At[TM];SRC Bt[TN];int i, h, j, l, blob;+// within each block// we first zero out Csfor (j = 0; j < TM * TN; ++j)Cs[j] = 0;for (blob = 0; blob < k; blob += BK) {- for (i = threadIdx.x; i < BK; i += blockDim.x) {- for (j = 0; j < BM + BN; ++j) {+ for (i = threadIdx.x; i < BK; i += blockDim.x)+ for (j = 0; j < BM + BN; ++j)As[(j * BK) + i] = 0;- }- }__syncthreads();- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x) {+ for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)// we copy into As from A- for (j = 0; j < BM && x + j < m; ++j) {+ for (j = 0; j < BM && x + j < m; ++j)As[(i * BM) + j] = READ(A, transa, lda, x + j, blob + i);- }- }- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x) {+ for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)// we copy into Bs from B- for (j = 0; j < BN && y + j < n; ++j) {+ for (j = 0; j < BN && y + j < n; ++j)Bs[(i * BN) + j] = READ(B, transb, ldb, blob + i, y + j);- }- }__syncthreads();// We matmul the blobs, basically Cs += matmul(As, Bs)@@ -100,11 +98,9 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //At[j] = As[(l * BM) + ii0 * TM + j];for (h = 0; h < TN; ++h)Bt[h] = Bs[(l * BN) + ii1 * TN + h];- for (j = 0; j < TM; ++j) {- for (h = 0; h < TN; ++h) {+ for (j = 0; j < TM; ++j)+ for (h = 0; h < TN; ++h)Cs[j * TN + h] += At[j] * Bt[h];- }- }}__syncthreads();}@@ -113,145 +109,142 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //// We write Cs out into Cx += ii0 * TM;y += ii1 * TN;- for (j = 0; j < TM && x + j < m; ++j) {+ for (j = 0; j < TM && x + j < m; ++j)for (l = 0; l < TN && y + l < n; ++l) {- *(C + (x + j) + (y + l) * ldc) =- alpha * typechange<SRC, DST>(Cs[j * TN + l]);+ C[(x + j) + (y + l) * ldc] = alpha * typechange<SRC, DST>(Cs[j * TN + l]);// + beta * (*(C + (x + j) + (y + l) * ldc));}- }__syncthreads();}template <int BM, int BN, int BK, int TM, int TN>static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const float *A, int lda, //- const float *B, int ldb, //- const float beta, //+ int m, int n, int k, //+ const float alpha, //+ const float *A, int lda, //+ const float *B, int ldb, //+ const float beta, //float *C, int ldc) {+int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;+int y = blockIdx.y * BN;const int jump2 = gridDim.y * BN;// each block handles a sub-matrix of C, of size BM * BN// each thread handles a sub-matrix of size TM * TN- for (x = blockIdx.x * BM; x < m; x += jump1) {- for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN>(transa, transb, m, n, k, x, y,- alpha, A, lda, B, ldb, beta, C,- ldc);- }- }+ for (x = blockIdx.x * BM; x < m; x += jump1)+ for (y = blockIdx.y * BN; y < n; y += jump2)+ matmul_block2d<BM, BN, BK, TM, TN>(transa, transb, m, n, k, x, y, alpha, A, lda, B, ldb,+ beta, C, ldc);}template <int BM, int BN, int BK, int TM, int TN>-static void tinyblasS_wrapper(tinyblasHandle_t stream, //+static void tinyblasS_wrapper(tinyblasHandle_t stream, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const float *A, int lda, //- const float *B, int ldb, //- const float beta, //+ int m, int n, int k, //+ const float alpha, //+ const float *A, int lda, //+ const float *B, int ldb, //+ const float beta, //float *C, int ldc) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);int maxthreads = ((BM * BN) / (TM * TN));-tinyblasS_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C,- ldc);+ <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);}-tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,- tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k,- const float *alpha, const float *A, int lda,- const float *B, int ldb, const float *beta,+tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float *alpha, //+ const float *A, int lda, //+ const float *B, int ldb, //+ const float *beta, //float *C, int ldc) {- tinyblasS_wrapper<32, 8, 128, 1, 2>(stream, transa, transb, m, n, k, *alpha,- A, lda, B, ldb, *beta, C, ldc);+ tinyblasS_wrapper<32, 8, 128, 1, 2>(stream, transa, transb, m, n, k, *alpha, A, lda, B, ldb,+ *beta, C, ldc);return TINYBLAS_STATUS_SUCCESS;}-static inline bool check_args(cudaDataType_t Atype, cudaDataType_t Btype,- cudaDataType_t Ctype,- tinyblasComputeType_t computeType) {- return (Atype == CUDA_R_16F && Btype == CUDA_R_16F &&- (Ctype == CUDA_R_16F || Ctype == CUDA_R_32F) &&- (computeType == TINYBLAS_COMPUTE_16F ||- computeType == TINYBLAS_COMPUTE_32F));+static bool check_args(cudaDataType_t Atype, cudaDataType_t Btype, cudaDataType_t Ctype,+ tinyblasComputeType_t computeType) {+ return Atype == CUDA_R_16F && //+ Btype == CUDA_R_16F && //+ (Ctype == CUDA_R_16F || //+ Ctype == CUDA_R_32F) && //+ (computeType == TINYBLAS_COMPUTE_16F || //+ computeType == TINYBLAS_COMPUTE_32F);}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmextemplate <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const DST beta, //+ int m, int n, int k, //+ const DST alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const DST beta, //DST *C, int ldc) {+int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;+int y = blockIdx.y * BN;const int jump2 = gridDim.y * BN;// each block handles a sub-matrix of C, of size BM * BN- for (x = blockIdx.x * BM; x < m; x += jump1) {- for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(- transa, transb, m, n, k, x, y, //- alpha, A, lda, B, ldb, //- beta, C, ldc);- }- }+ for (x = blockIdx.x * BM; x < m; x += jump1)+ for (y = blockIdx.y * BN; y < n; y += jump2)+ matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(transa, transb, m, n, k, x, y, alpha, A,+ lda, B, ldb, beta, C, ldc);}template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static void tinyblasGE_wrapper(tinyblasHandle_t stream, //+static void tinyblasGE_wrapper(tinyblasHandle_t stream, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const float beta, //+ int m, int n, int k, //+ const float alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const float beta, //void *C, cudaDataType_t Ctype, int ldc) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);int maxthreads = ((BM * BN) / (TM * TN));-- if (Ctype == CUDA_R_16F) {+ if (Ctype == CUDA_R_16F)tinyblasGE_entry<BM, BN, BK, TM, TN, SRC, half>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, __float2half(alpha), A, lda,- B, ldb, __float2half(beta), (half *)C, ldc);- } else {+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, m, n, k, __float2half(alpha), A, lda, B, ldb, __float2half(beta),+ (half *)C, ldc);+ elsetinyblasGE_entry<BM, BN, BK, TM, TN, SRC, float>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta,- (float *)C, ldc);- }+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, (float *)C, ldc);}-tinyblasStatus_t-tinyblasGemmEx(tinyblasHandle_t stream, tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k,- const void *alpha, const void *A, cudaDataType_t Atype, int lda,- const void *B, cudaDataType_t Btype, int ldb, const void *beta,- void *C, cudaDataType_t Ctype, int ldc,- tinyblasComputeType_t computeType, tinyblasGemmAlgo_t algo) {- if (!check_args(Atype, Btype, Ctype, computeType)) {+tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const void *alpha, //+ const void *A, cudaDataType_t Atype, int lda, //+ const void *B, cudaDataType_t Btype, int ldb, //+ const void *beta, //+ void *C, cudaDataType_t Ctype, int ldc, //+ tinyblasComputeType_t computeType, //+ tinyblasGemmAlgo_t algo) {++ if (!check_args(Atype, Btype, Ctype, computeType))return TINYBLAS_STATUS_NOT_SUPPORTED;- }float alpha0, beta0;-if (computeType == TINYBLAS_COMPUTE_32F) {alpha0 = *(float *)alpha;beta0 = *(float *)beta;@@ -260,84 +253,82 @@ tinyblasGemmEx(tinyblasHandle_t stream, tinyblasOperation_t transa,beta0 = __half2float(*(half *)beta);}- tinyblasGE_wrapper<48, 32, 64, 3, 8, half>(stream, transa, transb, m, n, k,- alpha0, (const half *)A, lda,- (const half *)B, ldb, //+ tinyblasGE_wrapper<48, 32, 64, 3, 8, half>(stream, transa, transb, m, n, k, alpha0,+ (const half *)A, lda, (const half *)B, ldb, //beta0, C, Ctype, ldc);return TINYBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //+static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const DST alpha, //const SRC *const Aarray[], int lda, //const SRC *const Barray[], int ldb, //- const DST beta, //- void *const Carray[], int ldc,+ const DST beta, //+ void *const Carray[], int ldc, //int batchCount) {+int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;+int y = blockIdx.y * BN;const int jump2 = gridDim.y * BN;+int z = blockIdx.z;const int jump3 = gridDim.z;// each block handles a sub-matrix of C, of size BM * BN- for (z = blockIdx.z; z < batchCount; z += jump3) {- for (x = blockIdx.x * BM; x < m; x += jump1) {- for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(- transa, transb, m, n, k, x, y, alpha, Aarray[z], lda,- Barray[z], ldb, beta, (DST *)(Carray[z]), ldc);- }- }- }+ for (z = blockIdx.z; z < batchCount; z += jump3)+ for (x = blockIdx.x * BM; x < m; x += jump1)+ for (y = blockIdx.y * BN; y < n; y += jump2)+ matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(transa, transb, m, n, k, x, y, alpha,+ Aarray[z], lda, Barray[z], ldb, beta,+ (DST *)(Carray[z]), ldc);}template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static void tinyblasGBE_wrapper(tinyblasHandle_t stream,- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //+static void tinyblasGBE_wrapper(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //const SRC *const Aarray[], int lda, //const SRC *const Barray[], int ldb, //- const float beta, //- void *const Carray[], cudaDataType_t Ctype,- int ldc, int batchCount) {+ const float beta, //+ void *const Carray[], cudaDataType_t Ctype, int ldc, //+ int batchCount) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);int maxthreads = ((BM * BN) / (TM * TN));- if (Ctype == CUDA_R_16F) {+ if (Ctype == CUDA_R_16F)tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, half>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, __float2half(alpha), Aarray,- lda, Barray, ldb, __float2half(beta), Carray, ldc,- batchCount);- } else {+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, m, n, k, __float2half(alpha), Aarray, lda, Barray, ldb,+ __float2half(beta), Carray, ldc, batchCount);+ elsetinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, float>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, alpha, Aarray, lda, Barray,- ldb, beta, Carray, ldc, batchCount);- }+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, m, n, k, alpha, Aarray, lda, Barray, ldb, beta, Carray, ldc,+ batchCount);}-tinyblasStatus_t tinyblasGemmBatchedEx(- tinyblasHandle_t stream, //- tinyblasOperation_t transa, tinyblasOperation_t transb, //- int m, int n, int k, const void *alpha, //- const void *const Aarray[], cudaDataType_t Atype, int lda, //- const void *const Barray[], cudaDataType_t Btype, int ldb, //- const void *beta, void *const Carray[], cudaDataType_t Ctype, int ldc, //- int batchCount, tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo) {- if (!check_args(Atype, Btype, Ctype, computeType)) {+tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const void *alpha, //+ const void *const Aarray[], cudaDataType_t Atype, int lda, //+ const void *const Barray[], cudaDataType_t Btype, int ldb, //+ const void *beta, //+ void *const Carray[], cudaDataType_t Ctype, int ldc, //+ int batchCount, //+ tinyblasComputeType_t computeType, //+ tinyblasGemmAlgo_t algo) {+ if (!check_args(Atype, Btype, Ctype, computeType))return TINYBLAS_STATUS_NOT_SUPPORTED;- }float alpha0, beta0;if (computeType == TINYBLAS_COMPUTE_32F) {@@ -348,103 +339,107 @@ tinyblasStatus_t tinyblasGemmBatchedEx(beta0 = __half2float(*(half *)beta);}- tinyblasGBE_wrapper<48, 32, 64, 3, 8, half>(- stream, transa, transb, m, n, k, alpha0, //- (const half **)Aarray, lda, //- (const half **)Barray, ldb, //- beta0, Carray, Ctype, ldc, batchCount);+ tinyblasGBE_wrapper<48, 32, 64, 3, 8, half>(stream, transa, transb, //+ m, n, k, //+ alpha0, //+ (const half **)Aarray, lda, //+ (const half **)Barray, ldb, //+ beta0, Carray, Ctype, ldc, //+ batchCount);return TINYBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedextemplate <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __global__ void-tinyblasGSBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //- const SRC *A, int lda, long long int strideA, //- const SRC *B, int ldb, long long int strideB, //- const DST beta, //- void *C, int ldc, long long int strideC, //- int batchCount) {+static __global__ void tinyblasGSBE_entry(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const DST alpha, //+ const SRC *A, int lda, long long int strideA, //+ const SRC *B, int ldb, long long int strideB, //+ const DST beta, //+ void *C, int ldc, long long int strideC, //+ int batchCount) {+int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;+int y = blockIdx.y * BN;const int jump2 = gridDim.y * BN;+int z = blockIdx.z;const int jump3 = gridDim.z;// each block handles a sub-matrix of C, of size BM * BN- for (z = blockIdx.z; z < batchCount; z += jump3) {- for (x = blockIdx.x * BM; x < m; x += jump1) {- for (y = blockIdx.y * BN; y < n; y += jump2) {- matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(- transa, transb, //- m, n, k, x, y, //- alpha, //- A + z * strideA, lda, //- B + z * strideB, ldb, //- beta, //- ((DST *)C + z * strideC), ldc);- }- }- }+ for (z = blockIdx.z; z < batchCount; z += jump3)+ for (x = blockIdx.x * BM; x < m; x += jump1)+ for (y = blockIdx.y * BN; y < n; y += jump2)+ matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(transa, transb, //+ m, n, k, x, y, //+ alpha, //+ A + z * strideA, lda, //+ B + z * strideB, ldb, //+ beta, //+ (DST *)C + z * strideC, ldc);}template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const SRC *A, int lda, //- long long int strideA, //- const SRC *B, int ldb, //- long long int strideB, //- const float beta, //- void *C, cudaDataType_t Ctype, int ldc, //- long long int strideC, int batchCount) {+static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const SRC *A, int lda, long long int strideA, //+ const SRC *B, int ldb, long long int strideB, //+ const float beta, //+ void *C, cudaDataType_t Ctype, int ldc, long long int strideC,+ int batchCount) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);int maxthreads = ((BM * BN) / (TM * TN));- if (Ctype == CUDA_R_16F) {+ if (Ctype == CUDA_R_16F)tinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, half>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, //- m, n, k, //- __float2half(alpha), //- A, lda, strideA, //- B, ldb, strideB, //- __half2float(beta), //- C, ldc, strideC, //- batchCount);- } else {+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, //+ m, n, k, //+ __float2half(alpha), //+ A, lda, strideA, //+ B, ldb, strideB, //+ __half2float(beta), //+ C, ldc, strideC, //+ batchCount);+ elsetinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, float>- <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, //- m, n, k, //- alpha, //- A, lda, strideA, //- B, ldb, strideB, //- beta, //- C, ldc, strideC, //- batchCount);- }+ <<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)), stream>>>(+ transa, transb, //+ m, n, k, //+ alpha, //+ A, lda, strideA, //+ B, ldb, strideB, //+ beta, //+ C, ldc, strideC, //+ batchCount);}-tinyblasStatus_t tinyblasGemmStridedBatchedEx(- tinyblasHandle_t stream, tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k, const void *pAlpha,- const void *A, cudaDataType_t Atype, int lda, long long int strideA,- const void *B, cudaDataType_t Btype, int ldb, long long int strideB,- const void *pBeta, void *C, cudaDataType_t Ctype, int ldc,- long long int strideC, int batchCount, tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo) {- if (!check_args(Atype, Btype, Ctype, computeType)) {+tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const void *pAlpha, //+ const void *A, cudaDataType_t Atype, int lda,+ long long int strideA, //+ const void *B, cudaDataType_t Btype, int ldb,+ long long int strideB, //+ const void *pBeta, //+ void *C, cudaDataType_t Ctype, int ldc,+ long long int strideC, //+ int batchCount, //+ tinyblasComputeType_t computeType, //+ tinyblasGemmAlgo_t algo) {+ if (!check_args(Atype, Btype, Ctype, computeType))return TINYBLAS_STATUS_NOT_SUPPORTED;- }- float alpha0, beta0;+ float alpha0, beta0;if (computeType == TINYBLAS_COMPUTE_32F) {alpha0 = *(float *)pAlpha;beta0 = *(float *)pBeta;@@ -453,10 +448,9 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(beta0 = __half2float(*(half *)pBeta);}- tinyblasGSBE_wrapper<32, 4, 64, 1, 2, half>(- stream, transa, transb, m, n, k, alpha0, (const half *)A, lda, strideA,- (const half *)B, ldb, strideB, beta0, C, Ctype, ldc, strideC,- batchCount);+ tinyblasGSBE_wrapper<32, 4, 64, 1, 2, half>(stream, transa, transb, m, n, k, alpha0,+ (const half *)A, lda, strideA, (const half *)B, ldb,+ strideB, beta0, C, Ctype, ldc, strideC, batchCount);return TINYBLAS_STATUS_SUCCESS;}diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.hindex d095e7a5..b00bb772 100644--- a/llamafile/tinyblas.h+++ b/llamafile/tinyblas.h@@ -1,105 +1,56 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once#ifdef GGML_USE_HIPBLAS-#include <hipblas/hipblas.h>#include <hip/hip_fp16.h>+#include <hipblas/hipblas.h>#else-#include <cuda_runtime.h>#include <cuda_fp16.h>+#include <cuda_runtime.h>#endifenum tinyblasOperation_t {- TINYBLAS_OP_N,- TINYBLAS_OP_T,+ TINYBLAS_OP_N,+ TINYBLAS_OP_T,};enum tinyblasStatus_t {- TINYBLAS_STATUS_SUCCESS,- TINYBLAS_STATUS_NOT_SUPPORTED,+ TINYBLAS_STATUS_SUCCESS,+ TINYBLAS_STATUS_NOT_SUPPORTED,};enum tinyblasComputeType_t {- TINYBLAS_COMPUTE_16F,- TINYBLAS_COMPUTE_32F,+ TINYBLAS_COMPUTE_16F,+ TINYBLAS_COMPUTE_32F,};enum tinyblasGemmAlgo_t {- TINYBLAS_GEMM_DEFAULT_TENSOR_OP,+ TINYBLAS_GEMM_DEFAULT_TENSOR_OP,};-#define tinyblasHandle_t cudaStream_t+typedef cudaStream_t tinyblasHandle_t;const char *tinyblasGetStatusString(tinyblasStatus_t);-tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle,- tinyblasOperation_t transa,- tinyblasOperation_t transb,- int m, int n, int k,- const float *alpha,- const float *A, int lda,- const float *B, int ldb,- const float *beta,- float *C, int ldc);--tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t handle,- tinyblasOperation_t transa,- tinyblasOperation_t transb,- int m,- int n,- int k,- const void *alpha,- const void *A,- cudaDataType_t Atype,- int lda,- const void *B,- cudaDataType_t Btype,- int ldb,- const void *beta,- void *C,- cudaDataType_t Ctype,- int ldc,- tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo);--tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t handle,- tinyblasOperation_t transa,- tinyblasOperation_t transb,- int m,- int n,- int k,- const void *alpha,- const void *const Aarray[],- cudaDataType_t Atype,- int lda,- const void *const Barray[],- cudaDataType_t Btype,- int ldb,- const void *beta,- void *const Carray[],- cudaDataType_t Ctype,- int ldc,- int batchCount,- tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo);--tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle,- tinyblasOperation_t transa,- tinyblasOperation_t transb,- int m, int n, int k,- const void *pAlpha,- const void *A,- cudaDataType_t Atype,- int lda,- long long int strideA,- const void *B,- cudaDataType_t Btype,- int ldb,- long long int strideB,- const void *pBeta,- void *C,- cudaDataType_t Ctype,- int ldc,- long long int strideC,- int batchCount,- tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo);+tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t, int, int,+ int, const float *, const float *, int, const float *, int,+ const float *, float *, int);++tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t, int,+ int, int, const void *, const void *, cudaDataType_t, int,+ const void *, cudaDataType_t, int, const void *, void *,+ cudaDataType_t, int, tinyblasComputeType_t, tinyblasGemmAlgo_t);++tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t,+ int, int, int, const void *, const void *const[],+ cudaDataType_t, int, const void *const[], cudaDataType_t,+ int, const void *, void *const[], cudaDataType_t, int, int,+ tinyblasComputeType_t, tinyblasGemmAlgo_t);++tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t, tinyblasOperation_t,+ tinyblasOperation_t, int, int, int, const void *,+ const void *, cudaDataType_t, int, long long,+ const void *, cudaDataType_t, int, long long,+ const void *, void *, cudaDataType_t, int, long long,+ int, tinyblasComputeType_t, tinyblasGemmAlgo_t);diff --git a/llamafile/version.h b/llamafile/version.hindex 17214af3..3f702f90 100644--- a/llamafile/version.h+++ b/llamafile/version.h@@ -3,7 +3,8 @@#define LLAMAFILE_MAJOR 0#define LLAMAFILE_MINOR 6#define LLAMAFILE_PATCH 2-#define LLAMAFILE_VERSION (100000000 * LLAMAFILE_MAJOR + 1000000 * LLAMAFILE_MINOR + LLAMAFILE_PATCH)+#define LLAMAFILE_VERSION \+ (100000000 * LLAMAFILE_MAJOR + 1000000 * LLAMAFILE_MINOR + LLAMAFILE_PATCH)#define MKVERSION__(x, y, z) #x "." #y "." #z#define MKVERSION_(x, y, z) MKVERSION__(x, y, z)diff --git a/llamafile/x.c b/llamafile/x.cindex 0899532b..5e0dff1f 100644--- a/llamafile/x.c+++ b/llamafile/x.c@@ -28,7 +28,8 @@ static wontreturn void oom(void) {char *xvasprintf(const char *fmt, va_list va) {char *buf;- if (vasprintf(&buf, fmt, va) == -1) oom();+ if (vasprintf(&buf, fmt, va) == -1)+ oom();return buf;}diff --git a/llamafile/zip.c b/llamafile/zip.cindex 6c26d622..36ef0c35 100644--- a/llamafile/zip.c+++ b/llamafile/zip.c@@ -19,40 +19,31 @@#include <stdint.h>int64_t get_zip_cfile_compressed_size(const uint8_t *z) {- if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu) {+ if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu)return ZIP_CFILE_COMPRESSEDSIZE(z);- }const uint8_t *p = ZIP_CFILE_EXTRA(z);const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);- for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p)) {- if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {- if (8 <= ZIP_EXTRA_CONTENTSIZE(p)) {+ for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))+ if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64)+ if (8 <= ZIP_EXTRA_CONTENTSIZE(p))return ZIP_READ64(ZIP_EXTRA_CONTENT(p));- }- }- }return -1;}int64_t get_zip_cfile_offset(const uint8_t *z) {- if (ZIP_CFILE_OFFSET(z) != 0xFFFFFFFFu) {+ if (ZIP_CFILE_OFFSET(z) != 0xFFFFFFFFu)return ZIP_CFILE_OFFSET(z);- }const uint8_t *p = ZIP_CFILE_EXTRA(z);const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);- for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p)) {+ for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {int offset = 0;- if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu) {+ if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu)offset += 8;- }- if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu) {+ if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu)offset += 8;- }- if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p)) {+ if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p))return ZIP_READ64(ZIP_EXTRA_CONTENT(p) + offset);- }}- }return -1;}diff --git a/llamafile/zip.h b/llamafile/zip.hindex f9b0a8c5..f165e14a 100644--- a/llamafile/zip.h+++ b/llamafile/zip.h@@ -17,104 +17,104 @@#define _ZE(x) x#endif-#define kZipOk 0-#define kZipErrorEocdNotFound _ZE(-1)-#define kZipErrorEocdOffsetOverflow _ZE(-2)-#define kZipErrorEocdMagicNotFound _ZE(-3)-#define kZipErrorEocdSizeOverflow _ZE(-4)-#define kZipErrorEocdDiskMismatch _ZE(-5)+#define kZipOk 0+#define kZipErrorEocdNotFound _ZE(-1)+#define kZipErrorEocdOffsetOverflow _ZE(-2)+#define kZipErrorEocdMagicNotFound _ZE(-3)+#define kZipErrorEocdSizeOverflow _ZE(-4)+#define kZipErrorEocdDiskMismatch _ZE(-5)#define kZipErrorEocdOffsetSizeOverflow _ZE(-6)-#define kZipErrorEocdRecordsMismatch _ZE(-7)-#define kZipErrorEocdRecordsOverflow _ZE(-8)-#define kZipErrorCdirOffsetPastEocd _ZE(-9)-#define kZipErrorEocdLocatorMagic _ZE(-10)-#define kZipErrorEocdLocatorOffset _ZE(-11)-#define kZipErrorRaceCondition _ZE(-12)-#define kZipErrorMapFailed _ZE(-13)-#define kZipErrorOpenFailed _ZE(-14)+#define kZipErrorEocdRecordsMismatch _ZE(-7)+#define kZipErrorEocdRecordsOverflow _ZE(-8)+#define kZipErrorCdirOffsetPastEocd _ZE(-9)+#define kZipErrorEocdLocatorMagic _ZE(-10)+#define kZipErrorEocdLocatorOffset _ZE(-11)+#define kZipErrorRaceCondition _ZE(-12)+#define kZipErrorMapFailed _ZE(-13)+#define kZipErrorOpenFailed _ZE(-14)#define kZipCosmopolitanVersion kZipEra2001-#define kZipOsDos 0-#define kZipOsAmiga 1-#define kZipOsOpenvms 2-#define kZipOsUnix 3-#define kZipOsVmcms 4-#define kZipOsAtarist 5-#define kZipOsOs2hpfs 6-#define kZipOsMacintosh 7-#define kZipOsZsystem 8-#define kZipOsCpm 9+#define kZipOsDos 0+#define kZipOsAmiga 1+#define kZipOsOpenvms 2+#define kZipOsUnix 3+#define kZipOsVmcms 4+#define kZipOsAtarist 5+#define kZipOsOs2hpfs 6+#define kZipOsMacintosh 7+#define kZipOsZsystem 8+#define kZipOsCpm 9#define kZipOsWindowsntfs 10#define kZipOsMvsos390zos 11-#define kZipOsVse 12-#define kZipOsAcornrisc 13-#define kZipOsVfat 14-#define kZipOsAltmvs 15-#define kZipOsBeos 16-#define kZipOsTandem 17-#define kZipOsOs400 18-#define kZipOsOsxdarwin 19+#define kZipOsVse 12+#define kZipOsAcornrisc 13+#define kZipOsVfat 14+#define kZipOsAltmvs 15+#define kZipOsBeos 16+#define kZipOsTandem 17+#define kZipOsOs400 18+#define kZipOsOsxdarwin 19#define kZipEra1989 10 /* PKZIP 1.0 */#define kZipEra1993 20 /* PKZIP 2.0: deflate/subdir/etc. support */#define kZipEra2001 45 /* PKZIP 4.5: kZipExtraZip64 support */#define kZipIattrBinary 0 /* first bit not set */-#define kZipIattrText 1 /* first bit set */+#define kZipIattrText 1 /* first bit set */-#define kZipCompressionNone 0+#define kZipCompressionNone 0#define kZipCompressionDeflate 8-#define kZipCdirHdrMagic ZM_(0x06054b50) /* PK♣♠ "PK\5\6" */-#define kZipCdirHdrMagicTodo ZM_(0x19184b50) /* PK♣♠ "PK\30\31" */-#define kZipCdirHdrMinSize 22-#define kZipCdirHdrLinkableSize 294-#define kZipCdirDiskOffset 4-#define kZipCdirStartingDiskOffset 6+#define kZipCdirHdrMagic ZM_(0x06054b50) /* PK♣♠ "PK\5\6" */+#define kZipCdirHdrMagicTodo ZM_(0x19184b50) /* PK♣♠ "PK\30\31" */+#define kZipCdirHdrMinSize 22+#define kZipCdirHdrLinkableSize 294+#define kZipCdirDiskOffset 4+#define kZipCdirStartingDiskOffset 6#define kZipCdirRecordsOnDiskOffset 8-#define kZipCdirRecordsOffset 10-#define kZipCdirSizeOffset 12-#define kZipCdirOffsetOffset 16-#define kZipCdirCommentSizeOffset 20+#define kZipCdirRecordsOffset 10+#define kZipCdirSizeOffset 12+#define kZipCdirOffsetOffset 16+#define kZipCdirCommentSizeOffset 20-#define kZipCdir64HdrMagic ZM_(0x06064b50) /* PK♠♠ "PK\6\6" */-#define kZipCdir64HdrMinSize 56+#define kZipCdir64HdrMagic ZM_(0x06064b50) /* PK♠♠ "PK\6\6" */+#define kZipCdir64HdrMinSize 56#define kZipCdir64LocatorMagic ZM_(0x07064b50) /* PK♠• "PK\6\7" */-#define kZipCdir64LocatorSize 20+#define kZipCdir64LocatorSize 20-#define kZipCfileHdrMagic ZM_(0x02014b50) /* PK☺☻ "PK\1\2" */-#define kZipCfileHdrMinSize 46-#define kZipCfileOffsetGeneralflag 8-#define kZipCfileOffsetCompressionmethod 10-#define kZipCfileOffsetLastmodifiedtime 12-#define kZipCfileOffsetLastmodifieddate 14-#define kZipCfileOffsetCrc32 16-#define kZipCfileOffsetCompressedsize 20-#define kZipCfileOffsetUncompressedsize 24-#define kZipCfileOffsetNamesize 28+#define kZipCfileHdrMagic ZM_(0x02014b50) /* PK☺☻ "PK\1\2" */+#define kZipCfileHdrMinSize 46+#define kZipCfileOffsetGeneralflag 8+#define kZipCfileOffsetCompressionmethod 10+#define kZipCfileOffsetLastmodifiedtime 12+#define kZipCfileOffsetLastmodifieddate 14+#define kZipCfileOffsetCrc32 16+#define kZipCfileOffsetCompressedsize 20+#define kZipCfileOffsetUncompressedsize 24+#define kZipCfileOffsetNamesize 28#define kZipCfileOffsetExternalattributes 38-#define kZipCfileOffsetOffset 42+#define kZipCfileOffsetOffset 42-#define kZipLfileHdrMagic ZM_(0x04034b50) /* PK♥♦ "PK\3\4" */-#define kZipLfileHdrMinSize 30-#define kZipLfileOffsetVersionNeeded 4-#define kZipLfileOffsetGeneralflag 6+#define kZipLfileHdrMagic ZM_(0x04034b50) /* PK♥♦ "PK\3\4" */+#define kZipLfileHdrMinSize 30+#define kZipLfileOffsetVersionNeeded 4+#define kZipLfileOffsetGeneralflag 6#define kZipLfileOffsetCompressionmethod 8-#define kZipLfileOffsetLastmodifiedtime 10-#define kZipLfileOffsetLastmodifieddate 12-#define kZipLfileOffsetCrc32 14-#define kZipLfileOffsetNamesize 26-#define kZipLfileOffsetCompressedsize 18-#define kZipLfileOffsetUncompressedsize 22+#define kZipLfileOffsetLastmodifiedtime 10+#define kZipLfileOffsetLastmodifieddate 12+#define kZipLfileOffsetCrc32 14+#define kZipLfileOffsetNamesize 26+#define kZipLfileOffsetCompressedsize 18+#define kZipLfileOffsetUncompressedsize 22#define kZipGflagUtf8 0x800-#define kZipExtraHdrSize 4-#define kZipExtraZip64 0x0001-#define kZipExtraNtfs 0x000a-#define kZipExtraUnix 0x000d-#define kZipExtraExtendedTimestamp 0x5455+#define kZipExtraHdrSize 4+#define kZipExtraZip64 0x0001+#define kZipExtraNtfs 0x000a+#define kZipExtraUnix 0x000d+#define kZipExtraExtendedTimestamp 0x5455#define kZipExtraInfoZipNewUnixExtra 0x7875#define kZipCfileMagic "PK\001\002"@@ -129,141 +129,133 @@#define ZIP_SWAP64_(P) (P)#endif-#define ZIP_READ16(P) \- (__extension__({ \- uint16_t __x; \- __builtin_memcpy(&__x, P, 16 / 8); \- ZIP_SWAP16_(__x); \- }))-#define ZIP_READ32(P) \- (__extension__({ \- uint32_t __x; \- __builtin_memcpy(&__x, P, 32 / 8); \- ZIP_SWAP32_(__x); \- }))-#define ZIP_READ64(P) \- (__extension__({ \- uint64_t __x; \- __builtin_memcpy(&__x, P, 64 / 8); \- ZIP_SWAP64_(__x); \- }))+#define ZIP_READ16(P) \+ __extension__({ \+ uint16_t __x; \+ __builtin_memcpy(&__x, P, 16 / 8); \+ ZIP_SWAP16_(__x); \+ })+#define ZIP_READ32(P) \+ __extension__({ \+ uint32_t __x; \+ __builtin_memcpy(&__x, P, 32 / 8); \+ ZIP_SWAP32_(__x); \+ })+#define ZIP_READ64(P) \+ __extension__({ \+ uint64_t __x; \+ __builtin_memcpy(&__x, P, 64 / 8); \+ ZIP_SWAP64_(__x); \+ })-#define ZIP_WRITE16(P, X) \- (__extension__({ \- __typeof__(&(P)[0]) __p = (P); \- uint16_t __x = ZIP_SWAP16_(X); \- __builtin_memcpy(__p, &__x, 16 / 8); \- __p + 16 / 8; \- }))-#define ZIP_WRITE32(P, X) \- (__extension__({ \- __typeof__(&(P)[0]) __p = (P); \- uint32_t __x = ZIP_SWAP32_(X); \- __builtin_memcpy(__p, &__x, 32 / 8); \- __p + 32 / 8; \- }))-#define ZIP_WRITE64(P, X) \- (__extension__({ \- __typeof__(&(P)[0]) __p = (P); \- uint64_t __x = ZIP_SWAP64_(X); \- __builtin_memcpy(__p, &__x, 64 / 8); \- __p + 64 / 8; \- }))+#define ZIP_WRITE16(P, X) \+ __extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint16_t __x = ZIP_SWAP16_(X); \+ __builtin_memcpy(__p, &__x, 16 / 8); \+ __p + 16 / 8; \+ })+#define ZIP_WRITE32(P, X) \+ __extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint32_t __x = ZIP_SWAP32_(X); \+ __builtin_memcpy(__p, &__x, 32 / 8); \+ __p + 32 / 8; \+ })+#define ZIP_WRITE64(P, X) \+ __extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint64_t __x = ZIP_SWAP64_(X); \+ __builtin_memcpy(__p, &__x, 64 / 8); \+ __p + 64 / 8; \+ })/* end of central directory record */-#define ZIP_CDIR_MAGIC(P) ZIP_READ32(P)-#define ZIP_CDIR_DISK(P) ZIP_READ16((P) + kZipCdirDiskOffset)-#define ZIP_CDIR_STARTINGDISK(P) ZIP_READ16((P) + kZipCdirStartingDiskOffset)+#define ZIP_CDIR_MAGIC(P) ZIP_READ32(P)+#define ZIP_CDIR_DISK(P) ZIP_READ16((P) + kZipCdirDiskOffset)+#define ZIP_CDIR_STARTINGDISK(P) ZIP_READ16((P) + kZipCdirStartingDiskOffset)#define ZIP_CDIR_RECORDSONDISK(P) ZIP_READ16((P) + kZipCdirRecordsOnDiskOffset)-#define ZIP_CDIR_RECORDS(P) ZIP_READ16((P) + kZipCdirRecordsOffset)-#define ZIP_CDIR_SIZE(P) ZIP_READ32((P) + kZipCdirSizeOffset)-#define ZIP_CDIR_OFFSET(P) ZIP_READ32((P) + kZipCdirOffsetOffset)-#define ZIP_CDIR_COMMENTSIZE(P) ZIP_READ16((P) + kZipCdirCommentSizeOffset)-#define ZIP_CDIR_COMMENT(P) ((P) + 22) /* recommend stopping at nul */-#define ZIP_CDIR_HDRSIZE(P) (ZIP_CDIR_COMMENTSIZE(P) + kZipCdirHdrMinSize)+#define ZIP_CDIR_RECORDS(P) ZIP_READ16((P) + kZipCdirRecordsOffset)+#define ZIP_CDIR_SIZE(P) ZIP_READ32((P) + kZipCdirSizeOffset)+#define ZIP_CDIR_OFFSET(P) ZIP_READ32((P) + kZipCdirOffsetOffset)+#define ZIP_CDIR_COMMENTSIZE(P) ZIP_READ16((P) + kZipCdirCommentSizeOffset)+#define ZIP_CDIR_COMMENT(P) ((P) + 22) /* recommend stopping at nul */+#define ZIP_CDIR_HDRSIZE(P) (ZIP_CDIR_COMMENTSIZE(P) + kZipCdirHdrMinSize)/* zip64 end of central directory record */-#define ZIP_CDIR64_MAGIC(P) ZIP_READ32(P)-#define ZIP_CDIR64_HDRSIZE(P) (ZIP_READ64((P) + 4) + 12)-#define ZIP_CDIR64_VERSIONMADE(P) ZIP_READ16((P) + 12)-#define ZIP_CDIR64_VERSIONNEED(P) ZIP_READ16((P) + 14)-#define ZIP_CDIR64_DISK(P) ZIP_READ32((P) + 16)-#define ZIP_CDIR64_STARTINGDISK(P) ZIP_READ32((P) + 20)+#define ZIP_CDIR64_MAGIC(P) ZIP_READ32(P)+#define ZIP_CDIR64_HDRSIZE(P) (ZIP_READ64((P) + 4) + 12)+#define ZIP_CDIR64_VERSIONMADE(P) ZIP_READ16((P) + 12)+#define ZIP_CDIR64_VERSIONNEED(P) ZIP_READ16((P) + 14)+#define ZIP_CDIR64_DISK(P) ZIP_READ32((P) + 16)+#define ZIP_CDIR64_STARTINGDISK(P) ZIP_READ32((P) + 20)#define ZIP_CDIR64_RECORDSONDISK(P) ZIP_READ64((P) + 24)-#define ZIP_CDIR64_RECORDS(P) ZIP_READ64((P) + 32)-#define ZIP_CDIR64_SIZE(P) ZIP_READ64((P) + 40)-#define ZIP_CDIR64_OFFSET(P) ZIP_READ64((P) + 48)-#define ZIP_CDIR64_COMMENTSIZE(P) \- (ZIP_CDIR64_HDRSIZE(P) >= 56 ? ZIP_CDIR64_HDRSIZE(P) - 56 : 0)-#define ZIP_CDIR64_COMMENT(P) ((P) + 56) /* recommend stopping at nul */-#define ZIP_LOCATE64_MAGIC(P) ZIP_READ32(P)+#define ZIP_CDIR64_RECORDS(P) ZIP_READ64((P) + 32)+#define ZIP_CDIR64_SIZE(P) ZIP_READ64((P) + 40)+#define ZIP_CDIR64_OFFSET(P) ZIP_READ64((P) + 48)+#define ZIP_CDIR64_COMMENTSIZE(P) (ZIP_CDIR64_HDRSIZE(P) >= 56 ? ZIP_CDIR64_HDRSIZE(P) - 56 : 0)+#define ZIP_CDIR64_COMMENT(P) ((P) + 56) /* recommend stopping at nul */+#define ZIP_LOCATE64_MAGIC(P) ZIP_READ32(P)#define ZIP_LOCATE64_STARTINGDISK(P) ZIP_READ32((P) + 4)-#define ZIP_LOCATE64_OFFSET(P) ZIP_READ64((P) + 8)-#define ZIP_LOCATE64_TOTALDISKS(P) ZIP_READ32((P) + 12)+#define ZIP_LOCATE64_OFFSET(P) ZIP_READ64((P) + 8)+#define ZIP_LOCATE64_TOTALDISKS(P) ZIP_READ32((P) + 12)/* central directory file header */-#define ZIP_CFILE_MAGIC(P) ZIP_READ32(P)-#define ZIP_CFILE_VERSIONMADE(P) (255 & (P)[4])+#define ZIP_CFILE_MAGIC(P) ZIP_READ32(P)+#define ZIP_CFILE_VERSIONMADE(P) (255 & (P)[4])#define ZIP_CFILE_FILEATTRCOMPAT(P) (255 & (P)[5])-#define ZIP_CFILE_VERSIONNEED(P) (255 & (P)[6])-#define ZIP_CFILE_OSNEED(P) (255 & (P)[7])-#define ZIP_CFILE_GENERALFLAG(P) ZIP_READ16((P) + kZipCfileOffsetGeneralflag)-#define ZIP_CFILE_COMPRESSIONMETHOD(P) \- ZIP_READ16((P) + kZipCfileOffsetCompressionmethod)+#define ZIP_CFILE_VERSIONNEED(P) (255 & (P)[6])+#define ZIP_CFILE_OSNEED(P) (255 & (P)[7])+#define ZIP_CFILE_GENERALFLAG(P) ZIP_READ16((P) + kZipCfileOffsetGeneralflag)+#define ZIP_CFILE_COMPRESSIONMETHOD(P) ZIP_READ16((P) + kZipCfileOffsetCompressionmethod)#define ZIP_CFILE_LASTMODIFIEDTIME(P) \- ZIP_READ16((P) + kZipCfileOffsetLastmodifiedtime) /* @see DOS_TIME() */+ ZIP_READ16((P) + kZipCfileOffsetLastmodifiedtime) /* @see DOS_TIME() */#define ZIP_CFILE_LASTMODIFIEDDATE(P) \- ZIP_READ16((P) + kZipCfileOffsetLastmodifieddate) /* @see DOS_DATE() */-#define ZIP_CFILE_CRC32(P) ZIP_READ32((P) + kZipCfileOffsetCrc32)+ ZIP_READ16((P) + kZipCfileOffsetLastmodifieddate) /* @see DOS_DATE() */+#define ZIP_CFILE_CRC32(P) ZIP_READ32((P) + kZipCfileOffsetCrc32)#define ZIP_CFILE_COMPRESSEDSIZE(P) ZIP_READ32(P + kZipCfileOffsetCompressedsize)-#define ZIP_CFILE_UNCOMPRESSEDSIZE(P) \- ZIP_READ32((P) + kZipCfileOffsetUncompressedsize)-#define ZIP_CFILE_NAMESIZE(P) ZIP_READ16((P) + kZipCfileOffsetNamesize)-#define ZIP_CFILE_EXTRASIZE(P) ZIP_READ16((P) + 30)-#define ZIP_CFILE_COMMENTSIZE(P) ZIP_READ16((P) + 32)-#define ZIP_CFILE_DISK(P) ZIP_READ16((P) + 34)+#define ZIP_CFILE_UNCOMPRESSEDSIZE(P) ZIP_READ32((P) + kZipCfileOffsetUncompressedsize)+#define ZIP_CFILE_NAMESIZE(P) ZIP_READ16((P) + kZipCfileOffsetNamesize)+#define ZIP_CFILE_EXTRASIZE(P) ZIP_READ16((P) + 30)+#define ZIP_CFILE_COMMENTSIZE(P) ZIP_READ16((P) + 32)+#define ZIP_CFILE_DISK(P) ZIP_READ16((P) + 34)#define ZIP_CFILE_INTERNALATTRIBUTES(P) ZIP_READ16((P) + 36)-#define ZIP_CFILE_EXTERNALATTRIBUTES(P) \- ZIP_READ32((P) + kZipCfileOffsetExternalattributes)+#define ZIP_CFILE_EXTERNALATTRIBUTES(P) ZIP_READ32((P) + kZipCfileOffsetExternalattributes)#define ZIP_CFILE_OFFSET(P) ZIP_READ32((P) + kZipCfileOffsetOffset)-#define ZIP_CFILE_NAME(P) ((const char *)((P) + 46)) /* not nul-terminated */-#define ZIP_CFILE_EXTRA(P) ((P) + 46 + ZIP_CFILE_NAMESIZE(P))-#define ZIP_CFILE_COMMENT(P) \- ((const char *)((P) + 46 + ZIP_CFILE_NAMESIZE(P) + \- ZIP_CFILE_EXTRASIZE(P))) /* recommend stopping at nul */-#define ZIP_CFILE_HDRSIZE(P) \- (ZIP_CFILE_NAMESIZE(P) + ZIP_CFILE_EXTRASIZE(P) + ZIP_CFILE_COMMENTSIZE(P) + \- kZipCfileHdrMinSize)+#define ZIP_CFILE_NAME(P) ((const char *)((P) + 46)) /* not nul-terminated */+#define ZIP_CFILE_EXTRA(P) ((P) + 46 + ZIP_CFILE_NAMESIZE(P))+#define ZIP_CFILE_COMMENT(P) \+ ((const char *)((P) + 46 + ZIP_CFILE_NAMESIZE(P) + \+ ZIP_CFILE_EXTRASIZE(P))) /* recommend stopping at nul */+#define ZIP_CFILE_HDRSIZE(P) \+ (ZIP_CFILE_NAMESIZE(P) + ZIP_CFILE_EXTRASIZE(P) + ZIP_CFILE_COMMENTSIZE(P) + \+ kZipCfileHdrMinSize)/* local file header */-#define ZIP_LFILE_MAGIC(P) ZIP_READ32(P)+#define ZIP_LFILE_MAGIC(P) ZIP_READ32(P)#define ZIP_LFILE_VERSIONNEED(P) (255 & (P)[4])-#define ZIP_LFILE_OSNEED(P) (255 & (P)[5])+#define ZIP_LFILE_OSNEED(P) (255 & (P)[5])#define ZIP_LFILE_GENERALFLAG(P) ZIP_READ16((P) + kZipLfileOffsetGeneralflag)-#define ZIP_LFILE_COMPRESSIONMETHOD(P) \- ZIP_READ16((P) + kZipLfileOffsetCompressionmethod)+#define ZIP_LFILE_COMPRESSIONMETHOD(P) ZIP_READ16((P) + kZipLfileOffsetCompressionmethod)#define ZIP_LFILE_LASTMODIFIEDTIME(P) \- ZIP_READ16((P) + kZipLfileOffsetLastmodifiedtime) /* @see DOS_TIME() */+ ZIP_READ16((P) + kZipLfileOffsetLastmodifiedtime) /* @see DOS_TIME() */#define ZIP_LFILE_LASTMODIFIEDDATE(P) \- ZIP_READ16((P) + kZipLfileOffsetLastmodifieddate) /* @see DOS_DATE() */+ ZIP_READ16((P) + kZipLfileOffsetLastmodifieddate) /* @see DOS_DATE() */#define ZIP_LFILE_CRC32(P) ZIP_READ32((P) + kZipLfileOffsetCrc32)-#define ZIP_LFILE_COMPRESSEDSIZE(P) \- ZIP_READ32((P) + kZipLfileOffsetCompressedsize)-#define ZIP_LFILE_UNCOMPRESSEDSIZE(P) \- ZIP_READ32((P) + kZipLfileOffsetUncompressedsize)-#define ZIP_LFILE_NAMESIZE(P) ZIP_READ16((P) + kZipLfileOffsetNamesize)+#define ZIP_LFILE_COMPRESSEDSIZE(P) ZIP_READ32((P) + kZipLfileOffsetCompressedsize)+#define ZIP_LFILE_UNCOMPRESSEDSIZE(P) ZIP_READ32((P) + kZipLfileOffsetUncompressedsize)+#define ZIP_LFILE_NAMESIZE(P) ZIP_READ16((P) + kZipLfileOffsetNamesize)#define ZIP_LFILE_EXTRASIZE(P) ZIP_READ16((P) + 28)-#define ZIP_LFILE_NAME(P) ((const char *)((P) + 30))-#define ZIP_LFILE_EXTRA(P) ((P) + 30 + ZIP_LFILE_NAMESIZE(P))-#define ZIP_LFILE_HDRSIZE(P) \- (ZIP_LFILE_NAMESIZE(P) + ZIP_LFILE_EXTRASIZE(P) + kZipLfileHdrMinSize)+#define ZIP_LFILE_NAME(P) ((const char *)((P) + 30))+#define ZIP_LFILE_EXTRA(P) ((P) + 30 + ZIP_LFILE_NAMESIZE(P))+#define ZIP_LFILE_HDRSIZE(P) (ZIP_LFILE_NAMESIZE(P) + ZIP_LFILE_EXTRASIZE(P) + kZipLfileHdrMinSize)#define ZIP_LFILE_CONTENT(P) ((P) + ZIP_LFILE_HDRSIZE(P))-#define ZIP_LFILE_SIZE(P) (ZIP_LFILE_HDRSIZE(P) + ZIP_LFILE_COMPRESSEDSIZE(P))+#define ZIP_LFILE_SIZE(P) (ZIP_LFILE_HDRSIZE(P) + ZIP_LFILE_COMPRESSEDSIZE(P))-#define ZIP_EXTRA_HEADERID(P) ZIP_READ16(P)+#define ZIP_EXTRA_HEADERID(P) ZIP_READ16(P)#define ZIP_EXTRA_CONTENTSIZE(P) ZIP_READ16((P) + 2)-#define ZIP_EXTRA_CONTENT(P) ((P) + 4)-#define ZIP_EXTRA_SIZE(P) (ZIP_EXTRA_CONTENTSIZE(P) + kZipExtraHdrSize)+#define ZIP_EXTRA_CONTENT(P) ((P) + 4)+#define ZIP_EXTRA_SIZE(P) (ZIP_EXTRA_CONTENTSIZE(P) + kZipExtraHdrSize)int64_t get_zip_cfile_offset(const uint8_t *);int64_t get_zip_cfile_compressed_size(const uint8_t *);diff --git a/llamafile/zipalign.c b/llamafile/zipalign.cindex cddbceca..c43a24ca 100644--- a/llamafile/zipalign.c+++ b/llamafile/zipalign.c@@ -15,30 +15,28 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "llamafile.h"#include "zip.h"-#include <time.h>-#include <fcntl.h>+#include <assert.h>#include <cosmo.h>-#include <stdio.h>+#include <fcntl.h>+#include <getopt.h>+#include <libgen.h>#include <limits.h>-#include <assert.h>+#include <stdbool.h>#include <stdint.h>+#include <stdio.h>#include <stdlib.h>-#include <libgen.h>-#include <getopt.h>-#include <stdbool.h>-#include <sys/uio.h>#include <sys/stat.h>-#include "llamafile.h"+#include <sys/uio.h>#include <third_party/zlib/zlib.h>+#include <time.h>#define CHUNK 2097152#define Min(a, b) ((a) < (b) ? (a) : (b))-#define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) \- (((YEAR)-1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1))-#define DOS_TIME(HOUR, MINUTE, SECOND) \- ((HOUR) << 11 | (MINUTE) << 5 | (SECOND) >> 1)+#define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) (((YEAR) - 1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1))+#define DOS_TIME(HOUR, MINUTE, SECOND) ((HOUR) << 11 | (MINUTE) << 5 | (SECOND) >> 1)static const char *prog;static int FLAG_junk;@@ -63,24 +61,25 @@ static wontreturn void DieOom(void) {static void *Malloc(size_t n) {void *p;- if (!(p = malloc(n))) DieOom();+ if (!(p = malloc(n)))+ DieOom();return p;}static char *StrDup(const char *s) {char *p;- if (!(p = strdup(s))) DieOom();+ if (!(p = strdup(s)))+ DieOom();return p;}static void *Realloc(void *p, size_t n) {- if (!(p = realloc(p, n))) DieOom();+ if (!(p = realloc(p, n)))+ DieOom();return p;}-static void GetDosLocalTime(int64_t utcunixts,- uint16_t *out_time,- uint16_t *out_date) {+static void GetDosLocalTime(int64_t utcunixts, uint16_t *out_time, uint16_t *out_date) {struct tm tm;localtime_r(&utcunixts, &tm);*out_time = DOS_TIME(tm.tm_hour, tm.tm_min, tm.tm_sec);@@ -89,8 +88,7 @@ static void GetDosLocalTime(int64_t utcunixts,int main(int argc, char *argv[]) {- if (llamafile_has(argv, "-h") ||- llamafile_has(argv, "-help") ||+ if (llamafile_has(argv, "-h") || llamafile_has(argv, "-help") ||llamafile_has(argv, "--help")) {llamafile_help("/zip/llamafile/zipalign.1.asc");__builtin_unreachable();@@ -98,60 +96,56 @@ int main(int argc, char *argv[]) {// get name of programprog = argv[0];- if (!prog) prog = "zipalign";+ if (!prog)+ prog = "zipalign";// parse flagsint opt;while ((opt = getopt(argc, argv, "0123456789vjNa:")) != -1) {switch (opt) {- case '0':- case '1':- case '2':- case '3':- case '4':- case '5':- case '6':- case '7':- case '8':- case '9':- FLAG_level = opt - '0';- break;- case 'v':- ++FLAG_verbose;- break;- case 'j':- FLAG_junk = true;- break;- case 'N':- FLAG_nondeterministic = true;- break;- case 'a':- FLAG_alignment = atoi(optarg);- if (FLAG_alignment < 1) {- Die(prog, "FLAG_alignment must be at least 1");- }- if (FLAG_alignment & (FLAG_alignment - 1)) {- Die(prog, "FLAG_alignment must be two power");- }- break;- default:- return 1;+ case '0':+ case '1':+ case '2':+ case '3':+ case '4':+ case '5':+ case '6':+ case '7':+ case '8':+ case '9':+ FLAG_level = opt - '0';+ break;+ case 'v':+ ++FLAG_verbose;+ break;+ case 'j':+ FLAG_junk = true;+ break;+ case 'N':+ FLAG_nondeterministic = true;+ break;+ case 'a':+ FLAG_alignment = atoi(optarg);+ if (FLAG_alignment < 1)+ Die(prog, "FLAG_alignment must be at least 1");+ if (FLAG_alignment & (FLAG_alignment - 1))+ Die(prog, "FLAG_alignment must be two power");+ break;+ default:+ return 1;}}- if (optind == argc) {+ if (optind == argc)Die(prog, "missing output argument");- }// open output fileint zfd;ssize_t zsize;const char *zpath = argv[optind++];- if ((zfd = open(zpath, O_CREAT | O_RDWR, 0644)) == -1) {+ if ((zfd = open(zpath, O_CREAT | O_RDWR, 0644)) == -1)DieSys(zpath);- }- if ((zsize = lseek(zfd, 0, SEEK_END)) == -1) {+ if ((zsize = lseek(zfd, 0, SEEK_END)) == -1)DieSys(zpath);- }// read last 64kb of fileint amt;@@ -164,9 +158,8 @@ int main(int argc, char *argv[]) {amt = zsize - off;}static char last64[65536];- if (pread(zfd, last64, amt, off) != amt) {+ if (pread(zfd, last64, amt, off) != amt)DieSys(zpath);- }// search backwards for the end-of-central-directory record// the eocd (cdir) says where the central directory (cfile) array is located@@ -175,8 +168,8 @@ int main(int argc, char *argv[]) {for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i >= 0; --i) {uint32_t magic = ZIP_READ32(last64 + i);if (magic == kZipCdir64LocatorMagic && i + kZipCdir64LocatorSize <= amt &&- pread(zfd, last64, kZipCdir64HdrMinSize,- ZIP_LOCATE64_OFFSET(last64 + i)) == (long)kZipCdir64HdrMinSize &&+ pread(zfd, last64, kZipCdir64HdrMinSize, ZIP_LOCATE64_OFFSET(last64 + i)) ==+ (long)kZipCdir64HdrMinSize &&ZIP_READ32(last64) == kZipCdir64HdrMagic &&ZIP_CDIR64_RECORDS(last64) == ZIP_CDIR64_RECORDSONDISK(last64) &&ZIP_CDIR64_RECORDS(last64) && ZIP_CDIR64_SIZE(last64) <= INT_MAX) {@@ -195,38 +188,31 @@ int main(int argc, char *argv[]) {break;}}- if (!cnt) {+ if (!cnt)amt = 0;- }// read central directorysize_t cdirsize = amt;uint8_t *cdir = Malloc(cdirsize);- if (pread(zfd, cdir, cdirsize, off) != cdirsize) {+ if (pread(zfd, cdir, cdirsize, off) != cdirsize)DieSys(zpath);- }// create array of zip entry nameschar **names = Malloc(sizeof(char *) * argc);for (int i = optind; i < argc; ++i) {names[i] = StrDup(argv[i]);- if (FLAG_junk) {+ if (FLAG_junk)names[i] = basename(names[i]);- } else {- while (*names[i] == '/') {+ else+ while (*names[i] == '/')++names[i];- }- }}// verify there's no duplicate zip asset names- for (int i = optind; i < argc; ++i) {- for (int j = i + 1; j < argc; ++j) {- if (!strcmp(names[i], names[j])) {+ for (int i = optind; i < argc; ++i)+ for (int j = i + 1; j < argc; ++j)+ if (!strcmp(names[i], names[j]))Die(names[i], "zip asset name specified multiple times");- }- }- }// delete central directory entries about to be replacedint new_count = 0;@@ -234,27 +220,24 @@ int main(int argc, char *argv[]) {unsigned entry_index, entry_offset;for (entry_index = entry_offset = 0;entry_index < cnt && entry_offset + kZipCfileHdrMinSize <= cdirsize &&- entry_offset + ZIP_CFILE_HDRSIZE(cdir + entry_offset) <= cdirsize;+ entry_offset + ZIP_CFILE_HDRSIZE(cdir + entry_offset) <= cdirsize;++entry_index, entry_offset += ZIP_CFILE_HDRSIZE(cdir + entry_offset)) {- if (ZIP_CFILE_MAGIC(cdir + entry_offset) != kZipCfileHdrMagic) {+ if (ZIP_CFILE_MAGIC(cdir + entry_offset) != kZipCfileHdrMagic)Die(zpath, "corrupted zip central directory entry magic");- }// check if entry name matches any of the new namesbool found = false;- for (int i = optind; i < argc; ++i) {+ for (int i = optind; i < argc; ++i)if (ZIP_CFILE_NAMESIZE(cdir + entry_offset) == strlen(names[i]) &&!memcmp(ZIP_CFILE_NAME(cdir + entry_offset), names[i],ZIP_CFILE_NAMESIZE(cdir + entry_offset))) {found = true;break;}- }// copy back central directory entryif (!found) {- memmove(cdir + new_index, cdir + entry_offset,- ZIP_CFILE_HDRSIZE(cdir + entry_offset));+ memmove(cdir + new_index, cdir + entry_offset, ZIP_CFILE_HDRSIZE(cdir + entry_offset));new_index += ZIP_CFILE_HDRSIZE(cdir + new_index);++new_count;}@@ -268,29 +251,25 @@ int main(int argc, char *argv[]) {// open input fileint fd;const char *path = argv[i];- if ((fd = open(path, O_RDONLY)) == -1) {+ if ((fd = open(path, O_RDONLY)) == -1)DieSys(path);- }// get information about fileuint64_t size;struct stat st;- if (fstat(fd, &st) == -1) {+ if (fstat(fd, &st) == -1)DieSys(path);- }size = st.st_size;- if (!S_ISREG(st.st_mode)) {+ if (!S_ISREG(st.st_mode))Die(path, "not a regular file");- }// get timeint64_t ts;uint16_t mtime, mdate;- if (FLAG_nondeterministic) {+ if (FLAG_nondeterministic)ts = st.st_mtime;- } else {+ elsets = 1700000000;- }GetDosLocalTime(ts, &mtime, &mdate);// determine size and alignment of local file header@@ -298,7 +277,8 @@ int main(int argc, char *argv[]) {size_t namlen = strlen(name);size_t extlen = (2 + 2 + 8 + 8);size_t hdrlen = kZipLfileHdrMinSize + namlen + extlen;- while ((zsize + hdrlen) & (FLAG_alignment - 1)) ++zsize;+ while ((zsize + hdrlen) & (FLAG_alignment - 1))+ ++zsize;// initialize zlib in raw deflate modez_stream zs;@@ -310,13 +290,14 @@ int main(int argc, char *argv[]) {zs.zalloc = 0;zs.zfree = 0;zs.opaque = 0;- switch (deflateInit2(&zs, FLAG_level, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {- case Z_OK:- break;- case Z_MEM_ERROR:- DieOom();- default:- unassert(!"deflateInit2() called with invalid parameters");+ switch (deflateInit2(&zs, FLAG_level, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,+ Z_DEFAULT_STRATEGY)) {+ case Z_OK:+ break;+ case Z_MEM_ERROR:+ DieOom();+ default:+ unassert(!"deflateInit2() called with invalid parameters");}}@@ -328,15 +309,13 @@ int main(int argc, char *argv[]) {_Alignas(4096) static uint8_t cdbuf[CHUNK];for (off_t i = 0; i < size; i += rc) {// read chunk- if ((rc = pread(fd, iobuf, Min(size, CHUNK), i)) <= 0) {+ if ((rc = pread(fd, iobuf, Min(size, CHUNK), i)) <= 0)DieSys(path);- }crc = crc32(crc, iobuf, rc);if (!FLAG_level) {// write uncompressed chunk to output- if (pwrite(zfd, iobuf, rc, zsize + hdrlen + compsize) != rc) {+ if (pwrite(zfd, iobuf, rc, zsize + hdrlen + compsize) != rc)DieSys(zpath);- }compsize += rc;} else {// compress chunk and write to output@@ -347,24 +326,22 @@ int main(int argc, char *argv[]) {zs.avail_out = CHUNK;int boop;switch ((boop = deflate(&zs, rc != CHUNK ? Z_FINISH : Z_FULL_FLUSH))) {- case Z_MEM_ERROR:- DieOom();- case Z_STREAM_ERROR:- unassert(!"deflate() stream error");- default:- break;+ case Z_MEM_ERROR:+ DieOom();+ case Z_STREAM_ERROR:+ unassert(!"deflate() stream error");+ default:+ break;}ssize_t have = CHUNK - zs.avail_out;- if (pwrite(zfd, cdbuf, have, zsize + hdrlen + compsize) != have) {+ if (pwrite(zfd, cdbuf, have, zsize + hdrlen + compsize) != have)DieSys(zpath);- }compsize += have;} while (!zs.avail_out);}}- if (FLAG_level) {+ if (FLAG_level)unassert(deflateEnd(&zs) == Z_OK);- }// write local file headeruint8_t *lochdr = Malloc(hdrlen);@@ -377,21 +354,20 @@ int main(int argc, char *argv[]) {p = ZIP_WRITE16(p, mtime);p = ZIP_WRITE16(p, mdate);p = ZIP_WRITE32(p, crc);- p = ZIP_WRITE32(p, 0xffffffffu); // compressed size- p = ZIP_WRITE32(p, 0xffffffffu); // uncompressed size+ p = ZIP_WRITE32(p, 0xffffffffu); // compressed size+ p = ZIP_WRITE32(p, 0xffffffffu); // uncompressed sizep = ZIP_WRITE16(p, namlen);p = ZIP_WRITE16(p, extlen);p = mempcpy(p, name, namlen);p = ZIP_WRITE16(p, kZipExtraZip64);p = ZIP_WRITE16(p, 8 + 8);- p = ZIP_WRITE64(p, size); // uncompressed size- p = ZIP_WRITE64(p, compsize); // compressed size+ p = ZIP_WRITE64(p, size); // uncompressed size+ p = ZIP_WRITE64(p, compsize); // compressed sizeunassert(p == lochdr + hdrlen);- if (pwrite(zfd, lochdr, hdrlen, zsize) != hdrlen) {+ if (pwrite(zfd, lochdr, hdrlen, zsize) != hdrlen)DieSys(zpath);- }free(lochdr);// create central directory entry@@ -403,82 +379,77 @@ int main(int argc, char *argv[]) {p = cdirhdr;p = ZIP_WRITE32(p, kZipCfileHdrMagic);- p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by- p = ZIP_WRITE16(p, kZipEra2001); // version needed to extract+ p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by+ p = ZIP_WRITE16(p, kZipEra2001); // version needed to extractp = ZIP_WRITE16(p, kZipGflagUtf8);p = ZIP_WRITE16(p, compression);p = ZIP_WRITE16(p, mtime);p = ZIP_WRITE16(p, mdate);p = ZIP_WRITE32(p, crc);- p = ZIP_WRITE32(p, 0xffffffffu); // compressed size- p = ZIP_WRITE32(p, 0xffffffffu); // uncompressed size+ p = ZIP_WRITE32(p, 0xffffffffu); // compressed size+ p = ZIP_WRITE32(p, 0xffffffffu); // uncompressed sizep = ZIP_WRITE16(p, namlen);p = ZIP_WRITE16(p, extlen);- p = ZIP_WRITE16(p, 0); // comment length- p = ZIP_WRITE16(p, 0); // disk number start+ p = ZIP_WRITE16(p, 0); // comment length+ p = ZIP_WRITE16(p, 0); // disk number startp = ZIP_WRITE16(p, kZipIattrBinary);- p = ZIP_WRITE32(p, st.st_mode << 16); // external file attributes+ p = ZIP_WRITE32(p, st.st_mode << 16); // external file attributesp = ZIP_WRITE32(p, 0xffffffffu); // lfile offsetp = mempcpy(p, name, namlen);p = ZIP_WRITE16(p, kZipExtraZip64);p = ZIP_WRITE16(p, 8 + 8 + 8);- p = ZIP_WRITE64(p, size); // uncompressed size- p = ZIP_WRITE64(p, compsize); // compressed size- p = ZIP_WRITE64(p, zsize); // lfile offset+ p = ZIP_WRITE64(p, size); // uncompressed size+ p = ZIP_WRITE64(p, compsize); // compressed size+ p = ZIP_WRITE64(p, zsize); // lfile offsetunassert(p == cdirhdr + hdrlen);// finish up++cnt;zsize += hdrlen + compsize;- if (close(fd)) {+ if (close(fd))DieSys(path);- }// log asset creation- if (FLAG_verbose) {+ if (FLAG_verbose)tinyprint(2, path, " -> ", name, "\n", NULL);- }}// write out central directory- if (pwrite(zfd, cdir, cdirsize, zsize) != cdirsize) {+ if (pwrite(zfd, cdir, cdirsize, zsize) != cdirsize)DieSys(zpath);- }free(cdir);// write out end of central directoryuint8_t eocd[kZipCdirHdrMinSize + kZipCdir64HdrMinSize + kZipCdir64LocatorSize];uint8_t *p = eocd;p = ZIP_WRITE32(p, kZipCdir64HdrMagic);- p = ZIP_WRITE64(p, kZipCdir64HdrMinSize - 12); // size of eocd64- p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by- p = ZIP_WRITE16(p, kZipEra2001); // version needed to extract- p = ZIP_WRITE32(p, 0); // number of this disk- p = ZIP_WRITE32(p, 0); // number of disk with start of central directory- p = ZIP_WRITE64(p, cnt); // number of records on disk- p = ZIP_WRITE64(p, cnt); // number of records- p = ZIP_WRITE64(p, cdirsize); // size of central directory- p = ZIP_WRITE64(p, zsize); // offset of start of central directory+ p = ZIP_WRITE64(p, kZipCdir64HdrMinSize - 12); // size of eocd64+ p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by+ p = ZIP_WRITE16(p, kZipEra2001); // version needed to extract+ p = ZIP_WRITE32(p, 0); // number of this disk+ p = ZIP_WRITE32(p, 0); // number of disk with start of central directory+ p = ZIP_WRITE64(p, cnt); // number of records on disk+ p = ZIP_WRITE64(p, cnt); // number of records+ p = ZIP_WRITE64(p, cdirsize); // size of central directory+ p = ZIP_WRITE64(p, zsize); // offset of start of central directoryp = ZIP_WRITE32(p, kZipCdir64LocatorMagic);- p = ZIP_WRITE32(p, 0); // number of disk with eocd64- p = ZIP_WRITE64(p, zsize + cdirsize); // offset of eocd64- p = ZIP_WRITE32(p, 1); // total number of disks+ p = ZIP_WRITE32(p, 0); // number of disk with eocd64+ p = ZIP_WRITE64(p, zsize + cdirsize); // offset of eocd64+ p = ZIP_WRITE32(p, 1); // total number of disksp = ZIP_WRITE32(p, kZipCdirHdrMagic);- p = ZIP_WRITE16(p, 0); // number of this disk- p = ZIP_WRITE16(p, 0); // number of disks- p = ZIP_WRITE16(p, cnt); // number of records on disk- p = ZIP_WRITE16(p, cnt); // number of records- p = ZIP_WRITE32(p, cdirsize); // size of central directory- p = ZIP_WRITE32(p, 0xffffffffu); // offset of central directory- p = ZIP_WRITE16(p, 0); // comment length+ p = ZIP_WRITE16(p, 0); // number of this disk+ p = ZIP_WRITE16(p, 0); // number of disks+ p = ZIP_WRITE16(p, cnt); // number of records on disk+ p = ZIP_WRITE16(p, cnt); // number of records+ p = ZIP_WRITE32(p, cdirsize); // size of central directory+ p = ZIP_WRITE32(p, 0xffffffffu); // offset of central directory+ p = ZIP_WRITE16(p, 0); // comment lengthunassert(p == eocd + sizeof(eocd));- if (pwrite(zfd, eocd, sizeof(eocd), zsize + cdirsize) != sizeof(eocd)) {+ if (pwrite(zfd, eocd, sizeof(eocd), zsize + cdirsize) != sizeof(eocd))DieSys(zpath);- }// close output- if (close(zfd)) {+ if (close(zfd))DieSys(zpath);- }}diff --git a/llamafile/zipcheck.c b/llamafile/zipcheck.cindex 59b8e732..1563be89 100644--- a/llamafile/zipcheck.c+++ b/llamafile/zipcheck.c@@ -1,7 +1,7 @@-// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi//-// Copyright 2023 Mozilla Foundation+// Copyright 2024 Mozilla Foundation//// Licensed under the Apache License, Version 2.0 (the "License");// you may not use this file except in compliance with the License.@@ -15,23 +15,22 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <time.h>-#include <fcntl.h>+#include "zip.h"+#include <assert.h>#include <cosmo.h>-#include <stdio.h>+#include <fcntl.h>+#include <getopt.h>#include <limits.h>-#include <assert.h>+#include <stdbool.h>#include <stdint.h>+#include <stdio.h>#include <stdlib.h>-#include <libgen.h>-#include <getopt.h>-#include <stdbool.h>#include <sys/uio.h>#include <third_party/zlib/zlib.h>-#include "zip.h"+#include <time.h>#define USAGE \- " [FLAGS] FILE...\n\+ " [FLAGS] FILE...\n\\n\DESCRIPTION\n\\n\@@ -62,18 +61,21 @@ static wontreturn void DieOom(void) {static void *Malloc(size_t n) {void *p;- if (!(p = malloc(n))) DieOom();+ if (!(p = malloc(n)))+ DieOom();return p;}static char *StrDup(const char *s) {char *p;- if (!(p = strdup(s))) DieOom();+ if (!(p = strdup(s)))+ DieOom();return p;}static void *Realloc(void *p, size_t n) {- if (!(p = realloc(p, n))) DieOom();+ if (!(p = realloc(p, n)))+ DieOom();return p;}@@ -87,12 +89,10 @@ static void CheckZip(const char *zpath) {// open zip archiveint zfd;ssize_t zsize;- if ((zfd = open(zpath, O_RDONLY)) == -1) {+ if ((zfd = open(zpath, O_RDONLY)) == -1)DieSys(zpath);- }- if ((zsize = lseek(zfd, 0, SEEK_END)) == -1) {+ if ((zsize = lseek(zfd, 0, SEEK_END)) == -1)DieSys(zpath);- }// read last 64kb of fileint amt;@@ -105,19 +105,21 @@ static void CheckZip(const char *zpath) {amt = zsize - off;}static char last64[65536];- if (pread(zfd, last64, amt, off) != amt) {+ if (pread(zfd, last64, amt, off) != amt)DieSys(zpath);- }- // search backwards for the end-of-central-directory record- // the eocd (cdir) says where the central directory (cfile) array is located- // we consistency check some legacy fields, to be extra sure that it is eocd+ // search backwards for the+ // end-of-central-directory record the eocd+ // (cdir) says where the central directory+ // (cfile) array is located we consistency+ // check some legacy fields, to be extra sure+ // that it is eocdunsigned cnt = 0;for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i >= 0; --i) {uint32_t magic = ZIP_READ32(last64 + i);if (magic == kZipCdir64LocatorMagic && i + kZipCdir64LocatorSize <= amt &&- pread(zfd, last64, kZipCdir64HdrMinSize,- ZIP_LOCATE64_OFFSET(last64 + i)) == (long)kZipCdir64HdrMinSize &&+ pread(zfd, last64, kZipCdir64HdrMinSize, ZIP_LOCATE64_OFFSET(last64 + i)) ==+ (long)kZipCdir64HdrMinSize &&ZIP_READ32(last64) == kZipCdir64HdrMagic &&ZIP_CDIR64_RECORDS(last64) == ZIP_CDIR64_RECORDSONDISK(last64) &&ZIP_CDIR64_RECORDS(last64) && ZIP_CDIR64_SIZE(last64) <= INT_MAX) {@@ -136,80 +138,71 @@ static void CheckZip(const char *zpath) {break;}}- if (!cnt) {+ if (!cnt)amt = 0;- }// read central directoryuint8_t *cdir = Malloc(amt);- if (pread(zfd, cdir, amt, off) != amt) {+ if (pread(zfd, cdir, amt, off) != amt)DieSys(zpath);- }- if (ZIP_READ32(cdir) != kZipCfileHdrMagic) {+ if (ZIP_READ32(cdir) != kZipCfileHdrMagic)Die(zpath, "unable to locate central directory");- }unsigned entry_index, entry_offset;for (entry_index = entry_offset = 0;entry_index < cnt && entry_offset + kZipCfileHdrMinSize <= amt &&- entry_offset + ZIP_CFILE_HDRSIZE(cdir + entry_offset) <= amt;+ entry_offset + ZIP_CFILE_HDRSIZE(cdir + entry_offset) <= amt;++entry_index, entry_offset += ZIP_CFILE_HDRSIZE(cdir + entry_offset)) {- if (ZIP_CFILE_MAGIC(cdir + entry_offset) != kZipCfileHdrMagic) {- Die(zpath, "corrupted zip central directory entry magic");- }+ if (ZIP_CFILE_MAGIC(cdir + entry_offset) != kZipCfileHdrMagic)+ Die(zpath, "corrupted zip central "+ "directory entry magic");- // ignore entries with no content (e.g. dirs)- if (!get_zip_cfile_compressed_size(cdir + entry_offset)) {+ // ignore entries with no content (e.g.+ // dirs)+ if (!get_zip_cfile_compressed_size(cdir + entry_offset))continue;- }// read local file headeroff_t off = get_zip_cfile_offset(cdir + entry_offset);uint8_t lfile[kZipLfileHdrMinSize];- if (pread(zfd, lfile, kZipLfileHdrMinSize, off) != kZipLfileHdrMinSize) {+ if (pread(zfd, lfile, kZipLfileHdrMinSize, off) != kZipLfileHdrMinSize)Die(zpath, "failed to pread lfile");- }- if (ZIP_LFILE_MAGIC(lfile) != kZipLfileHdrMagic) {+ if (ZIP_LFILE_MAGIC(lfile) != kZipLfileHdrMagic)Die(zpath, "corrupted zip local file magic");- }// get offset of local file contentoff += ZIP_LFILE_HDRSIZE(lfile);-+// get alignment of local filelong align = 1ull << _bsfl(off);- printf("%.*s has alignment of %ld\n",- (int)ZIP_CFILE_NAMESIZE(cdir + entry_offset),- ZIP_CFILE_NAME(cdir + entry_offset),- align);+ printf("%.*s has alignment of %ld\n", (int)ZIP_CFILE_NAMESIZE(cdir + entry_offset),+ ZIP_CFILE_NAME(cdir + entry_offset), align);}// close input- if (close(zfd)) {+ if (close(zfd))DieSys(zpath);- }}int main(int argc, char *argv[]) {// get name of programprog = argv[0];- if (!prog) prog = "zipcheck";+ if (!prog)+ prog = "zipcheck";// parse flagsint opt;while ((opt = getopt(argc, argv, "h")) != -1) {switch (opt) {- case 'h':- PrintUsage(1, 0);- default:- PrintUsage(2, 1);+ case 'h':+ PrintUsage(1, 0);+ default:+ PrintUsage(2, 1);}}- if (optind == argc) {+ if (optind == argc)Die(prog, "missing input");- }- for (int i = optind; i < argc; ++i) {+ for (int i = optind; i < argc; ++i)CheckZip(argv[i]);- }}