Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Change file format magic number for bf16
Justine Tunney257491d9de32
diff --git a/build/config.mk b/build/config.mkindex 4726d271..eb102981 100644--- a/build/config.mk+++ b/build/config.mk@@ -2,7 +2,7 @@#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘PREFIX = /usr/local-COSMOCC = .cosmocc/3.3.2+COSMOCC = .cosmocc/3.3.3TOOLCHAIN = $(COSMOCC)/bin/cosmoAR = $(TOOLCHAIN)ar@@ -50,5 +50,5 @@ clean:; rm -rf o.PHONY: distcleandistclean:; rm -rf o .cosmocc-.cosmocc/3.3.2:- build/download-cosmocc.sh $@ 3.3.2 a695012ffbeac5e26e3c4a740debc15273f47e9a8bdc55e8b76a623154d5914b+.cosmocc/3.3.3:+ build/download-cosmocc.sh $@ 3.3.3 e4d0fa63cd79cc3bfff6c2d015f1776db081409907625aea8ad40cefc1996d08diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mkindex 5e86dae3..d51223d6 100644--- a/llama.cpp/BUILD.mk+++ b/llama.cpp/BUILD.mk@@ -27,6 +27,20 @@ include llama.cpp/perplexity/BUILD.mk$(LLAMA_CPP_OBJS): private CCFLAGS += -DGGML_MULTIPLATFORM+o/$(MODE)/llama.cpp/ggml-alloc.o \+o/$(MODE)/llama.cpp/ggml-backend.o \+o/$(MODE)/llama.cpp/grammar-parser.o \+o/$(MODE)/llama.cpp/json-schema-to-grammar.o \+o/$(MODE)/llama.cpp/llama.o \+o/$(MODE)/llama.cpp/stb_image.o \+o/$(MODE)/llama.cpp/unicode.o \+o/$(MODE)/llama.cpp/sampling.o \+o/$(MODE)/llama.cpp/ggml-alloc.o \+o/$(MODE)/llama.cpp/common.o: private \+ CCFLAGS += -Os++$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk+.PHONY: o/$(MODE)/llama.cppo/$(MODE)/llama.cpp: \o/$(MODE)/llama.cpp/main \diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 99d61411..fa756255 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -371,7 +371,8 @@ extern "C" {GGML_TYPE_I32 = 26,GGML_TYPE_I64 = 27,GGML_TYPE_F64 = 28,- GGML_TYPE_BF16 = 29,+ GGML_TYPE_IQ1_M = 29,+ GGML_TYPE_BF16 = 30,GGML_TYPE_COUNT,};@@ -411,7 +412,8 @@ extern "C" {GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors- GGML_FTYPE_MOSTLY_BF16 = 23, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors+ GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors};// available tensor operations:diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 230f820b..0df5216f 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -119,7 +119,8 @@ extern "C" {LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensorsLLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensorsLLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors- LLAMA_FTYPE_MOSTLY_BF16 = 31, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensorsLLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mkindex 4308066c..9b6ef38a 100644--- a/llama.cpp/llava/BUILD.mk+++ b/llama.cpp/llava/BUILD.mk@@ -19,6 +19,14 @@ o/$(MODE)/llama.cpp/llava/llava-quantize: \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llama.cpp/llava/clip.o \+o/$(MODE)/llama.cpp/llava/llava.o \+o/$(MODE)/llama.cpp/llava/llava-cli.o \+o/$(MODE)/llama.cpp/llava/llava-quantize.o: private \+ CCFLAGS += -Os++$(LLAMA_CPP_LLAVA_OBJS): llama.cpp/llava/BUILD.mk+.PHONY: o/$(MODE)/llama.cpp/llavao/$(MODE)/llama.cpp/llava: \o/$(MODE)/llama.cpp/llava/llava.a \diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mkindex 090be82e..991986bf 100644--- a/llama.cpp/main/BUILD.mk+++ b/llama.cpp/main/BUILD.mk@@ -16,6 +16,11 @@ o/$(MODE)/llama.cpp/main/main: \o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)+o/$(MODE)/llama.cpp/main/main.o: private \+ CCFLAGS += -Os++$(LLAMA_CPP_MAIN_OBJS): llama.cpp/main/BUILD.mk+.PHONY: o/$(MODE)/llama.cpp/maino/$(MODE)/llama.cpp/main: \o/$(MODE)/llama.cpp/main/maindiff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mkindex 15173797..9cf6499a 100644--- a/llama.cpp/server/BUILD.mk+++ b/llama.cpp/server/BUILD.mk@@ -14,6 +14,11 @@ o/$(MODE)/llama.cpp/server/impl.o: private CXXFLAGS += -O1o/$(MODE)/llama.cpp/server/server.a: \$(LLAMA_CPP_SERVER_OBJS)+o/$(MODE)/llama.cpp/server/server.o: private \+ CCFLAGS += -Os++$(LLAMA_CPP_SERVER_OBJS): llama.cpp/server/BUILD.mk+.PHONY: o/$(MODE)/llama.cpp/servero/$(MODE)/llama.cpp/server: \o/$(MODE)/llama.cpp/server/server.adiff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex a9e95b51..3b4760ed 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -70,6 +70,10 @@ o/$(MODE)/llamafile/sgemm_sss_test: \o/$(MODE)/llamafile/sgemm_sss_test.o \o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llamafile/fun: \+ o/$(MODE)/llamafile/fun.o \+ o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llamafile/sgemm_vecdot_test: \o/$(MODE)/llamafile/sgemm_vecdot_test.o \o/$(MODE)/llama.cpp/llama.cpp.adiff --git a/llamafile/zip.c b/llamafile/zip.cindex 36ef0c35..ba623745 100644--- a/llamafile/zip.c+++ b/llamafile/zip.c@@ -18,9 +18,9 @@#include "zip.h"#include <stdint.h>-int64_t get_zip_cfile_compressed_size(const uint8_t *z) {- if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu)- return ZIP_CFILE_COMPRESSEDSIZE(z);+int64_t get_zip_cfile_uncompressed_size(const uint8_t *z) {+ if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) != 0xFFFFFFFFu)+ return ZIP_CFILE_UNCOMPRESSEDSIZE(z);const uint8_t *p = ZIP_CFILE_EXTRA(z);const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))@@ -30,6 +30,22 @@ int64_t get_zip_cfile_compressed_size(const uint8_t *z) {return -1;}+int64_t get_zip_cfile_compressed_size(const uint8_t *z) {+ if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu)+ return ZIP_CFILE_COMPRESSEDSIZE(z);+ const uint8_t *p = ZIP_CFILE_EXTRA(z);+ const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);+ for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))+ if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {+ int offset = 0;+ if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu)+ offset += 8;+ if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p))+ return ZIP_READ64(ZIP_EXTRA_CONTENT(p) + offset);+ }+ return -1;+}+int64_t get_zip_cfile_offset(const uint8_t *z) {if (ZIP_CFILE_OFFSET(z) != 0xFFFFFFFFu)return ZIP_CFILE_OFFSET(z);@@ -38,10 +54,10 @@ int64_t get_zip_cfile_offset(const uint8_t *z) {for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {int offset = 0;- if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu)- offset += 8;if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu)offset += 8;+ if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu)+ offset += 8;if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p))return ZIP_READ64(ZIP_EXTRA_CONTENT(p) + offset);}