Skip to content

Distribute and run LLMs with a single file.

Change file format magic number for bf16

Justine Tunney257491d9de32
diff --git a/build/config.mk b/build/config.mk
index 4726d271..eb102981 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -2,7 +2,7 @@
#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
PREFIX = /usr/local
-COSMOCC = .cosmocc/3.3.2
+COSMOCC = .cosmocc/3.3.3
TOOLCHAIN = $(COSMOCC)/bin/cosmo
AR = $(TOOLCHAIN)ar
@@ -50,5 +50,5 @@ clean:; rm -rf o
.PHONY: distclean
distclean:; rm -rf o .cosmocc
-.cosmocc/3.3.2:
- build/download-cosmocc.sh $@ 3.3.2 a695012ffbeac5e26e3c4a740debc15273f47e9a8bdc55e8b76a623154d5914b
+.cosmocc/3.3.3:
+ build/download-cosmocc.sh $@ 3.3.3 e4d0fa63cd79cc3bfff6c2d015f1776db081409907625aea8ad40cefc1996d08
diff --git a/llama.cpp/BUILD.mk b/llama.cpp/BUILD.mk
index 5e86dae3..d51223d6 100644
--- a/llama.cpp/BUILD.mk
+++ b/llama.cpp/BUILD.mk
@@ -27,6 +27,20 @@ include llama.cpp/perplexity/BUILD.mk
$(LLAMA_CPP_OBJS): private CCFLAGS += -DGGML_MULTIPLATFORM
+o/$(MODE)/llama.cpp/ggml-alloc.o \
+o/$(MODE)/llama.cpp/ggml-backend.o \
+o/$(MODE)/llama.cpp/grammar-parser.o \
+o/$(MODE)/llama.cpp/json-schema-to-grammar.o \
+o/$(MODE)/llama.cpp/llama.o \
+o/$(MODE)/llama.cpp/stb_image.o \
+o/$(MODE)/llama.cpp/unicode.o \
+o/$(MODE)/llama.cpp/sampling.o \
+o/$(MODE)/llama.cpp/ggml-alloc.o \
+o/$(MODE)/llama.cpp/common.o: private \
+ CCFLAGS += -Os
+
+$(LLAMA_CPP_OBJS): llama.cpp/BUILD.mk
+
.PHONY: o/$(MODE)/llama.cpp
o/$(MODE)/llama.cpp: \
o/$(MODE)/llama.cpp/main \
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index 99d61411..fa756255 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -371,7 +371,8 @@ extern "C" {
GGML_TYPE_I32 = 26,
GGML_TYPE_I64 = 27,
GGML_TYPE_F64 = 28,
- GGML_TYPE_BF16 = 29,
+ GGML_TYPE_IQ1_M = 29,
+ GGML_TYPE_BF16 = 30,
GGML_TYPE_COUNT,
};
@@ -411,7 +412,8 @@ extern "C" {
GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
- GGML_FTYPE_MOSTLY_BF16 = 23, // except 1d tensors
+ GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
+ GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
};
// available tensor operations:
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 230f820b..0df5216f 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -119,7 +119,8 @@ extern "C" {
LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors
LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors
LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors
- LLAMA_FTYPE_MOSTLY_BF16 = 31, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
};
diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mk
index 4308066c..9b6ef38a 100644
--- a/llama.cpp/llava/BUILD.mk
+++ b/llama.cpp/llava/BUILD.mk
@@ -19,6 +19,14 @@ o/$(MODE)/llama.cpp/llava/llava-quantize: \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a
+o/$(MODE)/llama.cpp/llava/clip.o \
+o/$(MODE)/llama.cpp/llava/llava.o \
+o/$(MODE)/llama.cpp/llava/llava-cli.o \
+o/$(MODE)/llama.cpp/llava/llava-quantize.o: private \
+ CCFLAGS += -Os
+
+$(LLAMA_CPP_LLAVA_OBJS): llama.cpp/llava/BUILD.mk
+
.PHONY: o/$(MODE)/llama.cpp/llava
o/$(MODE)/llama.cpp/llava: \
o/$(MODE)/llama.cpp/llava/llava.a \
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index 090be82e..991986bf 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -16,6 +16,11 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
+o/$(MODE)/llama.cpp/main/main.o: private \
+ CCFLAGS += -Os
+
+$(LLAMA_CPP_MAIN_OBJS): llama.cpp/main/BUILD.mk
+
.PHONY: o/$(MODE)/llama.cpp/main
o/$(MODE)/llama.cpp/main: \
o/$(MODE)/llama.cpp/main/main
diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mk
index 15173797..9cf6499a 100644
--- a/llama.cpp/server/BUILD.mk
+++ b/llama.cpp/server/BUILD.mk
@@ -14,6 +14,11 @@ o/$(MODE)/llama.cpp/server/impl.o: private CXXFLAGS += -O1
o/$(MODE)/llama.cpp/server/server.a: \
$(LLAMA_CPP_SERVER_OBJS)
+o/$(MODE)/llama.cpp/server/server.o: private \
+ CCFLAGS += -Os
+
+$(LLAMA_CPP_SERVER_OBJS): llama.cpp/server/BUILD.mk
+
.PHONY: o/$(MODE)/llama.cpp/server
o/$(MODE)/llama.cpp/server: \
o/$(MODE)/llama.cpp/server/server.a
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index a9e95b51..3b4760ed 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -70,6 +70,10 @@ o/$(MODE)/llamafile/sgemm_sss_test: \
o/$(MODE)/llamafile/sgemm_sss_test.o \
o/$(MODE)/llama.cpp/llama.cpp.a
+o/$(MODE)/llamafile/fun: \
+ o/$(MODE)/llamafile/fun.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a
+
o/$(MODE)/llamafile/sgemm_vecdot_test: \
o/$(MODE)/llamafile/sgemm_vecdot_test.o \
o/$(MODE)/llama.cpp/llama.cpp.a
diff --git a/llamafile/zip.c b/llamafile/zip.c
index 36ef0c35..ba623745 100644
--- a/llamafile/zip.c
+++ b/llamafile/zip.c
@@ -18,9 +18,9 @@
#include "zip.h"
#include <stdint.h>
-int64_t get_zip_cfile_compressed_size(const uint8_t *z) {
- if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu)
- return ZIP_CFILE_COMPRESSEDSIZE(z);
+int64_t get_zip_cfile_uncompressed_size(const uint8_t *z) {
+ if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) != 0xFFFFFFFFu)
+ return ZIP_CFILE_UNCOMPRESSEDSIZE(z);
const uint8_t *p = ZIP_CFILE_EXTRA(z);
const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);
for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))
@@ -30,6 +30,22 @@ int64_t get_zip_cfile_compressed_size(const uint8_t *z) {
return -1;
}
+int64_t get_zip_cfile_compressed_size(const uint8_t *z) {
+ if (ZIP_CFILE_COMPRESSEDSIZE(z) != 0xFFFFFFFFu)
+ return ZIP_CFILE_COMPRESSEDSIZE(z);
+ const uint8_t *p = ZIP_CFILE_EXTRA(z);
+ const uint8_t *pe = p + ZIP_CFILE_EXTRASIZE(z);
+ for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))
+ if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {
+ int offset = 0;
+ if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu)
+ offset += 8;
+ if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p))
+ return ZIP_READ64(ZIP_EXTRA_CONTENT(p) + offset);
+ }
+ return -1;
+}
+
int64_t get_zip_cfile_offset(const uint8_t *z) {
if (ZIP_CFILE_OFFSET(z) != 0xFFFFFFFFu)
return ZIP_CFILE_OFFSET(z);
@@ -38,10 +54,10 @@ int64_t get_zip_cfile_offset(const uint8_t *z) {
for (; p + ZIP_EXTRA_SIZE(p) <= pe; p += ZIP_EXTRA_SIZE(p))
if (ZIP_EXTRA_HEADERID(p) == kZipExtraZip64) {
int offset = 0;
- if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu)
- offset += 8;
if (ZIP_CFILE_UNCOMPRESSEDSIZE(z) == 0xFFFFFFFFu)
offset += 8;
+ if (ZIP_CFILE_COMPRESSEDSIZE(z) == 0xFFFFFFFFu)
+ offset += 8;
if (offset + 8 <= ZIP_EXTRA_CONTENTSIZE(p))
return ZIP_READ64(ZIP_EXTRA_CONTENT(p) + offset);
}