Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
--- a/llama.cpp/ggml/src/gguf.cpp
+++ b/llama.cpp/ggml/src/gguf.cpp
@@ -3,6 +3,10 @@
#include "ggml-impl.h"
#include "gguf.h"
+#ifdef COSMOCC
+#include "llamafile/llamafile.h"
+#endif
+
#include <cinttypes>
#include <cstddef>
#include <cstdint>
@@ -1014,7 +1018,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru
return gguf_init_from_reader(gr, params);
}
+#ifdef COSMOCC
+// [llamafile] Reader callback that reads through the llamafile API so we can
+// handle plain GGUF, /zip/ paths, [email protected], and .llamafile containers.
+struct gguf_llamafile_reader {
+ struct llamafile * lfile;
+ uint64_t offset;
+};
+
+static size_t gguf_llamafile_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) {
+ GGML_ASSERT(len > 0);
+
+ gguf_llamafile_reader & reader = *static_cast<gguf_llamafile_reader *>(userdata);
+
+ if (reader.offset != offset) {
+ if (!llamafile_seek(reader.lfile, static_cast<size_t>(offset), SEEK_SET)) {
+ return 0;
+ }
+ reader.offset = offset;
+ }
+
+ const long nread = llamafile_read(reader.lfile, output, len);
+ if (nread <= 0) {
+ return 0;
+ }
+ reader.offset += static_cast<uint64_t>(nread);
+ return static_cast<size_t>(nread);
+}
+#endif
+
struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
+#ifdef COSMOCC
+ // [llamafile] Route all GGUF file loads through the llamafile API so that
+ // /zip/ paths and .llamafile containers work transparently.
+ struct llamafile * lfile = llamafile_open_gguf(fname, "rb");
+ if (!lfile) {
+ GGML_LOG_ERROR("%s: failed to open GGUF file '%s': %s\n", __func__, fname, strerror(errno));
+ return nullptr;
+ }
+
+ gguf_llamafile_reader reader = {
+ /*.lfile = */ lfile,
+ /*.offset = */ 0,
+ };
+ const struct gguf_reader gr(gguf_llamafile_reader_callback, &reader, SIZE_MAX, 0, llamafile_size(lfile));
+ struct gguf_context * result = gguf_init_from_reader(gr, params);
+ llamafile_close(lfile);
+ return result;
+#else
FILE * file = ggml_fopen(fname, "rb");
if (!file) {
@@ -1025,6 +1076,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
struct gguf_context * result = gguf_init_from_file_ptr(file, params);
fclose(file);
return result;
+#endif
}
void gguf_free(struct gguf_context * ctx) {