Skip to content

Distribute and run LLMs with a single file.

2.5 KB
Raw Blame
diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
--- a/llama.cpp/ggml/src/gguf.cpp
+++ b/llama.cpp/ggml/src/gguf.cpp
@@ -3,6 +3,10 @@
 #include "ggml-impl.h"
 #include "gguf.h"
 
+#ifdef COSMOCC
+#include "llamafile/llamafile.h"
+#endif
+
 #include <cinttypes>
 #include <cstddef>
 #include <cstdint>
@@ -1014,7 +1018,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru
     return gguf_init_from_reader(gr, params);
 }
 
+#ifdef COSMOCC
+// [llamafile] Reader callback that reads through the llamafile API so we can
+// handle plain GGUF, /zip/ paths, [email protected], and .llamafile containers.
+struct gguf_llamafile_reader {
+    struct llamafile * lfile;
+    uint64_t offset;
+};
+
+static size_t gguf_llamafile_reader_callback(void * userdata, void * output, uint64_t offset, size_t len) {
+    GGML_ASSERT(len > 0);
+
+    gguf_llamafile_reader & reader = *static_cast<gguf_llamafile_reader *>(userdata);
+
+    if (reader.offset != offset) {
+        if (!llamafile_seek(reader.lfile, static_cast<size_t>(offset), SEEK_SET)) {
+            return 0;
+        }
+        reader.offset = offset;
+    }
+
+    const long nread = llamafile_read(reader.lfile, output, len);
+    if (nread <= 0) {
+        return 0;
+    }
+    reader.offset += static_cast<uint64_t>(nread);
+    return static_cast<size_t>(nread);
+}
+#endif
+
 struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
+#ifdef COSMOCC
+    // [llamafile] Route all GGUF file loads through the llamafile API so that
+    // /zip/ paths and .llamafile containers work transparently.
+    struct llamafile * lfile = llamafile_open_gguf(fname, "rb");
+    if (!lfile) {
+        GGML_LOG_ERROR("%s: failed to open GGUF file '%s': %s\n", __func__, fname, strerror(errno));
+        return nullptr;
+    }
+
+    gguf_llamafile_reader reader = {
+        /*.lfile  = */ lfile,
+        /*.offset = */ 0,
+    };
+    const struct gguf_reader gr(gguf_llamafile_reader_callback, &reader, SIZE_MAX, 0, llamafile_size(lfile));
+    struct gguf_context * result = gguf_init_from_reader(gr, params);
+    llamafile_close(lfile);
+    return result;
+#else
     FILE * file = ggml_fopen(fname, "rb");
 
     if (!file) {
@@ -1025,6 +1076,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
     struct gguf_context * result = gguf_init_from_file_ptr(file, params);
     fclose(file);
     return result;
+#endif
 }
 
 void gguf_free(struct gguf_context * ctx) {