Skip to content

Distribute and run LLMs with a single file.

6.1 KB
Raw Blame
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
--- a/llama.cpp/src/llama-mmap.cpp
+++ b/llama.cpp/src/llama-mmap.cpp
@@ -4,6 +4,11 @@
 
 #include "ggml.h"
 
+#ifdef COSMOCC
+#include "llamafile/llamafile.h"
+#include <cosmo.h>
+#endif
+
 #include <cstring>
 #include <climits>
 #include <cstdlib>
@@ -182,6 +187,15 @@ struct llama_file::impl {
     }
 #else
     impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
+#ifdef COSMOCC
+        // [llamafile] Use llamafile_open_gguf for all file opening
+        lfile = llamafile_open_gguf(fname, mode);
+        if (lfile == NULL) {
+            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+        }
+        size = llamafile_size(lfile);
+        return;
+#endif
 #ifdef __linux__
         // Try unbuffered I/O for read only
         if (use_direct_io && std::strcmp(mode, "rb") == 0) {
@@ -234,6 +248,9 @@ struct llama_file::impl {
     }
 
     size_t tell() const {
+#ifdef COSMOCC
+        return llamafile_tell(lfile);
+#endif
         if (fd == -1) {
             off_t ret = llama_mmap_ftell(fp);
             if (ret == -1) {
@@ -251,6 +268,11 @@ struct llama_file::impl {
     }
 
     void seek(size_t offset, int whence) const {
+#ifdef COSMOCC
+        if (!llamafile_seek(lfile, offset, whence)) {
+            throw std::runtime_error(format("seek error: %s", strerror(errno)));
+        }
+#else
         off_t ret = 0;
         if (fd == -1) {
             ret = llama_mmap_fseek(fp, offset, whence);
@@ -260,12 +282,22 @@ struct llama_file::impl {
         if (ret == -1) {
             throw std::runtime_error(format("seek error: %s", strerror(errno)));
         }
+#endif
     }
 
     void read_raw_unsafe(void * ptr, size_t len) {
         if (len == 0) {
             return;
         }
+#ifdef COSMOCC
+        long rc = llamafile_read(lfile, ptr, len);
+        if (rc == -1) {
+            throw std::runtime_error(format("read error: %s", strerror(errno)));
+        }
+        if ((size_t)rc != len) {
+            throw std::runtime_error(format("short read: expected %zu bytes, got %ld", len, rc));
+        }
+#else
         errno = 0;
         if (fd == -1) {
             const size_t curr_off = tell();
@@ -315,6 +347,7 @@ struct llama_file::impl {
                 bytes_read += (size_t) ret;
             }
         }
+#endif
     }
 
     void read_aligned_chunk(void * dest, size_t size) {
@@ -388,13 +421,20 @@ struct llama_file::impl {
     }
 
     ~impl() {
+#ifdef COSMOCC
+        llamafile_close(lfile);
+#else
         if (fd != -1) {
             close(fd);
         } else if (owns_fp) {
             std::fclose(fp);
         }
+#endif
     }
     int fd = -1;
+#ifdef COSMOCC
+    llamafile * lfile = nullptr;
+#endif
     std::string fname;
 #endif
 
@@ -423,6 +463,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
 bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
 
 int llama_file::file_id() const {
+#ifdef COSMOCC
+    FILE * fp = llamafile_fp(pimpl->lfile);
+    return fp ? fileno(fp) : -1;
+#else
 #ifdef _WIN32
     return _fileno(pimpl->fp);
 #else
@@ -430,9 +474,10 @@ int llama_file::file_id() const {
         return pimpl->fd;
     }
 #if defined(fileno)
-    return fileno(pimpl->fp);
+    return pimpl->fp ? fileno(pimpl->fp) : -1;
 #else
-    return ::fileno(pimpl->fp);
+    return pimpl->fp ? ::fileno(pimpl->fp) : -1;
+#endif
 #endif
 #endif
 }
@@ -450,6 +495,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
 void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
 void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
 
+bool llama_file::has_premapped_content() const {
+#ifdef COSMOCC
+    return !llamafile_fp(pimpl->lfile);
+#else
+    return false;
+#endif
+}
+
+void * llama_file::premapped_content() const {
+#ifdef COSMOCC
+    if (!llamafile_fp(pimpl->lfile)) {
+        return llamafile_content(pimpl->lfile);
+    }
+#endif
+    return nullptr;
+}
+
+void * llama_file::get_llamafile() const {
+#ifdef COSMOCC
+    return pimpl->lfile;
+#else
+    return nullptr;
+#endif
+}
+
 // llama_mmap
 
 #if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
@@ -478,9 +548,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t li
 struct llama_mmap::impl {
 #ifdef _POSIX_MAPPED_FILES
     std::vector<std::pair<size_t, size_t>> mapped_fragments;
+    bool is_owned = true;
+#ifdef COSMOCC
+    llamafile * lfile = nullptr;
+#endif
 
     impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
         size = file->size();
+
+        if (file->has_premapped_content()) {
+            addr = file->premapped_content();
+            is_owned = false;
+#ifdef COSMOCC
+            lfile = (llamafile *)file->get_llamafile();
+            if (lfile) {
+                llamafile_ref(lfile);
+            }
+#endif
+            return;
+        }
+
         int fd = file->file_id();
         int flags = MAP_SHARED;
         if (numa) { prefetch = 0; }
@@ -541,6 +628,18 @@ struct llama_mmap::impl {
     }
 
     void unmap_fragment(size_t first, size_t last) {
+        if (!is_owned) {
+            return;
+        }
+#ifdef COSMOCC
+        // Windows can only unmap a whole view (cosmo's munmap fails with ENOTSUP or
+        // EINVAL on a sub-range), so do what the _WIN32 branch below does: keep the
+        // fragments, and let the destructor unmap the whole file in one call.
+        if (IsWindows()) {
+            return;
+        }
+#endif
+
         int page_size = sysconf(_SC_PAGESIZE);
         align_range(&first, &last, page_size);
         size_t len = last - first;
@@ -577,6 +676,14 @@ struct llama_mmap::impl {
     }
 
     ~impl() {
+        if (!is_owned) {
+#ifdef COSMOCC
+            if (lfile) {
+                llamafile_unref(lfile);
+            }
+#endif
+            return;
+        }
         for (const auto & frag : mapped_fragments) {
             if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
                 LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));