Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
--- a/llama.cpp/src/llama-mmap.cpp
+++ b/llama.cpp/src/llama-mmap.cpp
@@ -4,6 +4,11 @@
#include "ggml.h"
+#ifdef COSMOCC
+#include "llamafile/llamafile.h"
+#include <cosmo.h>
+#endif
+
#include <cstring>
#include <climits>
#include <cstdlib>
@@ -182,6 +187,15 @@ struct llama_file::impl {
}
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
+#ifdef COSMOCC
+ // [llamafile] Use llamafile_open_gguf for all file opening
+ lfile = llamafile_open_gguf(fname, mode);
+ if (lfile == NULL) {
+ throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
+ }
+ size = llamafile_size(lfile);
+ return;
+#endif
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
@@ -234,6 +248,9 @@ struct llama_file::impl {
}
size_t tell() const {
+#ifdef COSMOCC
+ return llamafile_tell(lfile);
+#endif
if (fd == -1) {
off_t ret = llama_mmap_ftell(fp);
if (ret == -1) {
@@ -251,6 +268,11 @@ struct llama_file::impl {
}
void seek(size_t offset, int whence) const {
+#ifdef COSMOCC
+ if (!llamafile_seek(lfile, offset, whence)) {
+ throw std::runtime_error(format("seek error: %s", strerror(errno)));
+ }
+#else
off_t ret = 0;
if (fd == -1) {
ret = llama_mmap_fseek(fp, offset, whence);
@@ -260,12 +282,22 @@ struct llama_file::impl {
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
+#endif
}
void read_raw_unsafe(void * ptr, size_t len) {
if (len == 0) {
return;
}
+#ifdef COSMOCC
+ long rc = llamafile_read(lfile, ptr, len);
+ if (rc == -1) {
+ throw std::runtime_error(format("read error: %s", strerror(errno)));
+ }
+ if ((size_t)rc != len) {
+ throw std::runtime_error(format("short read: expected %zu bytes, got %ld", len, rc));
+ }
+#else
errno = 0;
if (fd == -1) {
const size_t curr_off = tell();
@@ -315,6 +347,7 @@ struct llama_file::impl {
bytes_read += (size_t) ret;
}
}
+#endif
}
void read_aligned_chunk(void * dest, size_t size) {
@@ -388,13 +421,20 @@ struct llama_file::impl {
}
~impl() {
+#ifdef COSMOCC
+ llamafile_close(lfile);
+#else
if (fd != -1) {
close(fd);
} else if (owns_fp) {
std::fclose(fp);
}
+#endif
}
int fd = -1;
+#ifdef COSMOCC
+ llamafile * lfile = nullptr;
+#endif
std::string fname;
#endif
@@ -423,6 +463,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
int llama_file::file_id() const {
+#ifdef COSMOCC
+ FILE * fp = llamafile_fp(pimpl->lfile);
+ return fp ? fileno(fp) : -1;
+#else
#ifdef _WIN32
return _fileno(pimpl->fp);
#else
@@ -430,9 +474,10 @@ int llama_file::file_id() const {
return pimpl->fd;
}
#if defined(fileno)
- return fileno(pimpl->fp);
+ return pimpl->fp ? fileno(pimpl->fp) : -1;
#else
- return ::fileno(pimpl->fp);
+ return pimpl->fp ? ::fileno(pimpl->fp) : -1;
+#endif
#endif
#endif
}
@@ -450,6 +495,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
+bool llama_file::has_premapped_content() const {
+#ifdef COSMOCC
+ return !llamafile_fp(pimpl->lfile);
+#else
+ return false;
+#endif
+}
+
+void * llama_file::premapped_content() const {
+#ifdef COSMOCC
+ if (!llamafile_fp(pimpl->lfile)) {
+ return llamafile_content(pimpl->lfile);
+ }
+#endif
+ return nullptr;
+}
+
+void * llama_file::get_llamafile() const {
+#ifdef COSMOCC
+ return pimpl->lfile;
+#else
+ return nullptr;
+#endif
+}
+
// llama_mmap
#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
@@ -478,9 +548,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t li
struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
+ bool is_owned = true;
+#ifdef COSMOCC
+ llamafile * lfile = nullptr;
+#endif
impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {
size = file->size();
+
+ if (file->has_premapped_content()) {
+ addr = file->premapped_content();
+ is_owned = false;
+#ifdef COSMOCC
+ lfile = (llamafile *)file->get_llamafile();
+ if (lfile) {
+ llamafile_ref(lfile);
+ }
+#endif
+ return;
+ }
+
int fd = file->file_id();
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
@@ -541,6 +628,18 @@ struct llama_mmap::impl {
}
void unmap_fragment(size_t first, size_t last) {
+ if (!is_owned) {
+ return;
+ }
+#ifdef COSMOCC
+ // Windows can only unmap a whole view (cosmo's munmap fails with ENOTSUP or
+ // EINVAL on a sub-range), so do what the _WIN32 branch below does: keep the
+ // fragments, and let the destructor unmap the whole file in one call.
+ if (IsWindows()) {
+ return;
+ }
+#endif
+
int page_size = sysconf(_SC_PAGESIZE);
align_range(&first, &last, page_size);
size_t len = last - first;
@@ -577,6 +676,14 @@ struct llama_mmap::impl {
}
~impl() {
+ if (!is_owned) {
+#ifdef COSMOCC
+ if (lfile) {
+ llamafile_unref(lfile);
+ }
+#endif
+ return;
+ }
for (const auto & frag : mapped_fragments) {
if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));