Skip to content

Distribute and run LLMs with a single file.

Make zipalign and slicehf gentler on system

Justine Tunney909f791666fb
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index ca9fb3b6..abcd9c9e 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -1567,7 +1567,8 @@ void ggml_once(atomic_uint * once, void init(void)) {
}
int ggml_delay(int backoff) {
- if (backoff < 12) {
+#if 0
+ if (backoff < 7) {
volatile int i;
for (i = 0; i != 1 << backoff; i++) {
}
@@ -1575,6 +1576,7 @@ int ggml_delay(int backoff) {
} else {
pthread_yield_np();
}
+#endif
return backoff;
}
@@ -1587,9 +1589,9 @@ void ggml_syncthreads(struct ggml_barrier * b, int nth) {
atomic_store_explicit(&b->phase, phase + 1, memory_order_release);
} else {
int backoff = 0;
- while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase) {
- backoff = ggml_delay(backoff);
- }
+ while (atomic_load_explicit(&b->phase, memory_order_acquire) == phase)
+ while (atomic_load_explicit(&b->phase, memory_order_relaxed) == phase)
+ backoff = ggml_delay(backoff);
}
sync_wait_end();
}
@@ -18682,7 +18684,11 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_comput
while (true) {
* node_n = atomic_load_explicit(&state->shared->node_n, memory_order_acquire);
if (* node_n != last_node_n) break;
- backoff = ggml_delay(backoff);
+ while (true) {
+ * node_n = atomic_load_explicit(&state->shared->node_n, memory_order_relaxed);
+ if (* node_n != last_node_n) break;
+ backoff = ggml_delay(backoff);
+ }
}
sync_wait_end();
}
@@ -18696,7 +18702,11 @@ static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_co
while (true) {
* task_phase = atomic_load_explicit(&state->shared->node_task, memory_order_acquire);
if (* task_phase != last_task_phase) break;
- backoff = ggml_delay(backoff);
+ while (true) {
+ * task_phase = atomic_load_explicit(&state->shared->node_task, memory_order_relaxed);
+ if (* task_phase != last_task_phase) break;
+ backoff = ggml_delay(backoff);
+ }
}
sync_wait_end();
}
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index bf1c62d4..aad9f494 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -3590,7 +3590,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {
}
}
-static const char * llama_model_type_name(e_model type) {
+static const char * llama_model_type_name(char *buf, size_t size, e_model type) {
switch (type) {
case MODEL_22M: return "22M";
case MODEL_33M: return "33M";
@@ -3622,7 +3622,9 @@ static const char * llama_model_type_name(e_model type) {
case MODEL_8x7B: return "8x7B";
case MODEL_8x22B: return "8x22B";
case MODEL_16x12B: return "16x12B";
- default: return "?B";
+ default:
+ snprintf(buf, size, "?%d", type);
+ return buf;
}
}
@@ -4554,6 +4556,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
const auto & vocab = model.vocab;
const char * rope_scaling_type = LLAMA_ROPE_SCALING_TYPES.at(hparams.rope_scaling_type_train);
+ char buf[64];
// hparams
LLAMA_LOG_INFO("%s: format = %s\n", __func__, llama_file_version_name(ml.fver));
@@ -4593,7 +4596,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);
LLAMA_LOG_INFO("%s: ssm_d_state = %u\n", __func__, hparams.ssm_d_state);
LLAMA_LOG_INFO("%s: ssm_dt_rank = %u\n", __func__, hparams.ssm_dt_rank);
- LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(model.type));
+ LLAMA_LOG_INFO("%s: model type = %s\n", __func__, llama_model_type_name(buf, sizeof(buf), model.type));
LLAMA_LOG_INFO("%s: model ftype = %s\n", __func__, llama_model_ftype_name(model.ftype).c_str());
if (ml.n_elements >= 1e12) {
LLAMA_LOG_INFO("%s: model params = %.2f T\n", __func__, ml.n_elements*1e-12);
@@ -15976,9 +15979,10 @@ int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int3
}
int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {
+ char buf2[64];
return snprintf(buf, buf_size, "%s %s %s",
llama_model_arch_name(model->arch),
- llama_model_type_name(model->type),
+ llama_model_type_name(buf2, sizeof(buf2), model->type),
llama_model_ftype_name(model->ftype).c_str());
}
diff --git a/llamafile/slicehf.c b/llamafile/slicehf.c
index 8ab42d2c..9463134f 100644
--- a/llamafile/slicehf.c
+++ b/llamafile/slicehf.c
@@ -26,6 +26,8 @@
#define MAX_SIZE (50L * 1000 * 1000 * 1000)
+static char in_path_noext[PATH_MAX + 1];
+
static wontreturn void Die(const char *thing, const char *reason) {
tinyprint(2, thing, ": ", reason, "\n", NULL);
exit(1);
@@ -37,9 +39,19 @@ static wontreturn void DieSys(const char *thing) {
}
void slicehf(const char *in_path) {
+
+ off_t in_off = 0;
int in_fd = open(in_path, O_RDONLY);
if (in_fd == -1)
DieSys(in_path);
+ posix_fadvise(in_fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+ // split file extension
+ char *ext;
+ strlcpy(in_path_noext, in_path, sizeof(in_path_noext));
+ if (!(ext = strrchr(in_path_noext, '.')) || strchr(ext, '/'))
+ Die(in_path, "file missing extension");
+ *ext++ = '\0';
struct stat st;
if (fstat(in_fd, &st) == -1)
@@ -55,17 +67,19 @@ void slicehf(const char *in_path) {
long amount = MIN(total, MAX_SIZE);
total -= amount;
char out_path[PATH_MAX];
- snprintf(out_path, PATH_MAX, "%s.cat%d", in_path, item++);
+ snprintf(out_path, PATH_MAX, "%s.cat%d.%s", in_path_noext, item++, ext);
+ off_t out_off = 0;
int out_fd = open(out_path, O_WRONLY | O_TRUNC | O_CREAT, 0644);
if (out_fd == -1)
DieSys(out_path);
while (amount) {
- long chunk = MIN(amount, 65536);
- long copied = copy_file_range(in_fd, 0, out_fd, 0, chunk, 0);
+ long chunk = MIN(amount, 2097152);
+ long copied = copy_file_range(in_fd, &in_off, out_fd, &out_off, chunk, 0);
if (copied == -1)
DieSys(out_path);
if (!copied)
Die(in_path, "unexpected eof");
+ posix_fadvise(in_fd, in_off - copied, copied, POSIX_FADV_DONTNEED);
amount -= copied;
}
close(out_fd);
@@ -75,7 +89,10 @@ void slicehf(const char *in_path) {
}
int main(int argc, char *argv[]) {
- for (int i = 1; i < argc; ++i) {
+
+ // use idle scheduling priority
+ verynice();
+
+ for (int i = 1; i < argc; ++i)
slicehf(argv[i]);
- }
}
diff --git a/llamafile/zipalign.c b/llamafile/zipalign.c
index 5fdf3304..a3775cb8 100644
--- a/llamafile/zipalign.c
+++ b/llamafile/zipalign.c
@@ -142,6 +142,9 @@ int main(int argc, char *argv[]) {
if (optind == argc)
Die(prog, "missing output argument");
+ // use idle scheduling priority
+ verynice();
+
// open output file
int zfd;
ssize_t zsize;
@@ -257,6 +260,7 @@ int main(int argc, char *argv[]) {
const char *path = argv[i];
if ((fd = open(path, O_RDONLY)) == -1)
DieSys(path);
+ posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
// get information about file
uint64_t size;
@@ -315,6 +319,7 @@ int main(int argc, char *argv[]) {
// read chunk
if ((rc = pread(fd, iobuf, Min(size, CHUNK), i)) <= 0)
DieSys(path);
+ posix_fadvise(fd, i, Min(size, CHUNK), POSIX_FADV_DONTNEED);
crc = crc32(crc, iobuf, rc);
if (!flag_level) {
// write uncompressed chunk to output