Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make ctrl-c work better
Justine Tunneybaf216ad1e97
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex ee591c74..9510b544 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -31,6 +31,7 @@ LOCAL MODIFICATIONS- Add support to main() programs for Cosmo /zip/.args files- Use condition variables for server rather than busy loop- Introduce pledge() SECCOMP sandboxing to improve security+ - Introduce ggml_interrupt() to gracefully handle signals- Remove log callback pointer API from Metal GPU module- Use runtime dispatching for matmul quants- Remove operating system #ifdef statementsdiff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 187641d4..9480a5bc 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -103,6 +103,12 @@ SOFTWARE.\"");// end of logging block//+static volatile bool is_interrupted;++void ggml_interrupt(bool state) {+ is_interrupted = state;+}+static void * ggml_aligned_malloc(size_t size) {if (size == 0) {GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");@@ -15955,7 +15961,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {int node_n = -1;- while (true) {+ while (!is_interrupted) {if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {state->shared->node_n += 1;return (thread_ret_t) GGML_EXIT_ABORTED;@@ -16025,7 +16031,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {} else {// wait for other threads to finishconst int last = node_n;- while (true) {+ while (!is_interrupted) {// TODO: this sched_yield can have significant impact on the performance - either positive or negative// depending on the workload and the operating system.// since it is not clear what is the best approach, it should potentially become user-configurablediff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex bdd8a398..db102bc8 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -2113,6 +2113,8 @@ extern "C" {GGML_API int ggml_cpu_has_ssse3 (void);GGML_API int ggml_cpu_has_vsx (void);+ GGML_API void ggml_interrupt(bool);+//// Internal types and functions exposed for tests and benchmarks//diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 86087441..e66ee094 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -30,12 +30,6 @@ static gpt_params * g_params;static volatile bool is_terminated;static volatile bool is_interacting;-static void acknowledge_shutdown(void) {- if (is_terminal) {- write(2, "^C", 2);- }-}-static void sigint_handler(int signo) {if (g_params->interactive && !is_interacting) {// if we're in interactive mode, ctrl-c may be used to get the@@ -47,7 +41,7 @@ static void sigint_handler(int signo) {// 2. if we're using a purely generative mode then ctrl-c can// be used to gracefully halt the text generation process.is_terminated = true;- acknowledge_shutdown();+ ggml_interrupt(true);}}@@ -95,17 +89,6 @@ static void write_logfile(fclose(logfile);}-static int Eval(struct llama_context * ctx, struct llama_batch batch) {- int rc;- sigset_t ss, oldss;- sigemptyset(&ss);- sigaddset(&ss, SIGINT);- sigprocmask(SIG_BLOCK, &ss, &oldss);- rc = llama_decode(ctx, batch);- sigprocmask(SIG_SETMASK, &oldss, 0);- return rc;-}-static bool has_argument(int argc, char ** argv, const char * arg) {for (int i = 1; i < argc; ++i) {if (!strcmp(argv[i], arg)) {@@ -609,7 +592,7 @@ int main(int argc, char ** argv) {for (int i = 0; i < input_size; i += params.n_batch) {int n_eval = std::min(input_size - i, params.n_batch);- if (Eval(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) {+ if (llama_decode(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) {LOG_TEE("%s : failed to eval\n", __func__);return 1;}@@ -626,7 +609,7 @@ int main(int argc, char ** argv) {LOG("eval: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, embd).c_str());- if (Eval(ctx, llama_batch_get_one(&embd[i], n_eval, n_past, 0))) {+ if (llama_decode(ctx, llama_batch_get_one(&embd[i], n_eval, n_past, 0))) {LOG_TEE("%s : failed to eval\n", __func__);return 1;}