Skip to content

Distribute and run LLMs with a single file.

Make ctrl-c work better

Justine Tunneybaf216ad1e97
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafile
index ee591c74..9510b544 100644
--- a/llama.cpp/README.llamafile
+++ b/llama.cpp/README.llamafile
@@ -31,6 +31,7 @@ LOCAL MODIFICATIONS
- Add support to main() programs for Cosmo /zip/.args files
- Use condition variables for server rather than busy loop
- Introduce pledge() SECCOMP sandboxing to improve security
+ - Introduce ggml_interrupt() to gracefully handle signals
- Remove log callback pointer API from Metal GPU module
- Use runtime dispatching for matmul quants
- Remove operating system #ifdef statements
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 187641d4..9480a5bc 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -103,6 +103,12 @@ SOFTWARE.\"");
// end of logging block
//
+static volatile bool is_interrupted;
+
+void ggml_interrupt(bool state) {
+ is_interrupted = state;
+}
+
static void * ggml_aligned_malloc(size_t size) {
if (size == 0) {
GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");
@@ -15955,7 +15961,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
int node_n = -1;
- while (true) {
+ while (!is_interrupted) {
if (cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) {
state->shared->node_n += 1;
return (thread_ret_t) GGML_EXIT_ABORTED;
@@ -16025,7 +16031,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
} else {
// wait for other threads to finish
const int last = node_n;
- while (true) {
+ while (!is_interrupted) {
// TODO: this sched_yield can have significant impact on the performance - either positive or negative
// depending on the workload and the operating system.
// since it is not clear what is the best approach, it should potentially become user-configurable
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index bdd8a398..db102bc8 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -2113,6 +2113,8 @@ extern "C" {
GGML_API int ggml_cpu_has_ssse3 (void);
GGML_API int ggml_cpu_has_vsx (void);
+ GGML_API void ggml_interrupt(bool);
+
//
// Internal types and functions exposed for tests and benchmarks
//
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 86087441..e66ee094 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -30,12 +30,6 @@ static gpt_params * g_params;
static volatile bool is_terminated;
static volatile bool is_interacting;
-static void acknowledge_shutdown(void) {
- if (is_terminal) {
- write(2, "^C", 2);
- }
-}
-
static void sigint_handler(int signo) {
if (g_params->interactive && !is_interacting) {
// if we're in interactive mode, ctrl-c may be used to get the
@@ -47,7 +41,7 @@ static void sigint_handler(int signo) {
// 2. if we're using a purely generative mode then ctrl-c can
// be used to gracefully halt the text generation process.
is_terminated = true;
- acknowledge_shutdown();
+ ggml_interrupt(true);
}
}
@@ -95,17 +89,6 @@ static void write_logfile(
fclose(logfile);
}
-static int Eval(struct llama_context * ctx, struct llama_batch batch) {
- int rc;
- sigset_t ss, oldss;
- sigemptyset(&ss);
- sigaddset(&ss, SIGINT);
- sigprocmask(SIG_BLOCK, &ss, &oldss);
- rc = llama_decode(ctx, batch);
- sigprocmask(SIG_SETMASK, &oldss, 0);
- return rc;
-}
-
static bool has_argument(int argc, char ** argv, const char * arg) {
for (int i = 1; i < argc; ++i) {
if (!strcmp(argv[i], arg)) {
@@ -609,7 +592,7 @@ int main(int argc, char ** argv) {
for (int i = 0; i < input_size; i += params.n_batch) {
int n_eval = std::min(input_size - i, params.n_batch);
- if (Eval(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) {
+ if (llama_decode(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) {
LOG_TEE("%s : failed to eval\n", __func__);
return 1;
}
@@ -626,7 +609,7 @@ int main(int argc, char ** argv) {
LOG("eval: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, embd).c_str());
- if (Eval(ctx, llama_batch_get_one(&embd[i], n_eval, n_past, 0))) {
+ if (llama_decode(ctx, llama_batch_get_one(&embd[i], n_eval, n_past, 0))) {
LOG_TEE("%s : failed to eval\n", __func__);
return 1;
}