Skip to content

Distribute and run LLMs with a single file.

Fix sync bug in new task pool

Justine Tunney835985b42021
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 91bd9270..736d055d 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -439,7 +439,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
.vec_dot = ggml_vec_dot_q8_0_q8_0,
.vec_dot_type = GGML_TYPE_Q8_0,
#if defined (__ARM_FEATURE_MATMUL_INT8)
- .nrows = 1,
+ .nrows = 2,
#else
.nrows = 1,
#endif
diff --git a/llamafile/pool.cpp b/llamafile/pool.cpp
index a2141cf5..e0c112bf 100644
--- a/llamafile/pool.cpp
+++ b/llamafile/pool.cpp
@@ -31,20 +31,20 @@ static void llamafile_thread_canceled(llamafile_thread *);
static ThreadLocal<llamafile_thread> g_key(llamafile_thread_canceled);
struct llamafile_task {
+ _Atomic(pthread_t) th = -1;
pthread_cond_t cv = PTHREAD_COND_INITIALIZER;
pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
void *(*func)(void *);
void *arg;
void *res;
- pthread_t th = -1;
};
struct llamafile_thread {
+ _Atomic(pthread_t) th = -1;
pthread_cond_t cv = PTHREAD_COND_INITIALIZER;
pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
llamafile_task *task;
llamafile_thread *next;
- pthread_t th;
};
static atomic_int g_active;
@@ -103,6 +103,13 @@ static void *llamafile_thread_worker(void *arg) {
void *res = thread->task->func(thread->task->arg);
pthread_setcancelstate(PTHREAD_CANCEL_MASKED, 0);
+ for (;;) {
+ if (thread->th != -1)
+ if (thread->task->th != -1)
+ break;
+ pthread_pause_np();
+ }
+
pthread_mutex_lock(&thread->task->mu);
thread->task->res = res;
thread->task->th = 0;
@@ -141,10 +148,10 @@ static errno_t llamafile_thread_create(llamafile_task *task) {
pthread_attr_setguardsize(&attr, sysconf(_SC_PAGESIZE));
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
pthread_attr_setsigaltstacksize_np(&attr, sysconf(_SC_MINSIGSTKSZ) + 32768);
- errno_t err = pthread_create(&thread->th, &attr, llamafile_thread_worker, thread);
+ errno_t err = pthread_create((pthread_t *)&thread->th, &attr, llamafile_thread_worker, thread);
pthread_attr_destroy(&attr);
if (!err) {
- task->th = thread->th;
+ task->th = thread->th.load();
} else {
delete thread;
}
@@ -160,7 +167,7 @@ errno_t llamafile_task_create(llamafile_task **out_task, void *(*func)(void *),
if ((thread = idle_pop())) {
pthread_mutex_lock(&thread->mu);
thread->task = task;
- task->th = thread->th;
+ task->th = thread->th.load();
pthread_cond_signal(&thread->cv);
pthread_mutex_unlock(&thread->mu);
err = 0;
@@ -195,13 +202,17 @@ errno_t llamafile_task_cancel(llamafile_task *task) {
}
void llamafile_task_shutdown(void) {
- llamafile_thread *thread;
- while ((thread = idle_pop()))
- if (thread->th)
- pthread_cancel(thread->th);
+ pthread_t th;
int backoff = 0;
- while (g_active)
+ llamafile_thread *thread;
+ for (;;) {
+ while ((thread = idle_pop()))
+ if ((th = thread->th))
+ pthread_cancel(th);
+ if (!g_active)
+ break;
backoff = pthread_delay_np(&g_idle, backoff);
+ }
}
static struct llamafile_tasks {