Skip to content

Distribute and run LLMs with a single file.

Use semaphore to limit GGML worker threads

Justine Tunney15d7fba1e143
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 5022a565..c157fcb0 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -245,6 +245,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
if (params.n_threads <= 0) {
params.n_threads = std::thread::hardware_concurrency();
}
+ FLAG_threads = params.n_threads; // [jart]
return true;
}
if (arg == "-tb" || arg == "--threads-batch") {
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index e58714f3..ec588647 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -59,6 +59,7 @@
#include <ctl/vector.h>
#include "llamafile/threadlocal.h"
+#include "llamafile/core_manager.h"
#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((__format__(__gnu_printf__, __VA_ARGS__)))
@@ -11167,6 +11168,15 @@ static void llama_graph_compute(
llama_context & lctx,
ggml_cgraph * gf,
int n_threads) {
+
+ // [jart] resource management
+ n_threads = g_core_manager.acquire(1, n_threads);
+ static ThreadLocal<void> cleanup(
+ [](void* n_threads) {
+ g_core_manager.release((intptr_t)n_threads);
+ });
+ cleanup.set((void *)(intptr_t)n_threads);
+
// #ifdef GGML_USE_METAL
if (ggml_backend_is_metal(lctx.backend_metal)) {
ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads);
@@ -11180,6 +11190,10 @@ static void llama_graph_compute(
ggml_backend_sched_graph_compute_async(lctx.sched, gf);
+ // [jart] resources management
+ cleanup.set(nullptr);
+ g_core_manager.release((intptr_t)n_threads);
+
// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));
}
diff --git a/llamafile/core_manager.cpp b/llamafile/core_manager.cpp
new file mode 100644
index 00000000..cb169426
--- /dev/null
+++ b/llamafile/core_manager.cpp
@@ -0,0 +1,78 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "core_manager.h"
+
+#include <assert.h>
+
+int cpu_get_num_math();
+
+CoreManager g_core_manager;
+
+CoreManager::CoreManager()
+ : used_(0),
+ total_(cpu_get_num_math()),
+ cv_(PTHREAD_COND_INITIALIZER),
+ mu_(PTHREAD_MUTEX_INITIALIZER) {
+}
+
+int CoreManager::acquire(int need, int greed) {
+ unassert(need >= 1);
+ unassert(greed >= need);
+
+ int got = 0;
+
+ while (got < need) {
+ pthread_mutex_lock(&mu_);
+ if (used_ < total_) {
+ ++got;
+ ++used_;
+ } else {
+ pthread_cond_wait(&cv_, &mu_);
+ }
+ pthread_mutex_unlock(&mu_);
+ }
+
+ while (got < greed) {
+ errno_t err;
+ if (pthread_mutex_trylock(&mu_))
+ break;
+ if (used_ < total_) {
+ ++got;
+ ++used_;
+ } else {
+ greed = got;
+ }
+ pthread_mutex_unlock(&mu_);
+ }
+
+ return got;
+}
+
+void CoreManager::release(int count) {
+ bool ok;
+ pthread_mutex_lock(&mu_);
+ if ((used_ -= count) >= 0) {
+ ok = true;
+ } else {
+ ok = false;
+ used_ = 0;
+ }
+ pthread_cond_signal(&cv_);
+ pthread_mutex_unlock(&mu_);
+ unassert(ok);
+}
diff --git a/llamafile/core_manager.h b/llamafile/core_manager.h
new file mode 100644
index 00000000..2d7bf95a
--- /dev/null
+++ b/llamafile/core_manager.h
@@ -0,0 +1,34 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <pthread.h>
+
+class CoreManager {
+ public:
+ CoreManager();
+ int acquire(int, int);
+ void release(int);
+
+ private:
+ int used_;
+ int total_;
+ pthread_cond_t cv_;
+ pthread_mutex_t mu_;
+};
+
+extern CoreManager g_core_manager;