Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Use semaphore to limit GGML worker threads
Justine Tunney15d7fba1e143
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 5022a565..c157fcb0 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -245,6 +245,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paif (params.n_threads <= 0) {params.n_threads = std::thread::hardware_concurrency();}+ FLAG_threads = params.n_threads; // [jart]return true;}if (arg == "-tb" || arg == "--threads-batch") {diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex e58714f3..ec588647 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -59,6 +59,7 @@#include <ctl/vector.h>#include "llamafile/threadlocal.h"+#include "llamafile/core_manager.h"#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((__format__(__gnu_printf__, __VA_ARGS__)))@@ -11167,6 +11168,15 @@ static void llama_graph_compute(llama_context & lctx,ggml_cgraph * gf,int n_threads) {++ // [jart] resource management+ n_threads = g_core_manager.acquire(1, n_threads);+ static ThreadLocal<void> cleanup(+ [](void* n_threads) {+ g_core_manager.release((intptr_t)n_threads);+ });+ cleanup.set((void *)(intptr_t)n_threads);+// #ifdef GGML_USE_METALif (ggml_backend_is_metal(lctx.backend_metal)) {ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads);@@ -11180,6 +11190,10 @@ static void llama_graph_compute(ggml_backend_sched_graph_compute_async(lctx.sched, gf);+ // [jart] resources management+ cleanup.set(nullptr);+ g_core_manager.release((intptr_t)n_threads);+// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));}diff --git a/llamafile/core_manager.cpp b/llamafile/core_manager.cppnew file mode 100644index 00000000..cb169426--- /dev/null+++ b/llamafile/core_manager.cpp@@ -0,0 +1,78 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "core_manager.h"++#include <assert.h>++int cpu_get_num_math();++CoreManager g_core_manager;++CoreManager::CoreManager()+ : used_(0),+ total_(cpu_get_num_math()),+ cv_(PTHREAD_COND_INITIALIZER),+ mu_(PTHREAD_MUTEX_INITIALIZER) {+}++int CoreManager::acquire(int need, int greed) {+ unassert(need >= 1);+ unassert(greed >= need);++ int got = 0;++ while (got < need) {+ pthread_mutex_lock(&mu_);+ if (used_ < total_) {+ ++got;+ ++used_;+ } else {+ pthread_cond_wait(&cv_, &mu_);+ }+ pthread_mutex_unlock(&mu_);+ }++ while (got < greed) {+ errno_t err;+ if (pthread_mutex_trylock(&mu_))+ break;+ if (used_ < total_) {+ ++got;+ ++used_;+ } else {+ greed = got;+ }+ pthread_mutex_unlock(&mu_);+ }++ return got;+}++void CoreManager::release(int count) {+ bool ok;+ pthread_mutex_lock(&mu_);+ if ((used_ -= count) >= 0) {+ ok = true;+ } else {+ ok = false;+ used_ = 0;+ }+ pthread_cond_signal(&cv_);+ pthread_mutex_unlock(&mu_);+ unassert(ok);+}diff --git a/llamafile/core_manager.h b/llamafile/core_manager.hnew file mode 100644index 00000000..2d7bf95a--- /dev/null+++ b/llamafile/core_manager.h@@ -0,0 +1,34 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <pthread.h>++class CoreManager {+ public:+ CoreManager();+ int acquire(int, int);+ void release(int);++ private:+ int used_;+ int total_;+ pthread_cond_t cv_;+ pthread_mutex_t mu_;+};++extern CoreManager g_core_manager;