Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Count number of cores correctly on Alderlake CPUs
Justine Tunney2ba0afd0dba2
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex 6321af81..d5d52099 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -14,6 +14,7 @@ ORIGINLOCAL MODIFICATIONS+ - Count the number of cores correctly on Intel's Alderlake architecture- Remove MAP_POPULATE because it makes mmap(tinyllama) block for 100ms- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs- Unify main, server, and llava-cli into single llamafile programdiff --git a/llama.cpp/common.h b/llama.cpp/common.hindex c9902dc5..7dd5903b 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -9,6 +9,7 @@#include "sampling.h"#include "llamafile/version.h"+#include "llamafile/llamafile.h"#define LOG_NO_FILE_LINE_FUNCTION#include "log.h"@@ -51,7 +52,7 @@ int32_t get_num_physical_cores();struct gpt_params {uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed- int32_t n_threads = get_num_physical_cores();+ int32_t n_threads = llamafile_get_math_cpu_count();int32_t n_threads_draft = -1;int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)int32_t n_threads_batch_draft = -1;diff --git a/llamafile/alderlake.c b/llamafile/alderlake.cnew file mode 100644index 00000000..b7e8eb32--- /dev/null+++ b/llamafile/alderlake.c@@ -0,0 +1,84 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <cosmo.h>+#include <cpuid.h>+#include <pthread.h>+#include <sched.h>+#include <stdio.h>++#ifdef __x86_64__++static errno_t pin_cpu(int cpu) {+ cpu_set_t mask;+ CPU_ZERO(&mask);+ CPU_SET(cpu, &mask);+ return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);+}++static bool is_hybrid_cpu(void) {+ int abcd[4];+ __cpuidex(abcd, 7, 0);+ return !!(abcd[3] & (1u << 15));+}++static bool is_running_on_efficiency_core(void) {+ int abcd[4];+ __cpuidex(abcd, 0x1a, 0);+ int intel_atom = 0x20;+ int core_type = (abcd[0] & 0xff000000u) >> 24;+ return core_type == intel_atom;+}++static int count_math_cpus(int cpu_count) {+ int result = 0;+ for (int cpu = 0; cpu < cpu_count; ++cpu) {+ if (pin_cpu(cpu))+ return cpu_count / 2; // xnu and openbsd don't support affinity+ if (is_running_on_efficiency_core())+ continue; // efficiency cores harm lockstep threading+ ++cpu; // hyperthreading isn't useful for linear algebra+ ++result;+ }+ return result;+}++static void *count_math_cpus_worker(void *arg) {+ return (void *)(intptr_t)count_math_cpus((intptr_t)arg);+}++#endif // __x86_64__++/**+ * Returns number of CPUs on system that are useful for math.+ */+int llamafile_get_math_cpu_count(void) {+ int cpu_count = __get_cpu_count();+ if (cpu_count < 1)+ return 1;+#ifdef __x86_64__+ if (is_hybrid_cpu()) {+ pthread_t th; // some OSes don't support getaffinity+ if (!pthread_create(&th, 0, count_math_cpus_worker, (void *)(intptr_t)cpu_count)) {+ void *result;+ if (!pthread_join(th, &result))+ return (intptr_t)result;+ }+ }+#endif+ return cpu_count / 2;+}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 7bf4aab3..58d97624 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -28,6 +28,7 @@ int llamafile_is_file_newer_than(const char *, const char *);void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);void llamafile_launch_browser(const char *);+int llamafile_get_math_cpu_count(void);extern bool FLAG_unsecure;