Skip to content

Distribute and run LLMs with a single file.

Count number of cores correctly on Alderlake CPUs

Justine Tunney2ba0afd0dba2
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafile
index 6321af81..d5d52099 100644
--- a/llama.cpp/README.llamafile
+++ b/llama.cpp/README.llamafile
@@ -14,6 +14,7 @@ ORIGIN
LOCAL MODIFICATIONS
+ - Count the number of cores correctly on Intel's Alderlake architecture
- Remove MAP_POPULATE because it makes mmap(tinyllama) block for 100ms
- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs
- Unify main, server, and llava-cli into single llamafile program
diff --git a/llama.cpp/common.h b/llama.cpp/common.h
index c9902dc5..7dd5903b 100644
--- a/llama.cpp/common.h
+++ b/llama.cpp/common.h
@@ -9,6 +9,7 @@
#include "sampling.h"
#include "llamafile/version.h"
+#include "llamafile/llamafile.h"
#define LOG_NO_FILE_LINE_FUNCTION
#include "log.h"
@@ -51,7 +52,7 @@ int32_t get_num_physical_cores();
struct gpt_params {
uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed
- int32_t n_threads = get_num_physical_cores();
+ int32_t n_threads = llamafile_get_math_cpu_count();
int32_t n_threads_draft = -1;
int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)
int32_t n_threads_batch_draft = -1;
diff --git a/llamafile/alderlake.c b/llamafile/alderlake.c
new file mode 100644
index 00000000..b7e8eb32
--- /dev/null
+++ b/llamafile/alderlake.c
@@ -0,0 +1,84 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cosmo.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <sched.h>
+#include <stdio.h>
+
+#ifdef __x86_64__
+
+static errno_t pin_cpu(int cpu) {
+ cpu_set_t mask;
+ CPU_ZERO(&mask);
+ CPU_SET(cpu, &mask);
+ return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);
+}
+
+static bool is_hybrid_cpu(void) {
+ int abcd[4];
+ __cpuidex(abcd, 7, 0);
+ return !!(abcd[3] & (1u << 15));
+}
+
+static bool is_running_on_efficiency_core(void) {
+ int abcd[4];
+ __cpuidex(abcd, 0x1a, 0);
+ int intel_atom = 0x20;
+ int core_type = (abcd[0] & 0xff000000u) >> 24;
+ return core_type == intel_atom;
+}
+
+static int count_math_cpus(int cpu_count) {
+ int result = 0;
+ for (int cpu = 0; cpu < cpu_count; ++cpu) {
+ if (pin_cpu(cpu))
+ return cpu_count / 2; // xnu and openbsd don't support affinity
+ if (is_running_on_efficiency_core())
+ continue; // efficiency cores harm lockstep threading
+ ++cpu; // hyperthreading isn't useful for linear algebra
+ ++result;
+ }
+ return result;
+}
+
+static void *count_math_cpus_worker(void *arg) {
+ return (void *)(intptr_t)count_math_cpus((intptr_t)arg);
+}
+
+#endif // __x86_64__
+
+/**
+ * Returns number of CPUs on system that are useful for math.
+ */
+int llamafile_get_math_cpu_count(void) {
+ int cpu_count = __get_cpu_count();
+ if (cpu_count < 1)
+ return 1;
+#ifdef __x86_64__
+ if (is_hybrid_cpu()) {
+ pthread_t th; // some OSes don't support getaffinity
+ if (!pthread_create(&th, 0, count_math_cpus_worker, (void *)(intptr_t)cpu_count)) {
+ void *result;
+ if (!pthread_join(th, &result))
+ return (intptr_t)result;
+ }
+ }
+#endif
+ return cpu_count / 2;
+}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 7bf4aab3..58d97624 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -28,6 +28,7 @@ int llamafile_is_file_newer_than(const char *, const char *);
void llamafile_schlep(const void *, size_t);
void llamafile_get_app_dir(char *, size_t);
void llamafile_launch_browser(const char *);
+int llamafile_get_math_cpu_count(void);
extern bool FLAG_unsecure;