Skip to content

Distribute and run LLMs with a single file.

Improve the chatbot

Justine Tunneyd484f6e3958a
diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cpp
index 1b70f4c6..07757109 100644
--- a/llama.cpp/llama-bench/llama-bench.cpp
+++ b/llama.cpp/llama-bench/llama-bench.cpp
@@ -22,7 +22,6 @@
#include <cosmo.h>
#include <libgen.h>
#include <sys/stat.h>
-#include <sys/auxv.h>
#include <libc/intrin/x86.h>
#include "llama.cpp/cores.h"
#include <libc/sysv/consts/hwcap.h>
@@ -33,6 +32,7 @@
#include "llama.cpp/common.h"
#include "llama.cpp/ggml-cuda.h"
#include "llamafile/llamafile.h"
+#include "llamafile/compute.h"
// utils
static uint64_t get_time_ns() {
@@ -93,92 +93,6 @@ static T stdev(const std::vector<T> & v) {
return stdev;
}
-#ifdef __x86_64__
-static void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {
- asm("movq\t%%rbx,%%rsi\n\t"
- "cpuid\n\t"
- "xchgq\t%%rbx,%%rsi"
- : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])
- : "0"(leaf), "2"(subleaf));
-}
-#endif // __x86_64__
-
-static std::string get_cpu_info() { // [jart]
- std::string id;
-
-#ifdef __x86_64__
- union { // [jart]
- char str[64];
- unsigned reg[16];
- } u = {0};
- cpuid(0x80000002, 0, u.reg + 0*4);
- cpuid(0x80000003, 0, u.reg + 1*4);
- cpuid(0x80000004, 0, u.reg + 2*4);
- int len = strlen(u.str);
- while (len > 0 && u.str[len - 1] == ' ')
- u.str[--len] = 0;
- id = u.str;
-#else
- if (IsLinux()) {
- FILE * f = fopen("/proc/cpuinfo", "r");
- if (f) {
- char buf[1024];
- while (fgets(buf, sizeof(buf), f)) {
- if (!strncmp(buf, "model name", 10) ||
- startswith(buf, "Model\t\t:")) { // e.g. raspi
- char * p = strchr(buf, ':');
- if (p) {
- p++;
- while (std::isspace(*p)) {
- p++;
- }
- while (std::isspace(p[strlen(p) - 1])) {
- p[strlen(p) - 1] = '\0';
- }
- id = p;
- break;
- }
- }
- }
- fclose(f);
- }
- }
- if (IsXnu()) {
- char cpu_name[128] = {0};
- size_t size = sizeof(cpu_name);
- if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {
- id = cpu_name;
- }
- }
-#endif
- id = replace_all(id, " 96-Cores", "");
- id = replace_all(id, "(TM)", "");
- id = replace_all(id, "(R)", "");
-
- std::string march;
-#ifdef __x86_64__
- if (__cpu_march(__cpu_model.__cpu_subtype))
- march = __cpu_march(__cpu_model.__cpu_subtype);
-#else
- long hwcap = getauxval(AT_HWCAP);
- if (hwcap & HWCAP_ASIMDHP)
- march += "+fp16";
- if (hwcap & HWCAP_ASIMDDP)
- march += "+dotprod";
-#endif
-
- if (!march.empty()) {
- bool empty = id.empty();
- if (!empty)
- id += " (";
- id += march;
- if (!empty)
- id += ")";
- }
-
- return id;
-}
-
static std::string get_gpu_info() {
std::string id;
#ifdef GGML_USE_CUDA
@@ -984,7 +898,7 @@ const bool test::metal = false; // !!ggml_cpu_has_metal(); // [jar
const bool test::gpu_blas = false; // !!ggml_cpu_has_gpublas(); // [jart]
const bool test::blas = false; // !!ggml_cpu_has_blas(); // [jart]
const bool test::sycl = false; // !!ggml_cpu_has_sycl(); // [jart]
-const std::string test::cpu_info = get_cpu_info();
+const std::string test::cpu_info = llamafile_describe_cpu();
const std::string test::gpu_info = ""; //get_gpu_info(); // [jart]
struct printer {
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 8872e1b6..7b870c7c 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -17256,6 +17256,10 @@ int32_t llama_n_layer(const struct llama_model * model) {
return model->hparams.n_layer;
}
+int32_t llama_n_gpu_layers(const struct llama_model * model) { // [jart]
+ return model->n_gpu_layers;
+}
+
float llama_rope_freq_scale_train(const struct llama_model * model) {
return model->hparams.rope_freq_scale_train;
}
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 562e7fe2..181fca09 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -473,6 +473,8 @@ extern "C" {
LLAMA_API int32_t llama_n_embd (const struct llama_model * model);
LLAMA_API int32_t llama_n_layer (const struct llama_model * model);
+ LLAMA_API int32_t llama_n_gpu_layers(const struct llama_model * model); // [jart]
+
// Get the model's RoPE frequency scaling factor
LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);
diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cpp
index 984524f5..526e8437 100644
--- a/llamafile/chatbot.cpp
+++ b/llamafile/chatbot.cpp
@@ -26,32 +26,47 @@
#include <vector>
#include "llama.cpp/common.h"
+#include "llama.cpp/ggml-cuda.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/server/server.h"
#include "llamafile/bestline.h"
+#include "llamafile/compute.h"
#include "llamafile/highlight.h"
#include "llamafile/llamafile.h"
+#define RESET "\e[0m"
#define BOLD "\e[1m"
#define FAINT "\e[2m"
#define UNBOLD "\e[22m"
#define RED "\e[31m"
#define GREEN "\e[32m"
#define MAGENTA "\e[35m"
+#define YELLOW "\e[33m"
+#define CYAN "\e[36m"
#define UNFOREGROUND "\e[39m"
#define BRIGHT_BLACK "\e[90m"
#define BRIGHT_RED "\e[91m"
#define BRIGHT_GREEN "\e[92m"
#define CLEAR_FORWARD "\e[K"
+enum Role {
+ ROLE_USER,
+ ROLE_ASSISTANT,
+ ROLE_SYSTEM,
+};
+
struct ServerArgs {
int argc;
char **argv;
};
static int n_past;
+static bool g_once;
+static bool g_manual;
static llama_model *g_model;
static llama_context *g_ctx;
+static std::vector<llama_token> g_history;
+static enum Role g_role = ROLE_USER;
static pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER;
static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;
static std::string g_listen_url;
@@ -69,6 +84,58 @@ static bool is_empty(const char *s) {
return true;
}
+static const char *get_role_name(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return "user";
+ case ROLE_ASSISTANT:
+ return "assistant";
+ case ROLE_SYSTEM:
+ return "system";
+ default:
+ __builtin_unreachable();
+ }
+}
+
+static const char *get_role_color(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return GREEN;
+ case ROLE_ASSISTANT:
+ return MAGENTA;
+ case ROLE_SYSTEM:
+ return YELLOW;
+ default:
+ __builtin_unreachable();
+ }
+}
+
+static enum Role get_next_role(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return ROLE_ASSISTANT;
+ case ROLE_ASSISTANT:
+ return ROLE_USER;
+ case ROLE_SYSTEM:
+ return ROLE_USER;
+ default:
+ __builtin_unreachable();
+ }
+}
+
+static enum Role cycle_role(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return ROLE_ASSISTANT;
+ case ROLE_ASSISTANT:
+ return ROLE_SYSTEM;
+ case ROLE_SYSTEM:
+ return ROLE_USER;
+ default:
+ __builtin_unreachable();
+ }
+}
+
static std::string basename(const std::string_view path) {
size_t i, e;
if ((e = path.size())) {
@@ -98,11 +165,167 @@ __attribute__((format(printf, 1, 2))) static std::string format(const char *fmt,
return res;
}
+static std::string join(const std::vector<std::string> &vec, const std::string_view &delim) {
+ std::string result;
+ for (size_t i = 0; i < vec.size(); i++) {
+ result += vec[i];
+ if (i < vec.size() - 1)
+ result += delim;
+ }
+ return result;
+}
+
+static std::string describe_compute(void) {
+ if (llama_n_gpu_layers(g_model) > 0 && llamafile_has_gpu()) {
+ if (llamafile_has_metal()) {
+ return "Apple Metal GPU";
+ } else {
+ std::vector<std::string> vec;
+ int count = ggml_backend_cuda_get_device_count();
+ for (int i = 0; i < count; i++) {
+ char buf[128];
+ ggml_backend_cuda_get_device_description(i, buf, sizeof(buf));
+ vec.emplace_back(buf);
+ }
+ return join(vec, ", ");
+ }
+ } else {
+ return llamafile_describe_cpu();
+ }
+}
+
+static void on_help(const std::vector<std::string> &args) {
+ if (args.size() == 1) {
+ fprintf(stderr, "\
+" BOLD "available commands" RESET "\n\
+ /context\n\
+ /dump [FILE]\n\
+ /exit\n\
+ /help [COMMAND]\n\
+ /manual [on|off]\n\
+ /stats\n\
+");
+ } else if (args[1] == "context") {
+ fprintf(stderr, "\
+usage: /context" RESET "\n\
+prints information about context window usage. this helps you know how\n\
+soon you're going to run out of tokens for the current conversation.\n\
+");
+ } else if (args[1] == "dump") {
+ fprintf(stderr, "\
+" BOLD "usage: /dump [FILE]" RESET "\n\
+dumps raw tokens for current conversation history. special tokens are\n\
+printed in the a model specific chat syntax. this is useful for seeing\n\
+specifically what data is being evaluated by the model. by default it\n\
+will be printed to the terminal. if a FILE argument is specified, then\n\
+the raw conversation history will be written to that filename.\n\
+");
+ } else if (args[1] == "exit") {
+ fprintf(stderr, "\
+" BOLD "usage: /exit" RESET "\n\
+this command will cause the process to exit. it is essentially the same\n\
+as typing ctrl-d which signals an eof condition. it also does the same\n\
+thing as typing ctrl-c when the >>> user input prompt is displayed.\n\
+");
+ } else if (args[1] == "manual") {
+ fprintf(stderr, "\
+" BOLD "usage: /manual [on|off]" RESET "\n\
+puts the chatbot in manual mode. this is useful if you want to inject\n\
+a response as the model rather than the user. it's also possible to add\n\
+additional system prompts to the conversation history. when the manual\n\
+mode is activated, a hint is displayed next to the '>>>' indicating\n\
+the current role, which can be 'user', 'assistant', or 'system'. if\n\
+enter is pressed on an empty line, then llamafile will cycle between\n\
+all three roles. when /manual is specified without an argument, it will\n\
+toggle manual mode. otherwise an 'on' or 'off' argument is supplied.\n\
+");
+ } else if (args[1] == "help") {
+ fprintf(stderr, "\
+" BOLD "usage: /help [COMMAND]" RESET "\n\
+shows help on how to issue commands to your llamafile. if no argument is\n\
+specified, then a synopsis of all available commands will be printed. if\n\
+a specific command name is given (e.g. /help dump) then documentation on\n\
+the usage of that specific command will be printed.\n\
+");
+ } else if (args[1] == "stats") {
+ fprintf(stderr, "\
+" BOLD "usage: /stats" RESET "\n\
+prints performance statistics for current session. this includes prompt\n\
+evaluation time in tokens per second, which indicates prefill speed, or\n\
+how quickly llamafile is able to read text. the 'eval time' statistic\n\
+gives you prediction or token generation speed, in tokens per second,\n\
+which tells you how quickly llamafile is able to write text.\n\
+");
+ } else {
+ fprintf(stderr, BRIGHT_RED "%s: unknown command" RESET "\n", args[1].c_str());
+ }
+}
+
+static void on_manual(const std::vector<std::string> &args) {
+ if (args.size() == 1) {
+ g_manual = !g_manual;
+ } else if (args.size() == 2 && (args[1] == "on" || args[1] == "off")) {
+ g_manual = args[1] == "on";
+ } else {
+ fprintf(stderr, BRIGHT_RED "error: bad /manual command" RESET "\n"
+ "usage: /manual [on|off]\n");
+ }
+ fprintf(stderr, FAINT "manual mode %s" RESET "\n", g_manual ? "enabled" : "disabled");
+ if (!g_manual)
+ g_role = ROLE_USER;
+}
+
+static void on_context(const std::vector<std::string> &args) {
+ int configured_context = llama_n_ctx(g_ctx);
+ int max_context = llama_n_ctx_train(g_model);
+ printf("%d out of %d context tokens used (%d tokens remaining)\n", n_past, configured_context,
+ configured_context - n_past);
+ if (configured_context < max_context)
+ printf("use the `-c %d` flag at startup for maximum context\n", max_context);
+}
+
+static void on_stats(const std::vector<std::string> &args) {
+ FLAG_log_disable = false;
+ llama_print_timings(g_ctx);
+ FLAG_log_disable = true;
+}
+
+static void on_dump(const std::vector<std::string> &args) {
+ int fd = 1;
+ if (args.size() >= 2) {
+ if ((fd = creat(args[1].c_str(), 0644)) == -1) {
+ perror(args[1].c_str());
+ return;
+ }
+ }
+ std::string s;
+ for (auto id : g_history)
+ s += llama_token_to_piece(g_ctx, id, true);
+ if (!s.empty() && s[s.size() - 1] != '\n')
+ s += '\n';
+ write(fd, s.data(), s.size());
+ if (args.size() >= 2)
+ close(fd);
+}
+
+static char *on_hint(const char *line, const char **ansi1, const char **ansi2) {
+ *ansi1 = FAINT;
+ *ansi2 = UNBOLD;
+ if (!*line && g_manual)
+ return strdup(get_role_name(g_role));
+ else if (!*line && !g_manual && !g_once)
+ return strdup("say something (or type /help for help)");
+ return strdup("");
+}
+
static void on_completion(const char *line, int pos, bestlineCompletions *comp) {
static const char *const kCompletions[] = {
- "/context", //
- "/exit", //
- "/stats", //
+ "/context", // usage: /context
+ "/dump", // usage: /dump [FILE]
+ "/exit", // usage: /exit
+ "/help", // usage: /help [COMMAND]
+ "/manual", // usage: /manual [on|off]
+ "/stats", // usage: /stats
};
for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)
if (startswith(kCompletions[i], line))
@@ -118,21 +341,20 @@ static bool handle_command(const char *command) {
std::string arg;
while (iss >> arg)
args.push_back(arg);
- if (args[0] == "exit") {
+ if (args[0] == "exit" || args[0] == "bye") {
exit(0);
+ } else if (args[0] == "help" || args[0] == "?") {
+ on_help(args);
} else if (args[0] == "stats") {
- FLAG_log_disable = false;
- llama_print_timings(g_ctx);
- FLAG_log_disable = true;
+ on_stats(args);
} else if (args[0] == "context") {
- int configured_context = llama_n_ctx(g_ctx);
- int max_context = llama_n_ctx_train(g_model);
- printf("%d out of %d context tokens used (%d tokens remaining)\n", n_past,
- configured_context, configured_context - n_past);
- if (configured_context < max_context)
- printf("use the `-c %d` flag at startup for maximum context\n", max_context);
+ on_context(args);
+ } else if (args[0] == "manual") {
+ on_manual(args);
+ } else if (args[0] == "dump") {
+ on_dump(args);
} else {
- printf("%s: unrecognized command\n", args[0].c_str());
+ fprintf(stderr, BRIGHT_RED "%s: unrecognized command" RESET "\n", args[0].c_str());
}
return true;
}
@@ -190,6 +412,7 @@ static void eval_tokens(std::vector<llama_token> tokens, int n_batch) {
n_eval = n_batch;
if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, n_past, 0)))
die_out_of_context();
+ g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);
n_past += n_eval;
}
}
@@ -276,6 +499,7 @@ int chatbot_main(int argc, char **argv) {
printf(BOLD "software" UNBOLD ": llamafile " LLAMAFILE_VERSION_STRING "\n" //
BOLD "model" UNBOLD ": %s\n",
basename(params.model).c_str());
+ printf(BOLD "compute" UNBOLD ": %s\n", describe_compute().c_str());
if (want_server)
printf(BOLD "server" UNBOLD ": %s\n", g_listen_url.c_str());
printf("\n");
@@ -311,8 +535,10 @@ int chatbot_main(int argc, char **argv) {
// run chatbot
for (;;) {
bestlineLlamaMode(true);
+ bestlineSetHintsCallback(on_hint);
+ bestlineSetFreeHintsCallback(free);
bestlineSetCompletionCallback(on_completion);
- write(1, GREEN, strlen(GREEN));
+ write(1, get_role_color(g_role), strlen(get_role_color(g_role)));
char *line = bestlineWithHistory(">>> ", "llamafile");
write(1, UNFOREGROUND, strlen(UNFOREGROUND));
if (!line) {
@@ -321,6 +547,10 @@ int chatbot_main(int argc, char **argv) {
break;
}
if (is_empty(line)) {
+ if (g_manual) {
+ g_role = cycle_role(g_role);
+ write(1, "\033[F", 3);
+ }
free(line);
continue;
}
@@ -328,9 +558,16 @@ int chatbot_main(int argc, char **argv) {
free(line);
continue;
}
- std::vector<llama_chat_msg> chat = {{"user", line}};
- std::string msg = llama_chat_apply_template(g_model, params.chat_template, chat, true);
+ g_once = true;
+ bool add_assi = !g_manual;
+ std::vector<llama_chat_msg> chat = {{get_role_name(g_role), line}};
+ std::string msg = llama_chat_apply_template(g_model, params.chat_template, chat, add_assi);
eval_string(msg, params.n_batch, false, true);
+ if (g_manual) {
+ g_role = get_next_role(g_role);
+ free(line);
+ continue;
+ }
while (!g_got_sigint) {
llama_token id = llama_sampling_sample(sampler, g_ctx, NULL);
llama_sampling_accept(sampler, g_ctx, id, true);
diff --git a/llamafile/compute.cpp b/llamafile/compute.cpp
new file mode 100644
index 00000000..51b1b869
--- /dev/null
+++ b/llamafile/compute.cpp
@@ -0,0 +1,110 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "compute.h"
+
+#include <cosmo.h>
+#include <libc/intrin/x86.h>
+#include <sys/auxv.h>
+
+#include "llama.cpp/string.h"
+
+#ifdef __x86_64__
+static void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {
+ asm("movq\t%%rbx,%%rsi\n\t"
+ "cpuid\n\t"
+ "xchgq\t%%rbx,%%rsi"
+ : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])
+ : "0"(leaf), "2"(subleaf));
+}
+#endif // __x86_64__
+
+/**
+ * Returns string describing host CPU.
+ */
+std::string llamafile_describe_cpu() {
+ std::string id;
+
+#ifdef __x86_64__
+ union {
+ char str[64];
+ unsigned reg[16];
+ } u = {0};
+ cpuid(0x80000002, 0, u.reg + 0 * 4);
+ cpuid(0x80000003, 0, u.reg + 1 * 4);
+ cpuid(0x80000004, 0, u.reg + 2 * 4);
+ int len = strlen(u.str);
+ while (len > 0 && u.str[len - 1] == ' ')
+ u.str[--len] = 0;
+ id = u.str;
+#else
+ if (IsLinux()) {
+ FILE *f = fopen("/proc/cpuinfo", "r");
+ if (f) {
+ char buf[1024];
+ while (fgets(buf, sizeof(buf), f)) {
+ if (!strncmp(buf, "model name", 10) ||
+ startswith(buf, "Model\t\t:")) { // e.g. raspi
+ char *p = strchr(buf, ':');
+ if (p) {
+ p++;
+ while (std::isspace(*p))
+ p++;
+ while (std::isspace(p[strlen(p) - 1]))
+ p[strlen(p) - 1] = '\0';
+ id = p;
+ break;
+ }
+ }
+ }
+ fclose(f);
+ }
+ }
+ if (IsXnu()) {
+ char cpu_name[128] = {0};
+ size_t size = sizeof(cpu_name);
+ if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1)
+ id = cpu_name;
+ }
+#endif
+ id = replace_all(id, " 96-Cores", "");
+ id = replace_all(id, "(TM)", "");
+ id = replace_all(id, "(R)", "");
+
+ std::string march;
+#ifdef __x86_64__
+ if (__cpu_march(__cpu_model.__cpu_subtype))
+ march = __cpu_march(__cpu_model.__cpu_subtype);
+#else
+ long hwcap = getauxval(AT_HWCAP);
+ if (hwcap & HWCAP_ASIMDHP)
+ march += "+fp16";
+ if (hwcap & HWCAP_ASIMDDP)
+ march += "+dotprod";
+#endif
+
+ if (!march.empty()) {
+ bool empty = id.empty();
+ if (!empty)
+ id += " (";
+ id += march;
+ if (!empty)
+ id += ")";
+ }
+
+ return id;
+}
diff --git a/llamafile/compute.h b/llamafile/compute.h
new file mode 100644
index 00000000..a8225098
--- /dev/null
+++ b/llamafile/compute.h
@@ -0,0 +1,21 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <string>
+
+std::string llamafile_describe_cpu();
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index b62c5e21..d1f8df7e 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -115,6 +115,7 @@ static struct Cuda {
typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count;
typeof(ggml_backend_cuda_unregister_host_buffer) *GGML_CALL unreg_host_buf;
typeof(ggml_backend_cuda_register_host_buffer) *GGML_CALL register_host_buffer;
+ typeof(ggml_backend_cuda_get_device_description) *GGML_CALL get_description;
} ggml_cuda;
static const char *Dlerror(void) {
@@ -731,6 +732,7 @@ static bool link_cuda_dso(const char *dso, const char *dir) {
ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_backend_cuda_get_device_count"));
ok &= !!(ggml_cuda.unreg_host_buf = imp(lib, "ggml_backend_cuda_unregister_host_buffer"));
ok &= !!(ggml_cuda.register_host_buffer = imp(lib, "ggml_backend_cuda_register_host_buffer"));
+ ok &= !!(ggml_cuda.get_description = imp(lib, "ggml_backend_cuda_get_device_description"));
if (!ok) {
tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);
cosmo_dlclose(lib);
@@ -1030,3 +1032,10 @@ GGML_CALL bool ggml_backend_cuda_register_host_buffer(void *buffer, size_t size)
return false;
return ggml_cuda.register_host_buffer(buffer, size);
}
+
+GGML_CALL void ggml_backend_cuda_get_device_description(int device, char *description,
+ size_t description_size) {
+ if (!llamafile_has_cuda())
+ return;
+ return ggml_cuda.get_description(device, description, description_size);
+}