Skip to content

Distribute and run LLMs with a single file.

13.4 KB
Raw Blame
#include "system.h"
#include <cstdio>
#include <sys/utsname.h>
#include <sys/sysinfo.h>
#include <cosmo.h>
#include <iostream>
#include <string>
#include <sstream>
#include "llama.cpp/string.h"

#include "cmd.h"
#include "utils.h"

#include "llama.cpp/ggml-metal.h"
#include "llama.cpp/ggml-cuda.h"
#include "llama.cpp/common.h"

#include <libc/intrin/x86.h>

#ifdef __x86_64__
void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {
    asm("movq\t%%rbx,%%rsi\n\t"
        "cpuid\n\t"
        "xchgq\t%%rbx,%%rsi"
        : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])
        : "0"(leaf), "2"(subleaf));
}

// TODO implement an arm version as well
char* get_cpu_manufacturer(void) {
    union {
        char str[13];      // 12 chars + null terminator
        unsigned reg[4];   // For the 4 registers (EAX, EBX, ECX, EDX)
    } u = {0};            // Initialize to zero

    // Get manufacturer ID with leaf 0
    cpuid(0, 0, u.reg);

    // Rearrange the registers to get the correct string
    // The manufacturer string is in EBX,EDX,ECX order
    unsigned temp = u.reg[1];        // Save EBX
    u.reg[0] = temp;                 // Move EBX to first position
    u.reg[1] = u.reg[3];             // Move EDX to second position
    u.reg[2] = u.reg[2];             // ECX stays in third position
    u.reg[3] = 0;                     // Ensure null termination

    const char* manufacturer = u.str;
    if (strcmp(manufacturer, "AuthenticAMD") == 0) {
        return strdup("AMD");
    } else if (strcmp(manufacturer, "GenuineIntel") == 0) {
        return strdup("Intel");
    }

    return strdup(manufacturer);  // Return the original string if unknown
}
#endif // __x86_64__

std::string get_cpu_info() { // [jart]
    std::string id;

#ifdef __x86_64__
    union { // [jart]
        char str[64];
        unsigned reg[16];
    } u = {0};
    cpuid(0x80000002, 0, u.reg + 0*4);
    cpuid(0x80000003, 0, u.reg + 1*4);
    cpuid(0x80000004, 0, u.reg + 2*4);
    int len = strlen(u.str);
    while (len > 0 && u.str[len - 1] == ' ')
        u.str[--len] = 0;
    id = u.str;
#else
    if (IsLinux()) {
        FILE * f = fopen("/proc/cpuinfo", "r");
        if (f) {
            char buf[1024];
            while (fgets(buf, sizeof(buf), f)) {
                if (!strncmp(buf, "model name", 10) ||
                    startswith(buf, "Model\t\t:")) { // e.g. raspi
                    char * p = strchr(buf, ':');
                    if (p) {
                        p++;
                        while (std::isspace(*p)) {
                            p++;
                        }
                        while (std::isspace(p[strlen(p) - 1])) {
                            p[strlen(p) - 1] = '\0';
                        }
                        id = p;
                        break;
                    }
                }
            }
            fclose(f);
        }
    }
    if (IsXnu()) {
        // TODO we can also do something similar to https://github.com/vladkens/macmon/blob/main/src/sources.rs#L424
        char cpu_name[128] = {0};
        size_t size = sizeof(cpu_name);
        if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {
            id = cpu_name;
        }

        // Get number of performance cores on macos
        int num_perf0_cpu;
        size = sizeof(num_perf0_cpu);
        if (sysctlbyname("hw.perflevel0.logicalcpu", &num_perf0_cpu, &size, NULL, 0) != -1) {
            id += " ";
            id += std::to_string(num_perf0_cpu);
            id += "P";
        }

        // Get number of efficiency cores on macos
        int num_perf1_cpu;
        size = sizeof(num_perf1_cpu);
        if (sysctlbyname("hw.perflevel1.logicalcpu", &num_perf1_cpu, &size, NULL, 0) != -1) {
            id += "+";
            id += std::to_string(num_perf1_cpu);
            id += "E";
        }

    }
#endif
    id = replace_all(id, " 96-Cores", "");
    id = replace_all(id, "(TM)", "");
    id = replace_all(id, "(R)", "");

    std::string march;
#ifdef __x86_64__
    if (__cpu_march(__cpu_model.__cpu_subtype))
        march = __cpu_march(__cpu_model.__cpu_subtype);
#else
    // TODO. We can do this separately as part of 'features' or something
    // long hwcap = getauxval(AT_HWCAP);
    // if (hwcap & HWCAP_ASIMDHP)
    //     march += "+fp16";
    // if (hwcap & HWCAP_ASIMDDP)
    //     march += "+dotprod";
#endif

    if (!march.empty()) {
        bool empty = id.empty();
        if (!empty)
            id += " (";
        id += march;
        if (!empty)
            id += ")";
    }

    return id;
}

void get_runtime_info(RuntimeInfo* info) {
    if (info == NULL) return;

    strncpy(info->llamafile_version, LLAMAFILE_VERSION_STRING, MAX_STRING_LENGTH - 1);
    strncpy(info->llama_commit, LLAMA_COMMIT, MAX_STRING_LENGTH - 1);

    fprintf(stderr, "%s\n", utils::color_str("\033[0;35m"));  // Sets purple color
    utils::print_centered(stderr, 70, '=', "%sLocalScore Runtime Information%s", utils::color_str("\033[1m"), utils::color_str("\033[0;35m"));
    fprintf(stderr, "\n");
    fprintf(stderr, "%-20s %s%s%s\n", "llamafile version:", utils::color_str("\033[1m"), info->llamafile_version, utils::color_str("\033[22m"));
    fprintf(stderr, "%-20s %s\n", "llama.cpp commit:", info->llama_commit);
    fprintf(stderr, "\n======================================================================\n\n%s", utils::color_str("\033[0m"));
}

double get_mem_gb() {
    struct sysinfo si;
    if (sysinfo(&si)) {
        return 0.0;
    }

    return utils::round_to_decimal(si.totalram * si.mem_unit / 1073741824.0, 1);
}

void get_sys_info(SystemInfo* info) {
    if (info == NULL) return;

    struct utsname names;
    if (uname(&names)) {
        return;
    }

    utils::sanitize_string(info->kernel_type, names.sysname, MAX_STRING_LENGTH);
    utils::sanitize_string(info->kernel_release, names.release, MAX_STRING_LENGTH);
    // TODO on darwin we might want to get from systemprofiler SPSoftwareDataType os_version
    utils::sanitize_string(info->version, names.version, MAX_STRING_LENGTH);
    utils::sanitize_string(info->system_architecture, names.machine, MAX_STRING_LENGTH);

    std::string cpu_info = get_cpu_info();
    strncpy(info->cpu, cpu_info.c_str(), MAX_STRING_LENGTH - 1);

    info->ram_gb = get_mem_gb();

    utils::print_centered(stderr, 70, '=', "%sSystem Information%s", utils::color_str("\033[1m"), utils::color_str("\033[0m"));
    fprintf(stderr, "\n");
    fprintf(stderr, "%-20s %s\n", "Kernel Type:", info->kernel_type);
    fprintf(stderr, "%-20s %s\n", "Kernel Release:", info->kernel_release);
    fprintf(stderr, "%-20s %s\n", "Version:", info->version);
    fprintf(stderr, "%-20s %s\n", "System Architecture:", info->system_architecture);
    fprintf(stderr, "%-20s %s\n", "CPU:", info->cpu);
    fprintf(stderr, "%-20s %.1f GiB\n", "RAM:", info->ram_gb);
    fprintf(stderr, "\n======================================================================\n\n");
}

void get_accelerator_info(AcceleratorInfo* info, cmd_params * params) {
    if (info == NULL) return;

    if (FLAG_gpu >= 0 && llamafile_has_gpu()) {
        if (llamafile_has_cuda()) {
            int count = ggml_backend_cuda_get_device_count();
            for (int i = 0; i < count; i++) {
                struct ggml_cuda_device_properties props;
                ggml_backend_cuda_get_device_properties(i, &props);

                if (params->verbose) {
                    printf("Raw GPU %d Memory %lld Bytes, %.2f GiB\n", i, props.totalGlobalMem, props.totalGlobalMem / 1073741824.0);
                }

                // TODO it would be much better to query NVML directly instead and similar for rocm
                double rounded_memory_gb = utils::round_to_decimal(props.totalGlobalMem / 1073741824.0, 0);

                if (i == params->main_gpu) {
                    strncpy(info->name, props.name, MAX_STRING_LENGTH - 1);

                    info->total_memory_gb = rounded_memory_gb;
                    info->core_count = props.multiProcessorCount;
                    info->capability = atof(props.compute);
                    strncpy(info->manufacturer, llamafile_has_amd_gpu() ? "AMD" : "NVIDIA", MAX_STRING_LENGTH - 1);
                }

                if (i == params->main_gpu) {
                    fprintf(stderr, "%s", utils::color_str("\033[0;32m"));  // Sets green color
                    utils::print_centered(stderr, 70, '=', "%sActive GPU (GPU %d) Information%s", utils::color_str("\033[1m"), i, utils::color_str("\033[0;32m"));
                    fprintf(stderr, "\n");
                } else {
                    fprintf(stderr, "%s", utils::color_str("\033[0;90m"));  // Sets gray color
                    utils::print_centered(stderr, 70, '=', "GPU %d Information", i);
                    fprintf(stderr, "\n");
                }

                fprintf(stderr, "%-26s %s\n", "GPU Name:", props.name);
                fprintf(stderr, "%-26s %.1f GiB\n", "VRAM:", rounded_memory_gb);
                fprintf(stderr, "%-26s %d\n", "Streaming Multiprocessors:", props.multiProcessorCount);
                fprintf(stderr, "%-26s %.1f\n", "CUDA Capability:", atof(props.compute));
                fprintf(stderr, "\n======================================================================\n\n%s", utils::color_str("\033[0m"));
            }
        }

        if (llamafile_has_metal()) {
            // TODO there is probably a cleaner way of doing this. we should only need to init once.
            // this is probably the same issue why the other thing is init multiple time too
            struct ggml_metal_device_properties props;

            std::string command = "system_profiler SPDisplaysDataType | grep \"Total Number of Cores:\" | awk '{print $5}'";
            std::string num_cores = utils::exec(command.c_str());
            props.core_count = std::stoi(num_cores);

            // Remove any trailing newline
            if (!num_cores.empty() && num_cores[num_cores.length()-1] == '\n') {
                num_cores.erase(num_cores.length()-1);
            }

            ggml_backend_t result = ggml_backend_metal_init();

            ggml_backend_metal_get_device_properties(result, &props);

            std::string cpu_info = get_cpu_info();
            cpu_info += "+" + num_cores + "GPU";
            strncpy(info->name, cpu_info.c_str(), MAX_STRING_LENGTH - 1);
            info->total_memory_gb = get_mem_gb();
            info->core_count = props.core_count;
            info->capability = props.metal_version;
            strncpy(info->manufacturer, "Apple", MAX_STRING_LENGTH - 1);


            fprintf(stderr, "%s===== GPU information =====\n\n", utils::color_str("\033[0;32m"));
            fprintf(stderr, "%-26s %s\n", "GPU Name:", props.name);
            fprintf(stderr, "%-26s %.1f GiB\n", "VRAM:", info->total_memory_gb);
            fprintf(stderr, "%-26s %d\n", "Core Count:", props.core_count);
            fprintf(stderr, "%-26s %d\n", "Metal Version:", props.metal_version);
            fprintf(stderr, "%-26s %d\n", "GPU Family:", props.gpu_family);
            fprintf(stderr, "%-26s %d\n", "Common GPU Family:", props.gpu_family_common);
            fprintf(stderr, "\n======================================================================\n\n%s", utils::color_str("\033[0m"));
        }
    } else {
        #ifdef __x86_64__
            strncpy(info->manufacturer, get_cpu_manufacturer(), MAX_STRING_LENGTH - 1);
        #else
            if IsXnu() {
                strncpy(info->manufacturer, "Apple", MAX_STRING_LENGTH - 1);
            } else {
                strncpy(info->manufacturer, "Unknown", MAX_STRING_LENGTH - 1);
            }
        #endif
        strncpy(info->name, get_cpu_info().c_str(), MAX_STRING_LENGTH - 1);
        info->total_memory_gb = get_mem_gb(); 
    }
}

void list_available_accelerators() {
    if (llamafile_has_gpu()) {
        if (llamafile_has_metal()) {
            fprintf(stderr, "Apple Metal\n");
        } else if (llamafile_has_cuda()) {
            int count = ggml_backend_cuda_get_device_count();
            fprintf(stderr, "\n%s==================== Available GPUs ====================\n\n", utils::color_str("\033[0;32m"));
            for (int i = 0; i < count; i++) {
                struct ggml_cuda_device_properties props;
                ggml_backend_cuda_get_device_properties(i, &props);
                fprintf(stderr, "%d: %s - %.2f GiB\n", i, props.name, props.totalGlobalMem / 1073741824.0);
            }
        } else {
            fprintf(stderr, "No Accelerator support available\n");
        }
    } else {
        fprintf(stderr, "No Accelerator support available\n");
    }
    fprintf(stderr, "\n======================================================================\n%s", utils::color_str("\033[0m"));
}

void get_model_info(ModelInfo *info, llama_model *model) {
    char buf[MAX_STRING_LENGTH];
    llama_model_desc(model, buf, sizeof(buf));
    strncpy(info->type, buf, sizeof(buf));
    llama_model_meta_val_str(model, "general.name", buf, sizeof(buf));
    strncpy(info->name, buf, sizeof(buf));

    // check if the model name is empty, if it is then exit the program
    if (strcmp(info->name, "") == 0) {
        fprintf(stderr, "Error: Model name is empty. Please use a valid .gguf.\n");
        exit(1);
    }

    llama_model_quant_str(model, buf, sizeof(buf));
    strncpy(info->quant, buf, sizeof(buf));
    llama_model_meta_val_str(model, "general.size_label", buf, sizeof(buf));
    strncpy(info->size_label, buf, sizeof(buf));
    info->size = llama_model_size(model);
    info->params = llama_model_n_params(model);
}