Skip to content

Distribute and run LLMs with a single file.

Remove ifdef statements we don't need

Justine Tunney06dfbbd04d30
diff --git a/llama.cpp/console.cpp b/llama.cpp/console.cpp
index 1f6a8675..bd9f1389 100644
--- a/llama.cpp/console.cpp
+++ b/llama.cpp/console.cpp
@@ -3,19 +3,6 @@
#include "console.h"
#include <vector>
#include <iostream>
-
-#if defined(_WIN32)
-#define WIN32_LEAN_AND_MEAN
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <windows.h>
-#include <fcntl.h>
-#include <io.h>
-#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
-#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
-#endif
-#else
#include <climits>
#include <sys/ioctl.h>
#include <unistd.h>
@@ -24,7 +11,6 @@
#include <stdlib.h>
#include <signal.h>
#include <termios.h>
-#endif
#define ANSI_COLOR_RED "\x1b[31m"
#define ANSI_COLOR_GREEN "\x1b[32m"
@@ -37,468 +23,303 @@
namespace console {
- //
- // Console state
- //
-
- static bool advanced_display = false;
- static bool simple_io = true;
- static display_t current_display = reset;
-
- static FILE* out = stdout;
-
-#if defined (_WIN32)
- static void* hConsole;
-#else
- static FILE* tty = nullptr;
- static termios initial_state;
-#endif
-
- //
- // Init and cleanup
- //
-
- void init(bool use_simple_io, bool use_advanced_display) {
- advanced_display = use_advanced_display;
- simple_io = use_simple_io;
-#if defined(_WIN32)
- // Windows-specific console initialization
- DWORD dwMode = 0;
- hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
- if (hConsole == INVALID_HANDLE_VALUE || !GetConsoleMode(hConsole, &dwMode)) {
- hConsole = GetStdHandle(STD_ERROR_HANDLE);
- if (hConsole != INVALID_HANDLE_VALUE && (!GetConsoleMode(hConsole, &dwMode))) {
- hConsole = nullptr;
- simple_io = true;
- }
- }
- if (hConsole) {
- // Check conditions combined to reduce nesting
- if (advanced_display && !(dwMode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) &&
- !SetConsoleMode(hConsole, dwMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
- advanced_display = false;
- }
- // Set console output codepage to UTF8
- SetConsoleOutputCP(CP_UTF8);
- }
- HANDLE hConIn = GetStdHandle(STD_INPUT_HANDLE);
- if (hConIn != INVALID_HANDLE_VALUE && GetConsoleMode(hConIn, &dwMode)) {
- // Set console input codepage to UTF16
- _setmode(_fileno(stdin), _O_WTEXT);
-
- // Set ICANON (ENABLE_LINE_INPUT) and ECHO (ENABLE_ECHO_INPUT)
- if (simple_io) {
- dwMode |= ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT;
- } else {
- dwMode &= ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT);
- }
- if (!SetConsoleMode(hConIn, dwMode)) {
- simple_io = true;
- }
+//
+// Console state
+//
+
+static bool advanced_display = false;
+static bool simple_io = true;
+static display_t current_display = reset;
+static FILE* out = stdout;
+static FILE* tty = nullptr;
+static termios initial_state;
+
+//
+// Init and cleanup
+//
+
+void init(bool use_simple_io, bool use_advanced_display) {
+ advanced_display = use_advanced_display;
+ simple_io = use_simple_io;
+ if (!simple_io) {
+ struct termios new_termios;
+ tcgetattr(STDIN_FILENO, &initial_state);
+ new_termios = initial_state;
+ new_termios.c_lflag &= ~(ICANON | ECHO);
+ new_termios.c_cc[VMIN] = 1;
+ new_termios.c_cc[VTIME] = 0;
+ tcsetattr(STDIN_FILENO, TCSANOW, &new_termios);
+ tty = fopen("/dev/tty", "w+");
+ if (tty != nullptr) {
+ out = tty;
}
-#else
- // POSIX-specific console initialization
- if (!simple_io) {
- struct termios new_termios;
- tcgetattr(STDIN_FILENO, &initial_state);
- new_termios = initial_state;
- new_termios.c_lflag &= ~(ICANON | ECHO);
- new_termios.c_cc[VMIN] = 1;
- new_termios.c_cc[VTIME] = 0;
- tcsetattr(STDIN_FILENO, TCSANOW, &new_termios);
-
- tty = fopen("/dev/tty", "w+");
- if (tty != nullptr) {
- out = tty;
- }
- }
-
- setlocale(LC_ALL, "");
-#endif
}
+ setlocale(LC_ALL, "");
+}
- void cleanup() {
- // Reset console display
- set_display(reset);
-
-#if !defined(_WIN32)
- // Restore settings on POSIX systems
- if (!simple_io) {
- if (tty != nullptr) {
- out = stdout;
- fclose(tty);
- tty = nullptr;
- }
- tcsetattr(STDIN_FILENO, TCSANOW, &initial_state);
+void cleanup() {
+ // Reset console display
+ set_display(reset);
+ // Restore settings
+ if (!simple_io) {
+ if (tty != nullptr) {
+ out = stdout;
+ fclose(tty);
+ tty = nullptr;
}
-#endif
+ tcsetattr(STDIN_FILENO, TCSANOW, &initial_state);
}
+}
- //
- // Display and IO
- //
-
- // Keep track of current display and only emit ANSI code if it changes
- void set_display(display_t display) {
- if (advanced_display && current_display != display) {
- fflush(stdout);
- switch(display) {
- case reset:
- fprintf(out, ANSI_COLOR_RESET);
- break;
- case prompt:
- fprintf(out, ANSI_COLOR_YELLOW);
- break;
- case user_input:
- fprintf(out, ANSI_BOLD ANSI_COLOR_GREEN);
- break;
- case error:
- fprintf(out, ANSI_BOLD ANSI_COLOR_RED);
- }
- current_display = display;
- fflush(out);
+//
+// Display and IO
+//
+
+// Keep track of current display and only emit ANSI code if it changes
+void set_display(display_t display) {
+ if (advanced_display && current_display != display) {
+ fflush(stdout);
+ switch(display) {
+ case reset:
+ fprintf(out, ANSI_COLOR_RESET);
+ break;
+ case prompt:
+ fprintf(out, ANSI_COLOR_YELLOW);
+ break;
+ case user_input:
+ fprintf(out, ANSI_BOLD ANSI_COLOR_GREEN);
+ break;
+ case error:
+ fprintf(out, ANSI_BOLD ANSI_COLOR_RED);
}
+ current_display = display;
+ fflush(out);
}
+}
- static char32_t getchar32() {
-#if defined(_WIN32)
- HANDLE hConsole = GetStdHandle(STD_INPUT_HANDLE);
- wchar_t high_surrogate = 0;
-
- while (true) {
- INPUT_RECORD record;
- DWORD count;
- if (!ReadConsoleInputW(hConsole, &record, 1, &count) || count == 0) {
- return WEOF;
- }
+static char32_t getchar32() {
+ wchar_t wc = getwchar();
+ if (static_cast<wint_t>(wc) == WEOF) {
+ return WEOF;
+ }
+ return static_cast<char32_t>(wc);
+}
- if (record.EventType == KEY_EVENT && record.Event.KeyEvent.bKeyDown) {
- wchar_t wc = record.Event.KeyEvent.uChar.UnicodeChar;
- if (wc == 0) {
- continue;
- }
+static void pop_cursor() {
+ putc('\b', out);
+}
- if ((wc >= 0xD800) && (wc <= 0xDBFF)) { // Check if wc is a high surrogate
- high_surrogate = wc;
- continue;
- }
- if ((wc >= 0xDC00) && (wc <= 0xDFFF)) { // Check if wc is a low surrogate
- if (high_surrogate != 0) { // Check if we have a high surrogate
- return ((high_surrogate - 0xD800) << 10) + (wc - 0xDC00) + 0x10000;
- }
- }
+static int estimateWidth(char32_t codepoint) {
+ return wcwidth(codepoint);
+}
- high_surrogate = 0; // Reset the high surrogate
- return static_cast<char32_t>(wc);
- }
- }
-#else
- wchar_t wc = getwchar();
- if (static_cast<wint_t>(wc) == WEOF) {
- return WEOF;
- }
+static int put_codepoint(const char* utf8_codepoint, size_t length, int expectedWidth) {
+ // We can trust expectedWidth if we've got one
+ if (expectedWidth >= 0 || tty == nullptr) {
+ fwrite(utf8_codepoint, length, 1, out);
+ return expectedWidth;
+ }
-#if WCHAR_MAX == 0xFFFF
-#error hi
- if ((wc >= 0xD800) && (wc <= 0xDBFF)) { // Check if wc is a high surrogate
- wchar_t low_surrogate = getwchar();
- if ((low_surrogate >= 0xDC00) && (low_surrogate <= 0xDFFF)) { // Check if the next wchar is a low surrogate
- return (static_cast<char32_t>(wc & 0x03FF) << 10) + (low_surrogate & 0x03FF) + 0x10000;
- }
- }
- if ((wc >= 0xD800) && (wc <= 0xDFFF)) { // Invalid surrogate pair
- return 0xFFFD; // Return the replacement character U+FFFD
- }
-#endif
+ fputs("\033[6n", tty); // Query cursor position
+ int x1;
+ int y1;
+ int x2;
+ int y2;
+ int results = 0;
+ results = fscanf(tty, "\033[%d;%dR", &y1, &x1);
- return static_cast<char32_t>(wc);
-#endif
- }
+ fwrite(utf8_codepoint, length, 1, tty);
- static void pop_cursor() {
-#if defined(_WIN32)
- if (hConsole != NULL) {
- CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
- GetConsoleScreenBufferInfo(hConsole, &bufferInfo);
-
- COORD newCursorPosition = bufferInfo.dwCursorPosition;
- if (newCursorPosition.X == 0) {
- newCursorPosition.X = bufferInfo.dwSize.X - 1;
- newCursorPosition.Y -= 1;
- } else {
- newCursorPosition.X -= 1;
- }
+ fputs("\033[6n", tty); // Query cursor position
+ results += fscanf(tty, "\033[%d;%dR", &y2, &x2);
- SetConsoleCursorPosition(hConsole, newCursorPosition);
- return;
- }
-#endif
- putc('\b', out);
+ if (results != 4) {
+ return expectedWidth;
}
- static int estimateWidth(char32_t codepoint) {
-#if defined(_WIN32)
- (void)codepoint;
- return 1;
-#else
- return wcwidth(codepoint);
-#endif
+ int width = x2 - x1;
+ if (width < 0) {
+ // Calculate the width considering text wrapping
+ struct winsize w;
+ ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
+ width += w.ws_col;
}
+ return width;
+}
- static int put_codepoint(const char* utf8_codepoint, size_t length, int expectedWidth) {
-#if defined(_WIN32)
- CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
- if (!GetConsoleScreenBufferInfo(hConsole, &bufferInfo)) {
- // go with the default
- return expectedWidth;
- }
- COORD initialPosition = bufferInfo.dwCursorPosition;
- DWORD nNumberOfChars = length;
- WriteConsole(hConsole, utf8_codepoint, nNumberOfChars, &nNumberOfChars, NULL);
-
- CONSOLE_SCREEN_BUFFER_INFO newBufferInfo;
- GetConsoleScreenBufferInfo(hConsole, &newBufferInfo);
-
- // Figure out our real position if we're in the last column
- if (utf8_codepoint[0] != 0x09 && initialPosition.X == newBufferInfo.dwSize.X - 1) {
- DWORD nNumberOfChars;
- WriteConsole(hConsole, &" \b", 2, &nNumberOfChars, NULL);
- GetConsoleScreenBufferInfo(hConsole, &newBufferInfo);
- }
-
- int width = newBufferInfo.dwCursorPosition.X - initialPosition.X;
- if (width < 0) {
- width += newBufferInfo.dwSize.X;
- }
- return width;
-#else
- // We can trust expectedWidth if we've got one
- if (expectedWidth >= 0 || tty == nullptr) {
- fwrite(utf8_codepoint, length, 1, out);
- return expectedWidth;
- }
-
- fputs("\033[6n", tty); // Query cursor position
- int x1;
- int y1;
- int x2;
- int y2;
- int results = 0;
- results = fscanf(tty, "\033[%d;%dR", &y1, &x1);
+static void replace_last(char ch) {
+ fprintf(out, "\b%c", ch);
+}
- fwrite(utf8_codepoint, length, 1, tty);
+static void append_utf8(char32_t ch, std::string & out) {
+ if (ch <= 0x7F) {
+ out.push_back(static_cast<unsigned char>(ch));
+ } else if (ch <= 0x7FF) {
+ out.push_back(static_cast<unsigned char>(0xC0 | ((ch >> 6) & 0x1F)));
+ out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
+ } else if (ch <= 0xFFFF) {
+ out.push_back(static_cast<unsigned char>(0xE0 | ((ch >> 12) & 0x0F)));
+ out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
+ out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
+ } else if (ch <= 0x10FFFF) {
+ out.push_back(static_cast<unsigned char>(0xF0 | ((ch >> 18) & 0x07)));
+ out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 12) & 0x3F)));
+ out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
+ out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
+ } else {
+ // Invalid Unicode code point
+ }
+}
- fputs("\033[6n", tty); // Query cursor position
- results += fscanf(tty, "\033[%d;%dR", &y2, &x2);
+// Helper function to remove the last UTF-8 character from a string
+static void pop_back_utf8_char(std::string & line) {
+ if (line.empty()) {
+ return;
+ }
- if (results != 4) {
- return expectedWidth;
- }
+ size_t pos = line.length() - 1;
- int width = x2 - x1;
- if (width < 0) {
- // Calculate the width considering text wrapping
- struct winsize w;
- ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
- width += w.ws_col;
+ // Find the start of the last UTF-8 character (checking up to 4 bytes back)
+ for (size_t i = 0; i < 3 && pos > 0; ++i, --pos) {
+ if ((line[pos] & 0xC0) != 0x80) {
+ break; // Found the start of the character
}
- return width;
-#endif
- }
-
- static void replace_last(char ch) {
-#if defined(_WIN32)
- pop_cursor();
- put_codepoint(&ch, 1, 1);
-#else
- fprintf(out, "\b%c", ch);
-#endif
}
+ line.erase(pos);
+}
- static void append_utf8(char32_t ch, std::string & out) {
- if (ch <= 0x7F) {
- out.push_back(static_cast<unsigned char>(ch));
- } else if (ch <= 0x7FF) {
- out.push_back(static_cast<unsigned char>(0xC0 | ((ch >> 6) & 0x1F)));
- out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
- } else if (ch <= 0xFFFF) {
- out.push_back(static_cast<unsigned char>(0xE0 | ((ch >> 12) & 0x0F)));
- out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
- out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
- } else if (ch <= 0x10FFFF) {
- out.push_back(static_cast<unsigned char>(0xF0 | ((ch >> 18) & 0x07)));
- out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 12) & 0x3F)));
- out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
- out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
- } else {
- // Invalid Unicode code point
- }
+static bool readline_advanced(std::string & line, bool multiline_input) {
+ if (out != stdout) {
+ fflush(stdout);
}
- // Helper function to remove the last UTF-8 character from a string
- static void pop_back_utf8_char(std::string & line) {
- if (line.empty()) {
- return;
- }
+ line.clear();
+ std::vector<int> widths;
+ bool is_special_char = false;
+ bool end_of_stream = false;
- size_t pos = line.length() - 1;
+ char32_t input_char;
+ while (true) {
+ fflush(out); // Ensure all output is displayed before waiting for input
+ input_char = getchar32();
- // Find the start of the last UTF-8 character (checking up to 4 bytes back)
- for (size_t i = 0; i < 3 && pos > 0; ++i, --pos) {
- if ((line[pos] & 0xC0) != 0x80) {
- break; // Found the start of the character
- }
+ if (input_char == '\r' || input_char == '\n') {
+ break;
}
- line.erase(pos);
- }
- static bool readline_advanced(std::string & line, bool multiline_input) {
- if (out != stdout) {
- fflush(stdout);
+ if (input_char == (char32_t) WEOF || input_char == 0x04 /* Ctrl+D*/) {
+ end_of_stream = true;
+ break;
}
- line.clear();
- std::vector<int> widths;
- bool is_special_char = false;
- bool end_of_stream = false;
-
- char32_t input_char;
- while (true) {
- fflush(out); // Ensure all output is displayed before waiting for input
- input_char = getchar32();
-
- if (input_char == '\r' || input_char == '\n') {
- break;
- }
-
- if (input_char == (char32_t) WEOF || input_char == 0x04 /* Ctrl+D*/) {
- end_of_stream = true;
- break;
- }
-
- if (is_special_char) {
- set_display(user_input);
- replace_last(line.back());
- is_special_char = false;
- }
+ if (is_special_char) {
+ set_display(user_input);
+ replace_last(line.back());
+ is_special_char = false;
+ }
- if (input_char == '\033') { // Escape sequence
- char32_t code = getchar32();
- if (code == '[' || code == 0x1B) {
- // Discard the rest of the escape sequence
- while ((code = getchar32()) != (char32_t) WEOF) {
- if ((code >= 'A' && code <= 'Z') || (code >= 'a' && code <= 'z') || code == '~') {
- break;
- }
+ if (input_char == '\033') { // Escape sequence
+ char32_t code = getchar32();
+ if (code == '[' || code == 0x1B) {
+ // Discard the rest of the escape sequence
+ while ((code = getchar32()) != (char32_t) WEOF) {
+ if ((code >= 'A' && code <= 'Z') || (code >= 'a' && code <= 'z') || code == '~') {
+ break;
}
}
- } else if (input_char == 0x08 || input_char == 0x7F) { // Backspace
- if (!widths.empty()) {
- int count;
- do {
- count = widths.back();
- widths.pop_back();
- // Move cursor back, print space, and move cursor back again
- for (int i = 0; i < count; i++) {
- replace_last(' ');
- pop_cursor();
- }
- pop_back_utf8_char(line);
- } while (count == 0 && !widths.empty());
- }
- } else {
- int offset = line.length();
- append_utf8(input_char, line);
- int width = put_codepoint(line.c_str() + offset, line.length() - offset, estimateWidth(input_char));
- if (width < 0) {
- width = 0;
- }
- widths.push_back(width);
}
-
- if (!line.empty() && (line.back() == '\\' || line.back() == '/')) {
- set_display(prompt);
- replace_last(line.back());
- is_special_char = true;
+ } else if (input_char == 0x08 || input_char == 0x7F) { // Backspace
+ if (!widths.empty()) {
+ int count;
+ do {
+ count = widths.back();
+ widths.pop_back();
+ // Move cursor back, print space, and move cursor back again
+ for (int i = 0; i < count; i++) {
+ replace_last(' ');
+ pop_cursor();
+ }
+ pop_back_utf8_char(line);
+ } while (count == 0 && !widths.empty());
+ }
+ } else {
+ int offset = line.length();
+ append_utf8(input_char, line);
+ int width = put_codepoint(line.c_str() + offset, line.length() - offset, estimateWidth(input_char));
+ if (width < 0) {
+ width = 0;
}
+ widths.push_back(width);
}
- bool has_more = multiline_input;
- if (is_special_char) {
- replace_last(' ');
- pop_cursor();
+ if (!line.empty() && (line.back() == '\\' || line.back() == '/')) {
+ set_display(prompt);
+ replace_last(line.back());
+ is_special_char = true;
+ }
+ }
- char last = line.back();
- line.pop_back();
- if (last == '\\') {
- line += '\n';
- fputc('\n', out);
- has_more = !has_more;
- } else {
- // llama will just eat the single space, it won't act as a space
- if (line.length() == 1 && line.back() == ' ') {
- line.clear();
- pop_cursor();
- }
- has_more = false;
- }
+ bool has_more = multiline_input;
+ if (is_special_char) {
+ replace_last(' ');
+ pop_cursor();
+
+ char last = line.back();
+ line.pop_back();
+ if (last == '\\') {
+ line += '\n';
+ fputc('\n', out);
+ has_more = !has_more;
} else {
- if (end_of_stream) {
- has_more = false;
- } else {
- line += '\n';
- fputc('\n', out);
+ // llama will just eat the single space, it won't act as a space
+ if (line.length() == 1 && line.back() == ' ') {
+ line.clear();
+ pop_cursor();
}
+ has_more = false;
+ }
+ } else {
+ if (end_of_stream) {
+ has_more = false;
+ } else {
+ line += '\n';
+ fputc('\n', out);
}
-
- fflush(out);
- return has_more;
}
- static bool readline_simple(std::string & line, bool multiline_input) {
-#if defined(_WIN32)
- std::wstring wline;
- if (!std::getline(std::wcin, wline)) {
- // Input stream is bad or EOF received
- line.clear();
- GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0);
- return false;
- }
+ fflush(out);
+ return has_more;
+}
- int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);
- line.resize(size_needed);
- WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);
-#else
- if (!std::getline(std::cin, line)) {
- // Input stream is bad or EOF received
- line.clear();
+static bool readline_simple(std::string & line, bool multiline_input) {
+ if (!std::getline(std::cin, line)) {
+ // Input stream is bad or EOF received
+ line.clear();
+ return false;
+ }
+ if (!line.empty()) {
+ char last = line.back();
+ if (last == '/') { // Always return control on '/' symbol
+ line.pop_back();
return false;
}
-#endif
- if (!line.empty()) {
- char last = line.back();
- if (last == '/') { // Always return control on '/' symbol
- line.pop_back();
- return false;
- }
- if (last == '\\') { // '\\' changes the default action
- line.pop_back();
- multiline_input = !multiline_input;
- }
+ if (last == '\\') { // '\\' changes the default action
+ line.pop_back();
+ multiline_input = !multiline_input;
}
- line += '\n';
-
- // By default, continue input if multiline_input is set
- return multiline_input;
}
+ line += '\n';
- bool readline(std::string & line, bool multiline_input) {
- set_display(user_input);
+ // By default, continue input if multiline_input is set
+ return multiline_input;
+}
- if (simple_io) {
- return readline_simple(line, multiline_input);
- }
- return readline_advanced(line, multiline_input);
+bool readline(std::string & line, bool multiline_input) {
+ set_display(user_input);
+ if (simple_io) {
+ return readline_simple(line, multiline_input);
}
-
+ return readline_advanced(line, multiline_input);
}
+
+} // namespace console
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 9d66e7c1..ca635796 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -1,106 +1,36 @@
// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous "unsafe" warnigns on Windows
-#define _USE_MATH_DEFINES // For M_PI on MSVC
-
#include "ggml.h"
#include "ggml-impl.h"
#include "ggml-quants.h"
+#include "ggml-cuda.h"
+#include "ggml-metal.h"
-#if defined(_MSC_VER) || defined(__MINGW32__)
-#include <malloc.h> // using malloc.h with MSC/MINGW
-#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
-#include <alloca.h>
-#endif
-
-#include <cosmo.h>
-#include <fcntl.h>
#include <assert.h>
+#include <cosmo.h>
#include <errno.h>
-#include <time.h>
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-#include <inttypes.h>
-#include <stdio.h>
+#include <fcntl.h>
#include <float.h>
+#include <inttypes.h>
#include <limits.h>
-#include <stdarg.h>
-#include <signal.h>
-#include <sys/mman.h>
-
-#ifdef GGML_USE_METAL
-#include <unistd.h>
-#endif
-
-#if defined(_MSC_VER)
-// disable "possible loss of data" to avoid hundreds of casts
-// we should just be careful :)
-#pragma warning(disable: 4244 4267)
-
-// disable POSIX deprecation warnigns
-// these functions are never going away, anyway
-#pragma warning(disable: 4996)
-#endif
-
-#if defined(_WIN32)
-
-#include <windows.h>
-
-typedef volatile LONG atomic_int;
-typedef atomic_int atomic_bool;
-
-static void atomic_store(atomic_int * ptr, LONG val) {
- InterlockedExchange(ptr, val);
-}
-static LONG atomic_load(atomic_int * ptr) {
- return InterlockedCompareExchange(ptr, 0, 0);
-}
-static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
- return InterlockedExchangeAdd(ptr, inc);
-}
-static LONG atomic_fetch_sub(atomic_int * ptr, LONG dec) {
- return atomic_fetch_add(ptr, -(dec));
-}
-
-typedef HANDLE pthread_t;
-
-typedef DWORD thread_ret_t;
-static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {
- (void) unused;
- HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);
- if (handle == NULL)
- {
- return EAGAIN;
- }
-
- *out = handle;
- return 0;
-}
-
-static int pthread_join(pthread_t thread, void * unused) {
- (void) unused;
- int ret = (int) WaitForSingleObject(thread, INFINITE);
- CloseHandle(thread);
- return ret;
-}
-
-static int sched_yield (void) {
- Sleep (0);
- return 0;
-}
-#else
+#include <math.h>
#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
#include <stdatomic.h>
-
-typedef void * thread_ret_t;
-
-#include <sys/types.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/auxv.h>
+#include <sys/mman.h>
#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
#include <unistd.h>
-#endif
+typedef void * thread_ret_t;
#ifdef GGML_USE_CPU_HBM
#include <hbwmalloc.h>
@@ -146,52 +76,23 @@ typedef void * thread_ret_t;
// end of logging block
//
-#ifdef GGML_USE_ACCELERATE
-// uncomment to use vDSP for soft max computation
-// note: not sure if it is actually faster
-//#define GGML_SOFT_MAX_ACCELERATE
-#endif
-
-#if defined(_MSC_VER) || defined(__MINGW32__)
-#define GGML_ALIGNED_MALLOC(size) _aligned_malloc(size, GGML_MEM_ALIGN)
-#define GGML_ALIGNED_FREE(ptr) _aligned_free(ptr)
-#else
-inline static void * ggml_aligned_malloc(size_t size) {
+static void * ggml_aligned_malloc(size_t size) {
if (size == 0) {
GGML_PRINT("WARNING: Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");
return NULL;
}
void * aligned_memory = NULL;
-#ifdef GGML_USE_CPU_HBM
- int result = hbw_posix_memalign(&aligned_memory, 16, size);
-#elif GGML_USE_METAL
- int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
-#else
- int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
-#endif
- if (result != 0) {
- // Handle allocation failure
- const char *error_desc = "unknown allocation error";
- switch (result) {
- case EINVAL:
- error_desc = "invalid alignment value";
- break;
- case ENOMEM:
- error_desc = "insufficient memory";
- break;
- }
- GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));
+ errno_t err = posix_memalign(&aligned_memory, getauxval(AT_PAGESZ), size);
+ if (err) {
+ GGML_PRINT("%s: %s (attempted to allocate %6.2f MB)\n", __func__, strerror(err), size/(1024.0*1024.0));
return NULL;
}
return aligned_memory;
}
-#define GGML_ALIGNED_MALLOC(size) ggml_aligned_malloc(size)
-#ifdef GGML_USE_CPU_HBM
-#define GGML_ALIGNED_FREE(ptr) if(NULL != ptr) hbw_free(ptr)
-#else
-#define GGML_ALIGNED_FREE(ptr) free(ptr)
-#endif
-#endif
+
+static void ggml_aligned_free(void *p) {
+ free(p);
+}
#define UNUSED GGML_UNUSED
#define SWAP(x, y, T) do { T SWAP = x; x = y; y = SWAP; } while (0)
@@ -214,26 +115,6 @@ inline static void * ggml_aligned_malloc(size_t size) {
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
-#if defined(GGML_USE_ACCELERATE)
-#include <Accelerate/Accelerate.h>
-#if defined(GGML_USE_CLBLAST) // allow usage of CLBlast alongside Accelerate functions
-#include "ggml-opencl.h"
-#endif
-#elif defined(GGML_USE_OPENBLAS)
-#if defined(GGML_BLAS_USE_MKL)
-#include <mkl.h>
-#else
-#include <cblas.h>
-#endif
-#elif defined(GGML_USE_CUBLAS)
-#include "ggml-cuda.h"
-#elif defined(GGML_USE_CLBLAST)
-#include "ggml-opencl.h"
-#endif
-
-#include "ggml-cuda.h"
-#include "ggml-metal.h"
-
// floating point type used to accumulate sums
typedef double ggml_float;
@@ -304,31 +185,8 @@ void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) {
// timing
//
-#if defined(_MSC_VER) || defined(__MINGW32__)
-static int64_t timer_freq, timer_start;
-void ggml_time_init(void) {
- LARGE_INTEGER t;
- QueryPerformanceFrequency(&t);
- timer_freq = t.QuadPart;
-
- // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq
- // and the uptime is high enough.
- // We subtract the program start time to reduce the likelihood of that happening.
- QueryPerformanceCounter(&t);
- timer_start = t.QuadPart;
-}
-int64_t ggml_time_ms(void) {
- LARGE_INTEGER t;
- QueryPerformanceCounter(&t);
- return ((t.QuadPart-timer_start) * 1000) / timer_freq;
-}
-int64_t ggml_time_us(void) {
- LARGE_INTEGER t;
- QueryPerformanceCounter(&t);
- return ((t.QuadPart-timer_start) * 1000000) / timer_freq;
-}
-#else
void ggml_time_init(void) {}
+
int64_t ggml_time_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -340,7 +198,6 @@ int64_t ggml_time_us(void) {
clock_gettime(CLOCK_MONOTONIC, &ts);
return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;
}
-#endif
int64_t ggml_cycles(void) {
return clock();
@@ -1844,7 +1701,10 @@ void ggml_numa_init(void) {
return;
}
-#ifdef __linux__
+ if (!IsLinux()) {
+ return; // todo
+ }
+
struct stat st;
char path[256];
int rv;
@@ -1897,9 +1757,6 @@ void ggml_numa_init(void) {
fclose(fptr);
}
}
-#else
- // TODO
-#endif
}
bool ggml_is_numa(void) {
@@ -2239,7 +2096,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
*ctx = (struct ggml_context) {
/*.mem_size =*/ mem_size,
- /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : GGML_ALIGNED_MALLOC(mem_size),
+ /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : ggml_aligned_malloc(mem_size),
/*.mem_buffer_owned =*/ params.mem_buffer ? false : true,
/*.no_alloc =*/ params.no_alloc,
/*.no_alloc_save =*/ params.no_alloc,
@@ -2275,7 +2132,7 @@ void ggml_free(struct ggml_context * ctx) {
__func__, i, ggml_used_mem(ctx));
if (ctx->mem_buffer_owned) {
- GGML_ALIGNED_FREE(ctx->mem_buffer);
+ ggml_aligned_free(ctx->mem_buffer);
}
found = true;
@@ -15880,64 +15737,7 @@ size_t ggml_graph_overhead(void) {
// I tried using spin locks, but not sure how to use them correctly - the things I tried were slower than busy loops
//
-#ifdef __APPLE__
-
-//#include <os/lock.h>
-//
-//typedef os_unfair_lock ggml_lock_t;
-//
-//#define ggml_lock_init(x) UNUSED(x)
-//#define ggml_lock_destroy(x) UNUSED(x)
-//#define ggml_lock_lock os_unfair_lock_lock
-//#define ggml_lock_unlock os_unfair_lock_unlock
-//
-//#define GGML_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
-
-typedef int ggml_lock_t;
-
-#define ggml_lock_init(x) UNUSED(x)
-#define ggml_lock_destroy(x) UNUSED(x)
-#define ggml_lock_lock(x) UNUSED(x)
-#define ggml_lock_unlock(x) UNUSED(x)
-
-#define GGML_LOCK_INITIALIZER 0
-
-typedef pthread_t ggml_thread_t;
-
-#define ggml_thread_create pthread_create
-#define ggml_thread_join pthread_join
-
-#else
-
-//typedef pthread_spinlock_t ggml_lock_t;
-
-//#define ggml_lock_init(x) pthread_spin_init(x, PTHREAD_PROCESS_PRIVATE)
-//#define ggml_lock_destroy pthread_spin_destroy
-//#define ggml_lock_lock pthread_spin_lock
-//#define ggml_lock_unlock pthread_spin_unlock
-
-typedef int ggml_lock_t;
-
-#define ggml_lock_init(x) UNUSED(x)
-#define ggml_lock_destroy(x) UNUSED(x)
-#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))
-#define ggml_lock_lock(x) _mm_pause()
-#else
-#define ggml_lock_lock(x) UNUSED(x)
-#endif
-#define ggml_lock_unlock(x) UNUSED(x)
-
-#define GGML_LOCK_INITIALIZER 0
-
-typedef pthread_t ggml_thread_t;
-
-#define ggml_thread_create pthread_create
-#define ggml_thread_join pthread_join
-
-#endif
-
// Android's libc implementation "bionic" does not support setting affinity
-#if defined(__linux__) && !defined(__BIONIC__)
static void set_numa_thread_affinity(int thread_n, int n_threads) {
if (!ggml_is_numa()) {
return;
@@ -15984,12 +15784,6 @@ static void clear_numa_thread_affinity(void) {
CPU_FREE(cpus);
}
-#else
-// TODO: Windows etc.
-// (the linux implementation may also work on BSD, someone should test)
-static void set_numa_thread_affinity(int thread_n, int n_threads) { UNUSED(thread_n); UNUSED(n_threads); }
-static void clear_numa_thread_affinity(void) {}
-#endif
struct ggml_compute_state_shared {
const struct ggml_cgraph * cgraph;
@@ -16009,7 +15803,7 @@ struct ggml_compute_state_shared {
};
struct ggml_compute_state {
- ggml_thread_t thrd;
+ pthread_t thrd;
int ith;
struct ggml_compute_state_shared * shared;
};
@@ -16657,7 +16451,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
.shared = &state_shared,
};
- const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);
+ const int rc = pthread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);
GGML_ASSERT(rc == 0);
UNUSED(rc);
}
@@ -16678,7 +16472,7 @@ int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
// join or kill thread pool
if (n_threads > 1) {
for (int j = 1; j < n_threads; j++) {
- const int rc = ggml_thread_join(workers[j].thrd, NULL);
+ const int rc = pthread_join(workers[j].thrd, NULL);
GGML_ASSERT(rc == 0);
}
}
@@ -18667,7 +18461,7 @@ static bool gguf_fread_str(struct llamafile * file, struct gguf_str * p, size_t
}
struct gguf_context * gguf_init_empty(void) {
- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));
+ struct gguf_context * ctx = ggml_aligned_malloc(sizeof(struct gguf_context));
memcpy(ctx->header.magic, GGUF_MAGIC, sizeof(ctx->header.magic));
ctx->header.version = GGUF_VERSION;
@@ -18711,7 +18505,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
bool ok = true;
- struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context));
+ struct gguf_context * ctx = ggml_aligned_malloc(sizeof(struct gguf_context));
// read the header
{
@@ -19019,7 +18813,7 @@ void gguf_free(struct gguf_context * ctx) {
free(ctx->infos);
}
- GGML_ALIGNED_FREE(ctx);
+ ggml_aligned_free(ctx);
}
const char * gguf_type_name(enum gguf_type type) {
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 33e6e865..b2940ec7 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -1,8 +1,6 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
// vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#include <cosmo.h>
-
#define LLAMA_API_INTERNAL
#include "llama.h"
@@ -41,22 +39,13 @@
#endif
#endif
-#if defined(_WIN32)
-#define WIN32_LEAN_AND_MEAN
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <windows.h>
-#include <io.h>
-#include <stdio.h> // for _fseeki64
-#endif
-
#include <algorithm>
#include <array>
#include <cassert>
#include <cinttypes>
#include <climits>
#include <cmath>
+#include <cosmo.h>
#include <cstdarg>
#include <cstddef>
#include <cstdint>
@@ -80,10 +69,6 @@
#include <thread>
#include <unordered_map>
-#if defined(_MSC_VER)
-#pragma warning(disable: 4244 4267) // possible loss of data
-#endif
-
#ifdef __GNUC__
#ifdef __MINGW32__
#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
@@ -632,20 +617,6 @@ inline void llama_host_free(void * ptr) {
}
}
-#if defined(_WIN32)
-static std::string llama_format_win_err(DWORD err) {
- LPSTR buf;
- size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
- NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
- if (!size) {
- return "FormatMessageA failed";
- }
- std::string ret(buf, size);
- LocalFree(buf);
- return ret;
-}
-#endif
-
struct llama_buffer {
void * data = NULL;
size_t size = 0;
@@ -746,7 +717,6 @@ struct llama_mmap {
llama_mmap(const llama_mmap &) = delete;
-#ifdef _POSIX_MAPPED_FILES
static constexpr bool SUPPORTED = true;
llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {
@@ -761,13 +731,9 @@ struct llama_mmap {
}
is_owned = true;
int fd = fileno(llamafile_fp(file->file));
- int flags = MAP_SHARED;
// prefetch/readahead impairs performance on NUMA systems
if (numa) { prefetch = 0; }
-#ifdef __linux__
- if (prefetch) { flags |= MAP_POPULATE; }
-#endif
- addr = mmap(NULL, size, PROT_READ, flags, fd, 0);
+ addr = mmap(NULL, size, PROT_READ, MAP_SHARED | MAP_POPULATE, fd, 0);
if (addr == MAP_FAILED) {
ThrowRuntimeError(format("mmap failed: %s", strerror(errno)));
}
@@ -797,69 +763,6 @@ struct llama_mmap {
munmap(addr, size);
}
}
-#elif defined(_WIN32)
- static constexpr bool SUPPORTED = true;
-
- llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {
- (void) numa;
-
- size = file->size;
-
- HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
-
- HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
- DWORD error = GetLastError();
-
- if (hMapping == NULL) {
- ThrowRuntimeError(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
- }
-
- addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
- error = GetLastError();
- CloseHandle(hMapping);
-
- if (addr == NULL) {
- ThrowRuntimeError(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
- }
-
- if (prefetch) {
- // PrefetchVirtualMemory is only present on Windows 8 and above, so we dynamically load it
- BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
- HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
-
- // may fail on pre-Windows 8 systems
- pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));
-
- if (pPrefetchVirtualMemory) {
- // advise the kernel to preload the mapped memory
- WIN32_MEMORY_RANGE_ENTRY range;
- range.VirtualAddress = addr;
- range.NumberOfBytes = (SIZE_T)size;
- if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
- fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- }
- }
- }
- }
-
- ~llama_mmap() {
- if (!UnmapViewOfFile(addr)) {
- fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",
- llama_format_win_err(GetLastError()).c_str());
- }
- }
-#else
- static constexpr bool SUPPORTED = false;
-
- llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {
- (void) file;
- (void) prefetch;
- (void) numa;
-
- ThrowRuntimeError(std::string("mmap not supported"));
- }
-#endif
};
// Represents some region of memory being locked using mlock or VirtualLock;
@@ -7413,11 +7316,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
// mmap consistently increases speed Linux, and also increases speed on Windows with
// hot cache. It may cause a slowdown on macOS, possibly related to free memory.
-#if defined(__linux__) || defined(_WIN32)
- constexpr bool use_mmap = true;
-#else
- constexpr bool use_mmap = false;
-#endif
+ bool use_mmap = !IsXnu();
llama_model_loader ml(fname_inp, use_mmap);
if (ml.use_mmap) {
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index e8dc04bb..4f05ff16 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -44,11 +44,6 @@
#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
#define LLAMA_SESSION_VERSION 2
-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL)
-// Defined when llama.cpp is compiled with support for offloading model layers to GPU.
-#define LLAMA_SUPPORTS_GPU_OFFLOAD
-#endif
-
#ifdef __cplusplus
extern "C" {
#endif
diff --git a/llama.cpp/server/.args b/llama.cpp/server/.args
deleted file mode 100644
index d077c336..00000000
--- a/llama.cpp/server/.args
+++ /dev/null
@@ -1,5 +0,0 @@
--m
-llava-v1.5-7b-Q8_0.gguf
---mmproj
-llava-v1.5-7b-mmproj-Q8_0.gguf
-...
diff --git a/llama.cpp/server/BUILD.mk b/llama.cpp/server/BUILD.mk
index 79d206b2..a302f57c 100644
--- a/llama.cpp/server/BUILD.mk
+++ b/llama.cpp/server/BUILD.mk
@@ -4,22 +4,16 @@
PKGS += LLAMA_CPP_SERVER
LLAMA_CPP_SERVER_FILES := $(wildcard llama.cpp/server/*)
+LLAMA_CPP_SERVER_ASSETS := $(wildcard llama.cpp/server/public/*)
LLAMA_CPP_SERVER_HDRS = $(filter %.h,$(LLAMA_CPP_SERVER_FILES))
LLAMA_CPP_SERVER_SRCS = $(filter %.cpp,$(LLAMA_CPP_SERVER_FILES))
-LLAMA_CPP_SERVER_ASSETS := \
- $(wildcard llama.cpp/server/public/*) \
- llama.cpp/server/.args
-
o/$(MODE)/llama.cpp/server/server: \
o/$(MODE)/llama.cpp/server/server.o \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
-# strip llama.cpp/server/ directory prefix off name in zip archive
-o/$(MODE)/llama.cpp/server/.args.zip.o: private ZIPOBJ_FLAGS += -B
-
.PHONY: o/$(MODE)/llama.cpp/server
o/$(MODE)/llama.cpp/server: \
o/$(MODE)/llama.cpp/server/server
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 4a8c52a0..61ae4461 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -1778,7 +1778,6 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");
}
printf(" --numa attempt optimizations that help on some NUMA systems\n");
-#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD
printf(" -ngl N, --n-gpu-layers N\n");
printf(" number of layers to store in VRAM\n");
printf(" -ts SPLIT --tensor-split SPLIT\n");
@@ -1787,7 +1786,6 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
printf(" -nommq, --no-mul-mat-q\n");
printf(" use cuBLAS instead of custom mul_mat_q CUDA kernels.\n");
printf(" Not recommended since this is both slower and uses more VRAM.\n");
-#endif
printf(" -m FNAME, --model FNAME\n");
printf(" model path (default: %s)\n", params.model.c_str());
printf(" -a ALIAS, --alias ALIAS\n");
@@ -1990,13 +1988,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
invalid_param = true;
break;
}
-#ifdef LLAMA_SUPPORTS_GPU_OFFLOAD
params.n_gpu_layers = std::stoi(argv[i]);
-#else
- LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "
- "See main README.md for information on enabling GPU BLAS support",
- {{"n_gpu_layers", params.n_gpu_layers}});
-#endif
}
else if (arg == "--tensor-split" || arg == "-ts")
{
@@ -2005,7 +1997,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
invalid_param = true;
break;
}
-#ifdef GGML_USE_CUBLAS
std::string arg_next = argv[i];
// split string by , and /
@@ -2025,17 +2016,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
params.tensor_split[i_device] = 0.0f;
}
}
-#else
- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a tensor split.\n", {});
-#endif // GGML_USE_CUBLAS
}
else if (arg == "--no-mul-mat-q" || arg == "-nommq")
{
-#ifdef GGML_USE_CUBLAS
params.mul_mat_q = false;
-#else
- LOG_WARNING("warning: llama.cpp was compiled without cuBLAS. Disabling mul_mat_q kernels has no effect.\n", {});
-#endif // GGML_USE_CUBLAS
}
else if (arg == "--main-gpu" || arg == "-mg")
{
@@ -2044,11 +2028,7 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
invalid_param = true;
break;
}
-#ifdef GGML_USE_CUBLAS
params.main_gpu = std::stoi(argv[i]);
-#else
- LOG_WARNING("llama.cpp was compiled without cuBLAS. It is not possible to set a main GPU.", {});
-#endif
}
else if (arg == "--lora")
{
diff --git a/llamafile/llamafile.c b/llamafile/llamafile.c
index 28f9ed84..4f0df41c 100644
--- a/llamafile/llamafile.c
+++ b/llamafile/llamafile.c
@@ -216,7 +216,7 @@ struct llamafile *llamafile_open(const char *fname, const char *mode) {
off_t mapoff = off & -pagesz;
long skew = off - mapoff;
file->mapsize = skew + file->size;
- file->mapping = mmap(0, file->mapsize, PROT_READ, MAP_SHARED, fd, mapoff);
+ file->mapping = mmap(0, file->mapsize, PROT_READ, MAP_SHARED | MAP_POPULATE, fd, mapoff);
if (file->mapping == MAP_FAILED) {
fprintf(stderr, "warning: failed to map %s from %s: %s\n",
fname, prog, strerror(errno));