Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add shell syntax highlighting
Justine Tunney3e1942c15ab3
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex babfe378..8960e6a9 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -68,6 +68,8 @@ Highlight *Highlight::create(const std::string_view &lang) {return new HighlightHaskell;} else if (lang == "perl" || lang == "pl") {return new HighlightPerl;+ } else if (lang == "shell" || lang == "bash" || lang == "sh") {+ return new HighlightShell;} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 7bc3fc7a..c37fa206 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -67,6 +67,8 @@ is_keyword_f is_keyword_lisp;is_keyword_f is_keyword_ada;is_keyword_f is_keyword_haskell;is_keyword_f is_keyword_perl;+is_keyword_f is_keyword_shell;+is_keyword_f is_keyword_shell_builtin;}class Highlight {@@ -288,3 +290,15 @@ class HighlightPerl : public Highlight {int t_ = 0;std::string word_;};++class HighlightShell : public Highlight {+ public:+ HighlightShell();+ ~HighlightShell() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cppindex 1a382328..89d462dd 100644--- a/llamafile/highlight_perl.cpp+++ b/llamafile/highlight_perl.cpp@@ -76,7 +76,6 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {}break;- Word:case WORD:if (!isascii(c) || isalnum(c) || c == '_') {word_ += c;diff --git a/llamafile/highlight_php.cpp b/llamafile/highlight_php.cppindex 9e0bc221..3a86e986 100644--- a/llamafile/highlight_php.cpp+++ b/llamafile/highlight_php.cpp@@ -73,7 +73,6 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {}break;- Word:case WORD:if (!isascii(c) || isalnum(c) || c == '_') {word_ += c;diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cppnew file mode 100644index 00000000..1a1c6b04--- /dev/null+++ b/llamafile/highlight_shell.cpp@@ -0,0 +1,212 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++#define NORMAL 0+#define WORD 1+#define QUOTE 2+#define DQUOTE 4+#define DQUOTE_BACKSLASH 5+#define TICK 6+#define TICK_BACKSLASH 7+#define VAR 8+#define VAR2 9+#define COMMENT 10++HighlightShell::HighlightShell() {+}++HighlightShell::~HighlightShell() {+}++void HighlightShell::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '`') {+ t_ = TICK;+ *r += HI_STRING;+ *r += c;+ } else if (c == '$') {+ *r += c;+ t_ = VAR;+ } else if (c == '#') {+ *r += HI_COMMENT;+ *r += c;+ t_ = COMMENT;+ } else {+ *r += c;+ }+ break;++ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_shell(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case VAR:+ if (c == '#' || //+ c == '*' || //+ c == '-' || //+ c == '?' || //+ c == '@' || //+ c == '!' || //+ c == '$') {+ *r += HI_VAR;+ *r += c;+ *r += HI_RESET;+ t_ = NORMAL;+ break;+ } else if (c == '{') {+ *r += c;+ *r += HI_VAR;+ t_ = VAR2;+ break;+ } else {+ *r += HI_VAR;+ t_ = VAR2;+ }+ // fallthrough++ case VAR2:+ if (!isascii(c) || isalnum(c) || c == '_') {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case COMMENT:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE_BACKSLASH;+ }+ break;++ case DQUOTE_BACKSLASH:+ *r += c;+ t_ = DQUOTE;+ break;++ case TICK:+ *r += c;+ if (c == '`') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = TICK_BACKSLASH;+ }+ break;++ case TICK_BACKSLASH:+ *r += c;+ t_ = TICK;+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightShell::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_shell(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case VAR2:+ case TICK:+ case TICK_BACKSLASH:+ case QUOTE:+ case DQUOTE:+ case DQUOTE_BACKSLASH:+ case COMMENT:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex 4781e7de..23eb482e 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -52,6 +52,7 @@ const char *const kLanguages[] = {"plain", //"python", //"rust", //+ "shell", //"sql", //};diff --git a/llamafile/is_keyword_shell.c b/llamafile/is_keyword_shell.cnew file mode 100644index 00000000..6668a4ed--- /dev/null+++ b/llamafile/is_keyword_shell.c@@ -0,0 +1,149 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_shell.gperf */+/* Computed positions: -k'2-3' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_shell.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 22+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 8+#define MIN_HASH_VALUE 2+#define MAX_HASH_VALUE 39+/* maximum key range = 38, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 10, 25, 40,+ 40, 5, 15, 40, 20, 10, 5, 40, 5, 5,+ 5, 0, 40, 0, 5, 0, 13, 0, 40, 40,+ 5, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,+ 40, 40, 40, 40, 40, 40, 40+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[2]+1];+ /*FALLTHROUGH*/+ case 2:+ hval += asso_values[(unsigned char)str[1]];+ break;+ }+ return hval;+}++const char *+is_keyword_shell (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "",+ "do",+ "for",+ "done",+ "",+ "coproc",+ "in",+ "function",+ "expr",+ "until",+ "return",+ "fi",+ "",+ "elif",+ "",+ "select",+ "if",+ "",+ "time",+ "", "",+ "else",+ "",+ "exec",+ "break",+ "",+ "case",+ "",+ "esac",+ "while",+ "", "", "",+ "trap",+ "", "", "", "",+ "then"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_shell.gperf b/llamafile/is_keyword_shell.gperfnew file mode 100644index 00000000..e4771ad5--- /dev/null+++ b/llamafile/is_keyword_shell.gperf@@ -0,0 +1,30 @@+%{+#include <string.h>+%}+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_shell+%%+break+case+coproc+do+done+elif+else+esac+exec+expr+fi+for+function+if+in+return+select+then+time+trap+until+whilediff --git a/llamafile/is_keyword_shell_builtin.c b/llamafile/is_keyword_shell_builtin.cnew file mode 100644index 00000000..4682a5f0--- /dev/null+++ b/llamafile/is_keyword_shell_builtin.c@@ -0,0 +1,165 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_shell_builtin.gperf */+/* Computed positions: -k'1-2' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_shell_builtin.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 41+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 9+#define MIN_HASH_VALUE 4+#define MAX_HASH_VALUE 69+/* maximum key range = 66, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 25, 25, 5,+ 45, 5, 15, 10, 10, 20, 1, 45, 5, 30,+ 15, 0, 5, 70, 20, 0, 0, 10, 55, 30,+ 45, 40, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,+ 70, 70, 70, 70, 70, 70+ };+ return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_shell_builtin (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "", "", "",+ "stop",+ "jobs",+ "source",+ "",+ "set",+ "test",+ "local",+ "logout",+ "command",+ "let",+ "echo",+ "shift",+ "",+ "suspend",+ "",+ "help",+ "chdir",+ "ulimit",+ "getopts",+ "",+ "true",+ "times",+ "enable",+ "fg",+ "",+ "read",+ "unset",+ "printf",+ "unalias",+ "",+ "readarray",+ "alias",+ "caller",+ "bg",+ "",+ "hash",+ "", "",+ "builtin",+ "",+ "type",+ "false",+ "",+ "typeset",+ "",+ "bind",+ "", "",+ "cd",+ "",+ "exit",+ "", "",+ "declare",+ "",+ "wait",+ "", "",+ "mapfile",+ "",+ "eval",+ "", "", "", "",+ "kill"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_shell_builtin.gperf b/llamafile/is_keyword_shell_builtin.gperfnew file mode 100644index 00000000..520cdf8a--- /dev/null+++ b/llamafile/is_keyword_shell_builtin.gperf@@ -0,0 +1,49 @@+%{+#include <string.h>+%}+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_shell_builtin+%%+alias+bg+bind+builtin+caller+cd+chdir+command+declare+echo+enable+eval+exit+false+fg+getopts+hash+help+jobs+kill+let+local+logout+mapfile+printf+read+readarray+set+shift+source+stop+suspend+test+times+true+type+typeset+ulimit+unalias+unset+wait