Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add Go syntax highlighting
Justine Tunney6d20fd5ac8be
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 5a294288..63e8c866 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -38,6 +38,8 @@ Highlight *Highlight::create(const std::string_view &lang) {return new HighlightCobol();} else if (lang == "pas" || lang == "pascal") {return new HighlightPascal();+ } else if (lang == "go") {+ return new HighlightC(is_keyword_go, is_keyword_go_type);} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 3ad68c18..7707e7a6 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -44,6 +44,8 @@ is_keyword_f is_keyword_fortran_type;is_keyword_f is_keyword_cobol;is_keyword_f is_keyword_pascal;is_keyword_f is_keyword_pascal_type;+is_keyword_f is_keyword_go;+is_keyword_f is_keyword_go_type;}class Highlight {@@ -64,7 +66,7 @@ class HighlightPlain : public Highlight {class HighlightC : public Highlight {public:- explicit HighlightC(is_keyword_f is_keyword = is_keyword_c);+ explicit HighlightC(is_keyword_f is_keyword = is_keyword_c, is_keyword_f is_type = nullptr);~HighlightC() override;void feed(std::string *result, std::string_view input) override;void flush(std::string *result) override;@@ -72,6 +74,7 @@ class HighlightC : public Highlight {private:int t_ = 0;std::string word_;+ is_keyword_f *is_type_;is_keyword_f *is_keyword_;};diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cppindex 58e42f5d..1e635713 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight_c.cpp@@ -29,7 +29,8 @@#define SLASH_STAR_STAR 7#define BACKSLASH 64-HighlightC::HighlightC(is_keyword_f *is_keyword) : is_keyword_(is_keyword) {+HighlightC::HighlightC(is_keyword_f *is_keyword, is_keyword_f *is_type)+ : is_keyword_(is_keyword), is_type_(is_type) {}HighlightC::~HighlightC() {@@ -81,6 +82,10 @@ void HighlightC::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;} else {*r += word_;}@@ -161,6 +166,10 @@ void HighlightC::flush(std::string *r) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;} else {*r += word_;}diff --git a/llamafile/is_keyword_go.c b/llamafile/is_keyword_go.cnew file mode 100644index 00000000..b4ef2d4b--- /dev/null+++ b/llamafile/is_keyword_go.c@@ -0,0 +1,138 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_go.gperf */+/* Computed positions: -k'1-2' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_go.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 25+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 11+#define MIN_HASH_VALUE 2+#define MAX_HASH_VALUE 36+/* maximum key range = 35, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 0, 10, 0,+ 0, 10, 0, 5, 30, 0, 37, 37, 15, 10,+ 15, 0, 5, 37, 5, 10, 15, 10, 15, 20,+ 37, 0, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,+ 37, 37, 37, 37, 37, 37+ };+ return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_go (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "",+ "if",+ "for",+ "case",+ "const",+ "",+ "go",+ "continue",+ "goto",+ "range",+ "fallthrough",+ "package",+ "map",+ "func",+ "defer",+ "import",+ "default",+ "var",+ "type",+ "break",+ "return",+ "", "",+ "interface",+ "",+ "select",+ "", "",+ "else",+ "",+ "struct",+ "", "",+ "chan",+ "",+ "switch"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_go.gperf b/llamafile/is_keyword_go.gperfnew file mode 100644index 00000000..981b3d23--- /dev/null+++ b/llamafile/is_keyword_go.gperf@@ -0,0 +1,33 @@+%{+#include <string.h>+%}+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_go+%%+break+case+chan+const+continue+default+defer+else+fallthrough+for+func+go+goto+if+import+interface+map+package+range+return+select+struct+switch+type+vardiff --git a/llamafile/is_keyword_go_type.c b/llamafile/is_keyword_go_type.cnew file mode 100644index 00000000..707fdad5--- /dev/null+++ b/llamafile/is_keyword_go_type.c@@ -0,0 +1,127 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf is_keyword_go_type.gperf */+/* Computed positions: -k'1,$' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "is_keyword_go_type.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 17+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 10+#define MIN_HASH_VALUE 4+#define MAX_HASH_VALUE 26+/* maximum key range = 23, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 10, 27, 5, 27, 20, 27, 0, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 5, 10,+ 27, 10, 0, 27, 27, 0, 27, 27, 27, 27,+ 27, 27, 27, 27, 0, 27, 5, 0, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,+ 27, 27, 27, 27, 27, 27+ };+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_go_type (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "", "", "",+ "int8",+ "uint8",+ "",+ "uintptr",+ "int",+ "uint",+ "int64",+ "uint64",+ "float64",+ "",+ "rune",+ "int32",+ "uint32",+ "float32",+ "",+ "byte",+ "complex128",+ "", "", "",+ "complex64",+ "int16",+ "uint16"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/is_keyword_go_type.gperfnew file mode 100644index 00000000..2f6eac5b--- /dev/null+++ b/llamafile/is_keyword_go_type.gperf@@ -0,0 +1,25 @@+%{+#include <string.h>+%}+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_go_type+%%+byte+complex128+complex64+float32+float64+int+int16+int32+int64+int8+rune+uint+uint16+uint32+uint64+uint8+uintptr