Skip to content

Distribute and run LLMs with a single file.

Add Go syntax highlighting

Justine Tunney6d20fd5ac8be
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 5a294288..63e8c866 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -38,6 +38,8 @@ Highlight *Highlight::create(const std::string_view &lang) {
return new HighlightCobol();
} else if (lang == "pas" || lang == "pascal") {
return new HighlightPascal();
+ } else if (lang == "go") {
+ return new HighlightC(is_keyword_go, is_keyword_go_type);
} else {
return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 3ad68c18..7707e7a6 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -44,6 +44,8 @@ is_keyword_f is_keyword_fortran_type;
is_keyword_f is_keyword_cobol;
is_keyword_f is_keyword_pascal;
is_keyword_f is_keyword_pascal_type;
+is_keyword_f is_keyword_go;
+is_keyword_f is_keyword_go_type;
}
class Highlight {
@@ -64,7 +66,7 @@ class HighlightPlain : public Highlight {
class HighlightC : public Highlight {
public:
- explicit HighlightC(is_keyword_f is_keyword = is_keyword_c);
+ explicit HighlightC(is_keyword_f is_keyword = is_keyword_c, is_keyword_f is_type = nullptr);
~HighlightC() override;
void feed(std::string *result, std::string_view input) override;
void flush(std::string *result) override;
@@ -72,6 +74,7 @@ class HighlightC : public Highlight {
private:
int t_ = 0;
std::string word_;
+ is_keyword_f *is_type_;
is_keyword_f *is_keyword_;
};
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cpp
index 58e42f5d..1e635713 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight_c.cpp
@@ -29,7 +29,8 @@
#define SLASH_STAR_STAR 7
#define BACKSLASH 64
-HighlightC::HighlightC(is_keyword_f *is_keyword) : is_keyword_(is_keyword) {
+HighlightC::HighlightC(is_keyword_f *is_keyword, is_keyword_f *is_type)
+ : is_keyword_(is_keyword), is_type_(is_type) {
}
HighlightC::~HighlightC() {
@@ -81,6 +82,10 @@ void HighlightC::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
} else {
*r += word_;
}
@@ -161,6 +166,10 @@ void HighlightC::flush(std::string *r) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
} else {
*r += word_;
}
diff --git a/llamafile/is_keyword_go.c b/llamafile/is_keyword_go.c
new file mode 100644
index 00000000..b4ef2d4b
--- /dev/null
+++ b/llamafile/is_keyword_go.c
@@ -0,0 +1,138 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_go.gperf */
+/* Computed positions: -k'1-2' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_go.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 25
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 11
+#define MIN_HASH_VALUE 2
+#define MAX_HASH_VALUE 36
+/* maximum key range = 35, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 0, 10, 0,
+ 0, 10, 0, 5, 30, 0, 37, 37, 15, 10,
+ 15, 0, 5, 37, 5, 10, 15, 10, 15, 20,
+ 37, 0, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37, 37, 37, 37, 37,
+ 37, 37, 37, 37, 37, 37
+ };
+ return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_go (register const char *str, register size_t len)
+{
+ static const char * const wordlist[] =
+ {
+ "", "",
+ "if",
+ "for",
+ "case",
+ "const",
+ "",
+ "go",
+ "continue",
+ "goto",
+ "range",
+ "fallthrough",
+ "package",
+ "map",
+ "func",
+ "defer",
+ "import",
+ "default",
+ "var",
+ "type",
+ "break",
+ "return",
+ "", "",
+ "interface",
+ "",
+ "select",
+ "", "",
+ "else",
+ "",
+ "struct",
+ "", "",
+ "chan",
+ "",
+ "switch"
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register const char *s = wordlist[key];
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_go.gperf b/llamafile/is_keyword_go.gperf
new file mode 100644
index 00000000..981b3d23
--- /dev/null
+++ b/llamafile/is_keyword_go.gperf
@@ -0,0 +1,33 @@
+%{
+#include <string.h>
+%}
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_go
+%%
+break
+case
+chan
+const
+continue
+default
+defer
+else
+fallthrough
+for
+func
+go
+goto
+if
+import
+interface
+map
+package
+range
+return
+select
+struct
+switch
+type
+var
diff --git a/llamafile/is_keyword_go_type.c b/llamafile/is_keyword_go_type.c
new file mode 100644
index 00000000..707fdad5
--- /dev/null
+++ b/llamafile/is_keyword_go_type.c
@@ -0,0 +1,127 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf is_keyword_go_type.gperf */
+/* Computed positions: -k'1,$' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "is_keyword_go_type.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 17
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 10
+#define MIN_HASH_VALUE 4
+#define MAX_HASH_VALUE 26
+/* maximum key range = 23, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 10, 27, 5, 27, 20, 27, 0, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 5, 10,
+ 27, 10, 0, 27, 27, 0, 27, 27, 27, 27,
+ 27, 27, 27, 27, 0, 27, 5, 0, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27
+ };
+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_go_type (register const char *str, register size_t len)
+{
+ static const char * const wordlist[] =
+ {
+ "", "", "", "",
+ "int8",
+ "uint8",
+ "",
+ "uintptr",
+ "int",
+ "uint",
+ "int64",
+ "uint64",
+ "float64",
+ "",
+ "rune",
+ "int32",
+ "uint32",
+ "float32",
+ "",
+ "byte",
+ "complex128",
+ "", "", "",
+ "complex64",
+ "int16",
+ "uint16"
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register const char *s = wordlist[key];
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/is_keyword_go_type.gperf
new file mode 100644
index 00000000..2f6eac5b
--- /dev/null
+++ b/llamafile/is_keyword_go_type.gperf
@@ -0,0 +1,25 @@
+%{
+#include <string.h>
+%}
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_go_type
+%%
+byte
+complex128
+complex64
+float32
+float64
+int
+int16
+int32
+int64
+int8
+rune
+uint
+uint16
+uint32
+uint64
+uint8
+uintptr