Skip to content

Distribute and run LLMs with a single file.

Add shell syntax highlighting

Justine Tunney3e1942c15ab3
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index babfe378..8960e6a9 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -68,6 +68,8 @@ Highlight *Highlight::create(const std::string_view &lang) {
return new HighlightHaskell;
} else if (lang == "perl" || lang == "pl") {
return new HighlightPerl;
+ } else if (lang == "shell" || lang == "bash" || lang == "sh") {
+ return new HighlightShell;
} else {
return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 7bc3fc7a..c37fa206 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -67,6 +67,8 @@ is_keyword_f is_keyword_lisp;
is_keyword_f is_keyword_ada;
is_keyword_f is_keyword_haskell;
is_keyword_f is_keyword_perl;
+is_keyword_f is_keyword_shell;
+is_keyword_f is_keyword_shell_builtin;
}
class Highlight {
@@ -288,3 +290,15 @@ class HighlightPerl : public Highlight {
int t_ = 0;
std::string word_;
};
+
+class HighlightShell : public Highlight {
+ public:
+ HighlightShell();
+ ~HighlightShell() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cpp
index 1a382328..89d462dd 100644
--- a/llamafile/highlight_perl.cpp
+++ b/llamafile/highlight_perl.cpp
@@ -76,7 +76,6 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
}
break;
- Word:
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
word_ += c;
diff --git a/llamafile/highlight_php.cpp b/llamafile/highlight_php.cpp
index 9e0bc221..3a86e986 100644
--- a/llamafile/highlight_php.cpp
+++ b/llamafile/highlight_php.cpp
@@ -73,7 +73,6 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {
}
break;
- Word:
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
word_ += c;
diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cpp
new file mode 100644
index 00000000..1a1c6b04
--- /dev/null
+++ b/llamafile/highlight_shell.cpp
@@ -0,0 +1,212 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+#define NORMAL 0
+#define WORD 1
+#define QUOTE 2
+#define DQUOTE 4
+#define DQUOTE_BACKSLASH 5
+#define TICK 6
+#define TICK_BACKSLASH 7
+#define VAR 8
+#define VAR2 9
+#define COMMENT 10
+
+HighlightShell::HighlightShell() {
+}
+
+HighlightShell::~HighlightShell() {
+}
+
+void HighlightShell::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '`') {
+ t_ = TICK;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '$') {
+ *r += c;
+ t_ = VAR;
+ } else if (c == '#') {
+ *r += HI_COMMENT;
+ *r += c;
+ t_ = COMMENT;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_shell(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case VAR:
+ if (c == '#' || //
+ c == '*' || //
+ c == '-' || //
+ c == '?' || //
+ c == '@' || //
+ c == '!' || //
+ c == '$') {
+ *r += HI_VAR;
+ *r += c;
+ *r += HI_RESET;
+ t_ = NORMAL;
+ break;
+ } else if (c == '{') {
+ *r += c;
+ *r += HI_VAR;
+ t_ = VAR2;
+ break;
+ } else {
+ *r += HI_VAR;
+ t_ = VAR2;
+ }
+ // fallthrough
+
+ case VAR2:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case COMMENT:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE;
+ break;
+
+ case TICK:
+ *r += c;
+ if (c == '`') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = TICK_BACKSLASH;
+ }
+ break;
+
+ case TICK_BACKSLASH:
+ *r += c;
+ t_ = TICK;
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightShell::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_shell(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case VAR2:
+ case TICK:
+ case TICK_BACKSLASH:
+ case QUOTE:
+ case DQUOTE:
+ case DQUOTE_BACKSLASH:
+ case COMMENT:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index 4781e7de..23eb482e 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -52,6 +52,7 @@ const char *const kLanguages[] = {
"plain", //
"python", //
"rust", //
+ "shell", //
"sql", //
};
diff --git a/llamafile/is_keyword_shell.c b/llamafile/is_keyword_shell.c
new file mode 100644
index 00000000..6668a4ed
--- /dev/null
+++ b/llamafile/is_keyword_shell.c
@@ -0,0 +1,149 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_shell.gperf */
+/* Computed positions: -k'2-3' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_shell.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 22
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 8
+#define MIN_HASH_VALUE 2
+#define MAX_HASH_VALUE 39
+/* maximum key range = 38, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 10, 25, 40,
+ 40, 5, 15, 40, 20, 10, 5, 40, 5, 5,
+ 5, 0, 40, 0, 5, 0, 13, 0, 40, 40,
+ 5, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 40, 40, 40, 40, 40
+ };
+ register unsigned int hval = len;
+
+ switch (hval)
+ {
+ default:
+ hval += asso_values[(unsigned char)str[2]+1];
+ /*FALLTHROUGH*/
+ case 2:
+ hval += asso_values[(unsigned char)str[1]];
+ break;
+ }
+ return hval;
+}
+
+const char *
+is_keyword_shell (register const char *str, register size_t len)
+{
+ static const char * const wordlist[] =
+ {
+ "", "",
+ "do",
+ "for",
+ "done",
+ "",
+ "coproc",
+ "in",
+ "function",
+ "expr",
+ "until",
+ "return",
+ "fi",
+ "",
+ "elif",
+ "",
+ "select",
+ "if",
+ "",
+ "time",
+ "", "",
+ "else",
+ "",
+ "exec",
+ "break",
+ "",
+ "case",
+ "",
+ "esac",
+ "while",
+ "", "", "",
+ "trap",
+ "", "", "", "",
+ "then"
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register const char *s = wordlist[key];
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_shell.gperf b/llamafile/is_keyword_shell.gperf
new file mode 100644
index 00000000..e4771ad5
--- /dev/null
+++ b/llamafile/is_keyword_shell.gperf
@@ -0,0 +1,30 @@
+%{
+#include <string.h>
+%}
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_shell
+%%
+break
+case
+coproc
+do
+done
+elif
+else
+esac
+exec
+expr
+fi
+for
+function
+if
+in
+return
+select
+then
+time
+trap
+until
+while
diff --git a/llamafile/is_keyword_shell_builtin.c b/llamafile/is_keyword_shell_builtin.c
new file mode 100644
index 00000000..4682a5f0
--- /dev/null
+++ b/llamafile/is_keyword_shell_builtin.c
@@ -0,0 +1,165 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_shell_builtin.gperf */
+/* Computed positions: -k'1-2' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_shell_builtin.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 41
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 9
+#define MIN_HASH_VALUE 4
+#define MAX_HASH_VALUE 69
+/* maximum key range = 66, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 25, 25, 5,
+ 45, 5, 15, 10, 10, 20, 1, 45, 5, 30,
+ 15, 0, 5, 70, 20, 0, 0, 10, 55, 30,
+ 45, 40, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70, 70, 70, 70, 70,
+ 70, 70, 70, 70, 70, 70
+ };
+ return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_shell_builtin (register const char *str, register size_t len)
+{
+ static const char * const wordlist[] =
+ {
+ "", "", "", "",
+ "stop",
+ "jobs",
+ "source",
+ "",
+ "set",
+ "test",
+ "local",
+ "logout",
+ "command",
+ "let",
+ "echo",
+ "shift",
+ "",
+ "suspend",
+ "",
+ "help",
+ "chdir",
+ "ulimit",
+ "getopts",
+ "",
+ "true",
+ "times",
+ "enable",
+ "fg",
+ "",
+ "read",
+ "unset",
+ "printf",
+ "unalias",
+ "",
+ "readarray",
+ "alias",
+ "caller",
+ "bg",
+ "",
+ "hash",
+ "", "",
+ "builtin",
+ "",
+ "type",
+ "false",
+ "",
+ "typeset",
+ "",
+ "bind",
+ "", "",
+ "cd",
+ "",
+ "exit",
+ "", "",
+ "declare",
+ "",
+ "wait",
+ "", "",
+ "mapfile",
+ "",
+ "eval",
+ "", "", "", "",
+ "kill"
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register const char *s = wordlist[key];
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_shell_builtin.gperf b/llamafile/is_keyword_shell_builtin.gperf
new file mode 100644
index 00000000..520cdf8a
--- /dev/null
+++ b/llamafile/is_keyword_shell_builtin.gperf
@@ -0,0 +1,49 @@
+%{
+#include <string.h>
+%}
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_shell_builtin
+%%
+alias
+bg
+bind
+builtin
+caller
+cd
+chdir
+command
+declare
+echo
+enable
+eval
+exit
+false
+fg
+getopts
+hash
+help
+jobs
+kill
+let
+local
+logout
+mapfile
+printf
+read
+readarray
+set
+shift
+source
+stop
+suspend
+test
+times
+true
+type
+typeset
+ulimit
+unalias
+unset
+wait