Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make more highlight improvements
Justine Tunneyebcd61c1cc3c
diff --git a/llamafile/high.cpp b/llamafile/high.cppindex 9c2e9d78..3fe1cd0c 100644--- a/llamafile/high.cpp+++ b/llamafile/high.cpp@@ -48,40 +48,7 @@ static std::string extname(const std::string_view path) {return std::string(name.substr(dot_pos + 1));}-int main(int argc, char *argv[]) {-- // process flags- int opt;- int infd = 0;- int outfd = 1;- const char *lang = nullptr;- const char *inpath = nullptr;- while ((opt = getopt(argc, argv, "hl:o:")) != -1) {- switch (opt) {- case 'h':- printf("usage: %s [-l LANG] [-o OUTFILE] [INFILE]\n", argv[0]);- exit(0);- case 'l':- lang = optarg;- break;- case 'o':- if ((outfd = creat(optarg, 0644)) == -1) {- perror(optarg);- exit(1);- }- break;- default:- exit(1);- }- }- if (optind < argc) {- inpath = argv[optind];- if ((infd = open(inpath, O_RDONLY)) == -1) {- perror(inpath);- exit(1);- }- }-+static void highlight(int infd, int outfd, const char *lang, const char *inpath) {// create syntax highlighterHighlight *h;const char *ext;@@ -123,3 +90,44 @@ int main(int argc, char *argv[]) {H.flush(&res);write(outfd, res.data(), res.size());}++int main(int argc, char *argv[]) {++ // process flags+ int opt;+ int outfd = 1;+ const char *lang = nullptr;+ while ((opt = getopt(argc, argv, "hl:o:")) != -1) {+ switch (opt) {+ case 'h':+ printf("usage: %s [-l LANG] [-o OUTFILE] [INFILE]\n", argv[0]);+ exit(0);+ case 'l':+ lang = optarg;+ break;+ case 'o':+ if ((outfd = creat(optarg, 0644)) == -1) {+ perror(optarg);+ exit(1);+ }+ break;+ default:+ exit(1);+ }+ }++ // process files+ if (optind == argc) {+ highlight(0, outfd, lang, 0);+ } else {+ for (int i = optind; i < argc; ++i) {+ int infd;+ const char *inpath = argv[i];+ if ((infd = open(inpath, O_RDONLY)) == -1) {+ perror(inpath);+ exit(1);+ }+ highlight(infd, outfd, lang, inpath);+ }+ }+}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 54d2e286..35957808 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -471,8 +471,11 @@ class HighlightShell : public Highlight {int u_ = 0;int t_ = 0;int i_ = 0;+ int curl_ = 0;+ int last_ = 0;bool pending_heredoc_ = false;bool indented_heredoc_ = false;+ bool no_interpolation_ = false;std::string word_;std::string heredoc_;};diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cppindex 79ec29fc..b86e13a1 100644--- a/llamafile/highlight_shell.cpp+++ b/llamafile/highlight_shell.cpp@@ -25,11 +25,17 @@ enum {WORD,QUOTE,DQUOTE,+ DQUOTE_VAR,+ DQUOTE_VAR2,+ DQUOTE_CURL,+ DQUOTE_CURL_BACKSLASH,DQUOTE_BACKSLASH,TICK,TICK_BACKSLASH,VAR,VAR2,+ CURL,+ CURL_BACKSLASH,COMMENT,LT,LT_LT,@@ -37,6 +43,10 @@ enum {LT_LT_QNAME,HEREDOC_BOL,HEREDOC,+ HEREDOC_VAR,+ HEREDOC_VAR2,+ HEREDOC_CURL,+ HEREDOC_CURL_BACKSLASH,BACKSLASH,};@@ -50,9 +60,10 @@ void HighlightShell::feed(std::string *r, std::string_view input) {for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;+ last_ = c_;if (!u_) {if (b < 0300) {- c = b;+ c_ = c = b;} else {c_ = ThomPikeByte(b);u_ = ThomPikeLen(b) - 1;@@ -97,7 +108,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {} else if (c == '<') {t_ = LT;*r += '<';- } else if (c == '#') {+ } else if (c == '#' && (!last_ || isspace(last_))) {*r += HI_COMMENT;*r += '#';t_ = COMMENT;@@ -157,9 +168,10 @@ void HighlightShell::feed(std::string *r, std::string_view input) {t_ = NORMAL;break;} else if (c == '{') {- append_wchar(r, c);+ *r += '{';*r += HI_VAR;- t_ = VAR2;+ t_ = CURL;+ curl_ = 1;break;} else {*r += HI_VAR;@@ -177,6 +189,37 @@ void HighlightShell::feed(std::string *r, std::string_view input) {}break;+ case CURL:+ if (c == '\\') {+ t_ = CURL_BACKSLASH;+ *r += HI_RESET;+ *r += HI_ESCAPE;+ *r += '\\';+ } else if (c == '{') {+ *r += HI_RESET;+ *r += '{';+ *r += HI_VAR;+ ++curl_;+ } else if (c == '}') {+ *r += HI_RESET;+ *r += '}';+ if (!--curl_) {+ t_ = NORMAL;+ }+ } else if (ispunct(c)) {+ *r += HI_RESET;+ append_wchar(r, c);+ } else {+ append_wchar(r, c);+ }+ break;++ case CURL_BACKSLASH:+ append_wchar(r, c);+ *r += HI_RESET;+ t_ = CURL;+ break;+case COMMENT:append_wchar(r, c);if (c == '\n') {@@ -193,13 +236,19 @@ void HighlightShell::feed(std::string *r, std::string_view input) {}break;+ Dquote:case DQUOTE:- append_wchar(r, c);if (c == '"') {+ append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;} else if (c == '\\') {+ append_wchar(r, c);t_ = DQUOTE_BACKSLASH;+ } else if (c == '$') {+ t_ = DQUOTE_VAR;+ } else {+ append_wchar(r, c);}break;@@ -208,6 +257,73 @@ void HighlightShell::feed(std::string *r, std::string_view input) {t_ = DQUOTE;break;+ case DQUOTE_VAR:+ if (c == '!' || //+ c == '#' || //+ c == '$' || //+ c == '*' || //+ c == '-' || //+ c == '?' || //+ c == '@' || //+ c == '\\' || //+ c == '^') {+ *r += HI_BOLD;+ *r += '$';+ append_wchar(r, c);+ *r += HI_UNBOLD;+ t_ = DQUOTE;+ break;+ } else if (c == '{') {+ *r += HI_BOLD;+ *r += "${";+ t_ = DQUOTE_CURL;+ curl_ = 1;+ break;+ } else if (c == '(') {+ *r += '$';+ t_ = DQUOTE_VAR2;+ } else {+ *r += HI_BOLD;+ *r += '$';+ t_ = DQUOTE_VAR2;+ }+ // fallthrough++ case DQUOTE_VAR2:+ if (!isascii(c) || isalnum(c) || c == '_') {+ append_wchar(r, c);+ } else {+ *r += HI_UNBOLD;+ t_ = DQUOTE;+ goto Dquote;+ }+ break;++ case DQUOTE_CURL:+ if (c == '\\') {+ t_ = DQUOTE_CURL_BACKSLASH;+ *r += '\\';+ } else if (c == '{') {+ *r += '{';+ ++curl_;+ } else if (c == '}') {+ *r += '}';+ if (!--curl_) {+ *r += HI_UNBOLD;+ t_ = DQUOTE;+ }+ } else if (ispunct(c)) {+ append_wchar(r, c);+ } else {+ append_wchar(r, c);+ }+ break;++ case DQUOTE_CURL_BACKSLASH:+ append_wchar(r, c);+ t_ = DQUOTE_CURL;+ break;+case TICK:append_wchar(r, c);if (c == '`') {@@ -230,6 +346,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {heredoc_.clear();pending_heredoc_ = false;indented_heredoc_ = false;+ no_interpolation_ = false;} else {t_ = NORMAL;goto Normal;@@ -246,6 +363,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {t_ = LT_LT_QNAME;*r += HI_STRING;append_wchar(r, c);+ no_interpolation_ = true;} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;append_wchar(&heredoc_, c);@@ -302,10 +420,83 @@ void HighlightShell::feed(std::string *r, std::string_view input) {}break;+ Heredoc:case HEREDOC:- append_wchar(r, c);- if (c == '\n')+ if (c == '\n') {+ *r += '\n';t_ = HEREDOC_BOL;+ } else if (c == '$' && !no_interpolation_) {+ t_ = HEREDOC_VAR;+ } else {+ append_wchar(r, c);+ }+ break;++ case HEREDOC_VAR:+ if (c == '!' || //+ c == '#' || //+ c == '$' || //+ c == '*' || //+ c == '-' || //+ c == '?' || //+ c == '@' || //+ c == '\\' || //+ c == '^') {+ *r += HI_BOLD;+ *r += '$';+ append_wchar(r, c);+ *r += HI_UNBOLD;+ t_ = HEREDOC;+ break;+ } else if (c == '{') {+ *r += HI_BOLD;+ *r += "${";+ t_ = HEREDOC_CURL;+ curl_ = 1;+ break;+ } else if (c == '(') {+ *r += '$';+ t_ = HEREDOC_VAR2;+ } else {+ *r += HI_BOLD;+ *r += '$';+ t_ = HEREDOC_VAR2;+ }+ // fallthrough++ case HEREDOC_VAR2:+ if (!isascii(c) || isalnum(c) || c == '_') {+ append_wchar(r, c);+ } else {+ *r += HI_UNBOLD;+ t_ = HEREDOC;+ goto Heredoc;+ }+ break;++ case HEREDOC_CURL:+ if (c == '\\') {+ t_ = HEREDOC_CURL_BACKSLASH;+ *r += '\\';+ } else if (c == '{') {+ *r += '{';+ ++curl_;+ } else if (c == '}') {+ *r += '}';+ if (!--curl_) {+ *r += HI_UNBOLD;+ t_ = HEREDOC;+ }+ } else if (ispunct(c)) {+ append_wchar(r, c);+ } else {+ append_wchar(r, c);+ }+ break;++ case HEREDOC_CURL_BACKSLASH:+ append_wchar(r, c);+ t_ = HEREDOC_CURL;break;default:@@ -330,15 +521,31 @@ void HighlightShell::flush(std::string *r) {}word_.clear();break;+ case DQUOTE_VAR:+ *r += '$';+ *r += HI_RESET;+ break;+ case HEREDOC_VAR:+ *r += '$';+ *r += HI_RESET;+ break;case VAR2:+ case CURL:+ case CURL_BACKSLASH:case TICK:case TICK_BACKSLASH:case QUOTE:case DQUOTE:+ case DQUOTE_VAR2:+ case DQUOTE_CURL:+ case DQUOTE_CURL_BACKSLASH:case DQUOTE_BACKSLASH:case COMMENT:case HEREDOC_BOL:case HEREDOC:+ case HEREDOC_VAR2:+ case HEREDOC_CURL:+ case HEREDOC_CURL_BACKSLASH:case LT_LT_QNAME:case BACKSLASH:*r += HI_RESET;@@ -349,4 +556,5 @@ void HighlightShell::flush(std::string *r) {c_ = 0;u_ = 0;t_ = NORMAL;+ last_ = 0;}diff --git a/llamafile/highlight_typescript.cpp b/llamafile/highlight_typescript.cppindex 5680e76b..0bdde0be 100644--- a/llamafile/highlight_typescript.cpp+++ b/llamafile/highlight_typescript.cpp@@ -259,9 +259,9 @@ void HighlightTypescript::feed(std::string *r, std::string_view input) {case TICK_DOLLAR:if (c == '{' && nesti_ < sizeof(nest_)) {- // this is how the typescript playground highlights- *r += HI_RESET;+ *r += HI_BOLD;*r += '$';+ *r += HI_UNBOLD;*r += HI_STRING;*r += '{';*r += HI_RESET;diff --git a/llamafile/is_keyword_typescript.gperf b/llamafile/is_keyword_typescript.gperfindex 99f7426d..c9b3f30c 100644--- a/llamafile/is_keyword_typescript.gperf+++ b/llamafile/is_keyword_typescript.gperf@@ -54,6 +54,7 @@ setstaticswitchtarget+thisthrowtrytype