Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add syntax highlighting for Rust
Justine Tunneyb63cebf418fa
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 93f9546a..50401f07 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -30,6 +30,8 @@ Highlight *Highlight::create(const std::string_view &lang) {return new HighlightC(is_keyword_java);} else if (lang == "py" || lang == "python") {return new HighlightPython();+ } else if (lang == "rs" || lang == "rust") {+ return new HighlightRust();} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex f71210d1..5bb06417 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -19,11 +19,13 @@#include <string>#include <string_view>-#define RESET "\033[0m"-#define BOLDE "\033[1m"-#define KEYWORD "\033[1;34m" // bold blue-#define STRING "\033[0;32m" // green-#define COMMENT "\033[0;31m" // red+#define HI_RESET "\033[0m"+#define HI_BOLD "\033[1m"+#define HI_KEYWORD "\033[1;34m" // bold blue+#define HI_STRING "\033[32m" // green+#define HI_COMMENT "\033[31m" // red+#define HI_MACRO "\033[35m" // magenta+#define HI_ATTRIB "\033[35m" // magentatypedef const char *is_keyword_f(const char *, size_t);@@ -33,6 +35,7 @@ is_keyword_f is_keyword_cxx;is_keyword_f is_keyword_js;is_keyword_f is_keyword_java;is_keyword_f is_keyword_python;+is_keyword_f is_keyword_rust;}class Highlight {@@ -88,3 +91,16 @@ class HighlightMarkdown : public Highlight {std::string lang_;Highlight *highlighter_ = nullptr;};++class HighlightRust : public Highlight {+ public:+ HighlightRust();+ ~HighlightRust() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ int nest_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cppindex f0c02108..58e42f5d 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight_c.cpp@@ -61,11 +61,11 @@ void HighlightC::feed(std::string *r, std::string_view input) {t_ = SLASH;} else if (c == '\'') {t_ = QUOTE;- *r += STRING;+ *r += HI_STRING;*r += c;} else if (c == '"') {t_ = DQUOTE;- *r += STRING;+ *r += HI_STRING;*r += c;} else {*r += c;@@ -78,9 +78,9 @@ void HighlightC::feed(std::string *r, std::string_view input) {word_ += c;} else {if (is_keyword_(word_.data(), word_.size())) {- *r += KEYWORD;+ *r += HI_KEYWORD;*r += word_;- *r += RESET;+ *r += HI_RESET;} else {*r += word_;}@@ -92,11 +92,11 @@ void HighlightC::feed(std::string *r, std::string_view input) {case SLASH:if (c == '/') {- *r += COMMENT;+ *r += HI_COMMENT;*r += "//";t_ = SLASH_SLASH;} else if (c == '*') {- *r += COMMENT;+ *r += HI_COMMENT;*r += "/*";t_ = SLASH_STAR;} else {@@ -108,7 +108,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {case SLASH_SLASH:if (c == '\n') {- *r += RESET;+ *r += HI_RESET;*r += c;t_ = NORMAL;} else {@@ -125,7 +125,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {case SLASH_STAR_STAR:*r += c;if (c == '/') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;} else {t_ = SLASH_STAR;@@ -135,7 +135,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {case QUOTE:*r += c;if (c == '\'') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;}break;@@ -143,7 +143,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {case DQUOTE:*r += c;if (c == '"') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;}break;@@ -158,9 +158,9 @@ void HighlightC::flush(std::string *r) {switch (t_) {case WORD:if (is_keyword_(word_.data(), word_.size())) {- *r += KEYWORD;+ *r += HI_KEYWORD;*r += word_;- *r += RESET;+ *r += HI_RESET;} else {*r += word_;}@@ -174,7 +174,7 @@ void HighlightC::flush(std::string *r) {case SLASH_SLASH:case SLASH_STAR:case SLASH_STAR_STAR:- *r += RESET;+ *r += HI_RESET;break;default:break;diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight_c_test.cppindex 3b336dbe..5ca13e43 100644--- a/llamafile/highlight_c_test.cpp+++ b/llamafile/highlight_c_test.cpp@@ -25,49 +25,53 @@ using namespace std::string_literals;HighlightC g_highlight;void test(std::string input, std::string expect) {- std::string got;- g_highlight.feed(&got, input);- if (got != expect) {- fprintf(stderr,- "error: highlighting failed\n"- "\t input: %`'s\n"- "\texpected: %`'s\n"- "\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());- exit(1);- }+ std::string got;+ g_highlight.feed(&got, input);+ if (got != expect) {+ fprintf(stderr,+ "error: highlighting failed\n"+ "\t input: %`'s\n"+ "\texpected: %`'s\n"+ "\t got: %`'s\n",+ input.c_str(), expect.c_str(), got.c_str());+ exit(1);+ }}void testf(std::string input, std::string expect) {- std::string got;- g_highlight.feed(&got, input);- g_highlight.flush(&got);- if (got != expect) {- fprintf(stderr,- "error: highlighting failed\n"- "\t input: %`'s\n"- "\texpected: %`'s\n"- "\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());- exit(1);- }+ std::string got;+ g_highlight.feed(&got, input);+ g_highlight.flush(&got);+ if (got != expect) {+ fprintf(stderr,+ "error: highlighting failed\n"+ "\t input: %`'s\n"+ "\texpected: %`'s\n"+ "\t got: %`'s\n",+ input.c_str(), expect.c_str(), got.c_str());+ exit(1);+ }}int main(int argc, char *argv[]) {- test("", "");- test("vola", "");- test("tile ", ""s + KEYWORD + "volatile" + RESET + " ");- test("yo // hi\nthere ", "yo "s + COMMENT + "// hi" + RESET + "\nthere ");- testf("// sup", ""s + COMMENT + "// sup" + RESET);- test("yo /* hi */ there ", "yo "s + COMMENT + "/* hi */" + RESET + " there ");- testf("yo /* hi */ there", "yo "s + COMMENT + "/* hi */" + RESET + " there");- testf("yo /* h*i */ there", "yo "s + COMMENT + "/* h*i */" + RESET + " there");- testf("vola", "vola");- testf("volatile", ""s + KEYWORD + "volatile" + RESET);- testf("/hi/", "/hi/");- testf("x '' y", "x "s + STRING + "''" + RESET + " y");- testf("x 'c' y", "x "s + STRING + "'c'" + RESET + " y");- testf("x '\\'' y", "x "s + STRING + "'\\''" + RESET + " y");- testf("x \"c\" y", "x "s + STRING + "\"c\"" + RESET + " y");- testf("x \"\\\"\" y", "x "s + STRING + "\"\\\"\"" + RESET + " y");+ test("", "");+ test("vola", "");+ test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");+ test("yo // hi\nthere ",+ "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");+ testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);+ test("yo /* hi */ there ",+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");+ testf("yo /* hi */ there",+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");+ testf("yo /* h*i */ there",+ "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");+ testf("vola", "vola");+ testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);+ testf("/hi/", "/hi/");+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");}diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cppindex c77b6611..63f3c9e3 100644--- a/llamafile/highlight_markdown.cpp+++ b/llamafile/highlight_markdown.cpp@@ -64,7 +64,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case STAR:if (c == '*') {t_ = STRONG;- *r += BOLDE;+ *r += HI_BOLD;} else {t_ = NORMAL;}@@ -82,7 +82,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {*r += c;if (c == '*') {t_ = NORMAL;- *r += RESET;+ *r += HI_RESET;} else {t_ = STRONG;}@@ -170,7 +170,7 @@ void HighlightMarkdown::flush(std::string *r) {break;case STRONG:case STRONG_STAR:- *r += RESET;+ *r += HI_RESET;break;case CODE:case CODE_TICK:diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cppindex ec0f4058..c59890d3 100644--- a/llamafile/highlight_python.cpp+++ b/llamafile/highlight_python.cpp@@ -69,15 +69,15 @@ void HighlightPython::feed(std::string *r, std::string_view input) {goto Word;} else if (c == '#') {t_ = COM;- *r += COMMENT;+ *r += HI_COMMENT;*r += c;} else if (c == '\'') {t_ = SQUOTE;- *r += STRING;+ *r += HI_STRING;*r += c;} else if (c == '"') {t_ = DQUOTE;- *r += STRING;+ *r += HI_STRING;*r += c;} else {*r += c;@@ -90,9 +90,9 @@ void HighlightPython::feed(std::string *r, std::string_view input) {word_ += c;} else {if (is_keyword_python(word_.data(), word_.size())) {- *r += KEYWORD;+ *r += HI_KEYWORD;*r += word_;- *r += RESET;+ *r += HI_RESET;} else {*r += word_;}@@ -104,7 +104,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {case COM:if (c == '\n') {- *r += RESET;+ *r += HI_RESET;*r += c;t_ = NORMAL;} else {@@ -124,7 +124,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {case SQUOTESTR:*r += c;if (c == '\'') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;}break;@@ -135,7 +135,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {*r += c;t_ = SQUOTE3;} else {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;goto Normal;}@@ -156,7 +156,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {case SQUOTE32:*r += c;if (c == '\'') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;} else {t_ = SQUOTE3;@@ -175,7 +175,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {case DQUOTESTR:*r += c;if (c == '"') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;}break;@@ -186,7 +186,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {*r += c;t_ = DQUOTE3;} else {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;goto Normal;}@@ -207,7 +207,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {case DQUOTE32:*r += c;if (c == '"') {- *r += RESET;+ *r += HI_RESET;t_ = NORMAL;} else {t_ = DQUOTE3;@@ -224,9 +224,9 @@ void HighlightPython::flush(std::string *r) {switch (t_) {case WORD:if (is_keyword_python(word_.data(), word_.size())) {- *r += KEYWORD;+ *r += HI_KEYWORD;*r += word_;- *r += RESET;+ *r += HI_RESET;} else {*r += word_;}@@ -245,7 +245,7 @@ void HighlightPython::flush(std::string *r) {case DQUOTE3:case DQUOTE31:case DQUOTE32:- *r += RESET;+ *r += HI_RESET;break;default:break;diff --git a/llamafile/highlight_python_test.cpp b/llamafile/highlight_python_test.cppindex 7510ec46..143bb8b9 100644--- a/llamafile/highlight_python_test.cpp+++ b/llamafile/highlight_python_test.cpp@@ -56,22 +56,22 @@ void testf(std::string input, std::string expect) {int main(int argc, char *argv[]) {test("", "");test("d", "");- test("ef ", ""s + KEYWORD + "def" + RESET + " ");- test("yo # hi\nthere ", "yo "s + COMMENT + "# hi" + RESET + "\nthere ");- testf("# sup", ""s + COMMENT + "# sup" + RESET);+ test("ef ", ""s + HI_KEYWORD + "def" + HI_RESET + " ");+ test("yo # hi\nthere ", "yo "s + HI_COMMENT + "# hi" + HI_RESET + "\nthere ");+ testf("# sup", ""s + HI_COMMENT + "# sup" + HI_RESET);testf("de", "de");- testf("def", ""s + KEYWORD + "def" + RESET);+ testf("def", ""s + HI_KEYWORD + "def" + HI_RESET);testf("/hi/", "/hi/");- testf("x '' y", "x "s + STRING + "''" + RESET + " y");- testf("x '''''' y", "x "s + STRING + "''''''" + RESET + " y");- testf("x '''xxx''' y", "x "s + STRING + "'''xxx'''" + RESET + " y");- testf("x '''xx'x''' y", "x "s + STRING + "'''xx'x'''" + RESET + " y");- testf("x '''xx''x''' y", "x "s + STRING + "'''xx''x'''" + RESET + " y");- testf("x \"\" y", "x "s + STRING + "\"\"" + RESET + " y");- testf("x \"\"\"\"\"\" y", "x "s + STRING + "\"\"\"\"\"\"" + RESET + " y");- testf("x \"\"\"xxx\"\"\" y", "x "s + STRING + "\"\"\"xxx\"\"\"" + RESET + " y");- testf("x 'c' y", "x "s + STRING + "'c'" + RESET + " y");- testf("x '\\'' y", "x "s + STRING + "'\\''" + RESET + " y");- testf("x \"c\" y", "x "s + STRING + "\"c\"" + RESET + " y");- testf("x \"\\\"\" y", "x "s + STRING + "\"\\\"\"" + RESET + " y");+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");+ testf("x '''''' y", "x "s + HI_STRING + "''''''" + HI_RESET + " y");+ testf("x '''xxx''' y", "x "s + HI_STRING + "'''xxx'''" + HI_RESET + " y");+ testf("x '''xx'x''' y", "x "s + HI_STRING + "'''xx'x'''" + HI_RESET + " y");+ testf("x '''xx''x''' y", "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");+ testf("x \"\" y", "x "s + HI_STRING + "\"\"" + HI_RESET + " y");+ testf("x \"\"\"\"\"\" y", "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");+ testf("x \"\"\"xxx\"\"\" y", "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");}diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight_rust.cppnew file mode 100644index 00000000..5ccd85ca--- /dev/null+++ b/llamafile/highlight_rust.cpp@@ -0,0 +1,262 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++#define NORMAL 0+#define WORD 1+#define QUOTE 2+#define QUOTE2 3+#define DQUOTE 4+#define SLASH 5+#define SLASH_SLASH 6+#define SLASH_STAR 7+#define SLASH_STAR_STAR 8+#define HASH 9+#define HASH_EXCLAIM 10+#define ATTRIB 11+#define BACKSLASH 64++HighlightRust::HighlightRust() {+}++HighlightRust::~HighlightRust() {+}++void HighlightRust::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;++ if (t_ & BACKSLASH) {+ t_ &= ~BACKSLASH;+ *r += c;+ if (t_ == QUOTE)+ t_ = QUOTE2;+ continue;+ } else if (c == '\\') {+ *r += c;+ t_ |= BACKSLASH;+ continue;+ }++ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ goto Word;+ } else if (c == '/') {+ t_ = SLASH;+ } else if (c == '#') {+ t_ = HASH;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_' || c == '!') {+ word_ += c;+ } else {+ if (is_keyword_rust(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (!word_.empty() && word_[word_.size() - 1] == '!') {+ *r += HI_MACRO;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH:+ if (c == '/') {+ *r += HI_COMMENT;+ *r += "//";+ t_ = SLASH_SLASH;+ } else if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ = SLASH_STAR;+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH_SLASH:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*')+ t_ = SLASH_STAR_STAR;+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ t_ = SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ t_ = QUOTE2;+ }+ break;++ case QUOTE2:+ if (c == '\'') {+ *r += c;+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ }+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case HASH:+ if (c == '!') {+ t_ = HASH_EXCLAIM;+ } else if (c == '[') {+ *r += HI_ATTRIB;+ *r += "#[";+ t_ = ATTRIB;+ } else {+ *r += '#';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case HASH_EXCLAIM:+ if (c == '[') {+ *r += HI_ATTRIB;+ *r += "#![";+ t_ = ATTRIB;+ } else {+ *r += "#!";+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case ATTRIB:+ *r += c;+ if (c == '[') {+ ++nest_;+ } else if (c == ']') {+ if (nest_) {+ --nest_;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightRust::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_rust(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (!word_.empty() && word_[word_.size() - 1] == '!') {+ *r += HI_MACRO;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case SLASH:+ *r += '/';+ break;+ case QUOTE:+ case QUOTE2:+ case DQUOTE:+ case ATTRIB:+ case SLASH_SLASH:+ case SLASH_STAR:+ case SLASH_STAR_STAR:+ *r += HI_RESET;+ break;+ case HASH:+ *r += '#';+ break;+ case HASH_EXCLAIM:+ *r += "#!";+ break;+ default:+ break;+ }+ nest_ = 0;+ t_ = NORMAL;+}diff --git a/llamafile/is_keyword_rust.c b/llamafile/is_keyword_rust.cnew file mode 100644index 00000000..7225ba86--- /dev/null+++ b/llamafile/is_keyword_rust.c@@ -0,0 +1,187 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf is_keyword_rust.gperf */+/* Computed positions: -k'1-3' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "is_keyword_rust.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 52+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 8+#define MIN_HASH_VALUE 2+#define MAX_HASH_VALUE 85+/* maximum key range = 84, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 10, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 0, 15, 40,+ 40, 20, 10, 15, 10, 15, 15, 86, 25, 0,+ 5, 5, 10, 35, 0, 0, 0, 0, 10, 55,+ 5, 20, 5, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,+ 86, 86, 86, 86, 86, 86, 86+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[2]+1];+ /*FALLTHROUGH*/+ case 2:+ hval += asso_values[(unsigned char)str[1]];+ /*FALLTHROUGH*/+ case 1:+ hval += asso_values[(unsigned char)str[0]];+ break;+ }+ return hval;+}++const char *+is_keyword_rust (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "",+ "as",+ "mut",+ "",+ "match",+ "struct",+ "",+ "try",+ "",+ "async",+ "unsafe",+ "unsized",+ "use",+ "true",+ "false",+ "",+ "fn",+ "for",+ "",+ "trait",+ "static",+ "in",+ "abstract",+ "self",+ "union",+ "return",+ "if",+ "mod",+ "priv",+ "break",+ "extern",+ "virtual",+ "override",+ "Self",+ "final",+ "", "",+ "ref",+ "enum",+ "super",+ "", "",+ "box",+ "loop",+ "macro",+ "",+ "do",+ "let",+ "else",+ "yield",+ "", "",+ "pub",+ "impl",+ "const",+ "", "",+ "continue",+ "type",+ "crate",+ "typeof",+ "", "",+ "move",+ "", "", "",+ "dyn",+ "", "", "", "", "", "",+ "await",+ "", "", "", "",+ "where",+ "become",+ "", "", "",+ "while"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_rust.gperf b/llamafile/is_keyword_rust.gperfnew file mode 100644index 00000000..35841cf5--- /dev/null+++ b/llamafile/is_keyword_rust.gperf@@ -0,0 +1,60 @@+%{+#include <string.h>+%}+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_rust+%%+Self+abstract+as+async+await+become+box+break+const+continue+crate+do+dyn+else+enum+extern+false+final+fn+for+if+impl+in+let+loop+macro+match+mod+move+mut+override+priv+pub+ref+return+self+static+struct+super+trait+true+try+type+typeof+union+unsafe+unsized+use+virtual+where+while+yield