Skip to content

Distribute and run LLMs with a single file.

Add syntax highlighting for Rust

Justine Tunneyb63cebf418fa
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 93f9546a..50401f07 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -30,6 +30,8 @@ Highlight *Highlight::create(const std::string_view &lang) {
return new HighlightC(is_keyword_java);
} else if (lang == "py" || lang == "python") {
return new HighlightPython();
+ } else if (lang == "rs" || lang == "rust") {
+ return new HighlightRust();
} else {
return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index f71210d1..5bb06417 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -19,11 +19,13 @@
#include <string>
#include <string_view>
-#define RESET "\033[0m"
-#define BOLDE "\033[1m"
-#define KEYWORD "\033[1;34m" // bold blue
-#define STRING "\033[0;32m" // green
-#define COMMENT "\033[0;31m" // red
+#define HI_RESET "\033[0m"
+#define HI_BOLD "\033[1m"
+#define HI_KEYWORD "\033[1;34m" // bold blue
+#define HI_STRING "\033[32m" // green
+#define HI_COMMENT "\033[31m" // red
+#define HI_MACRO "\033[35m" // magenta
+#define HI_ATTRIB "\033[35m" // magenta
typedef const char *is_keyword_f(const char *, size_t);
@@ -33,6 +35,7 @@ is_keyword_f is_keyword_cxx;
is_keyword_f is_keyword_js;
is_keyword_f is_keyword_java;
is_keyword_f is_keyword_python;
+is_keyword_f is_keyword_rust;
}
class Highlight {
@@ -88,3 +91,16 @@ class HighlightMarkdown : public Highlight {
std::string lang_;
Highlight *highlighter_ = nullptr;
};
+
+class HighlightRust : public Highlight {
+ public:
+ HighlightRust();
+ ~HighlightRust() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ int nest_ = 0;
+ std::string word_;
+};
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cpp
index f0c02108..58e42f5d 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight_c.cpp
@@ -61,11 +61,11 @@ void HighlightC::feed(std::string *r, std::string_view input) {
t_ = SLASH;
} else if (c == '\'') {
t_ = QUOTE;
- *r += STRING;
+ *r += HI_STRING;
*r += c;
} else if (c == '"') {
t_ = DQUOTE;
- *r += STRING;
+ *r += HI_STRING;
*r += c;
} else {
*r += c;
@@ -78,9 +78,9 @@ void HighlightC::feed(std::string *r, std::string_view input) {
word_ += c;
} else {
if (is_keyword_(word_.data(), word_.size())) {
- *r += KEYWORD;
+ *r += HI_KEYWORD;
*r += word_;
- *r += RESET;
+ *r += HI_RESET;
} else {
*r += word_;
}
@@ -92,11 +92,11 @@ void HighlightC::feed(std::string *r, std::string_view input) {
case SLASH:
if (c == '/') {
- *r += COMMENT;
+ *r += HI_COMMENT;
*r += "//";
t_ = SLASH_SLASH;
} else if (c == '*') {
- *r += COMMENT;
+ *r += HI_COMMENT;
*r += "/*";
t_ = SLASH_STAR;
} else {
@@ -108,7 +108,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
case SLASH_SLASH:
if (c == '\n') {
- *r += RESET;
+ *r += HI_RESET;
*r += c;
t_ = NORMAL;
} else {
@@ -125,7 +125,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
case SLASH_STAR_STAR:
*r += c;
if (c == '/') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
} else {
t_ = SLASH_STAR;
@@ -135,7 +135,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
case QUOTE:
*r += c;
if (c == '\'') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
}
break;
@@ -143,7 +143,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
case DQUOTE:
*r += c;
if (c == '"') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
}
break;
@@ -158,9 +158,9 @@ void HighlightC::flush(std::string *r) {
switch (t_) {
case WORD:
if (is_keyword_(word_.data(), word_.size())) {
- *r += KEYWORD;
+ *r += HI_KEYWORD;
*r += word_;
- *r += RESET;
+ *r += HI_RESET;
} else {
*r += word_;
}
@@ -174,7 +174,7 @@ void HighlightC::flush(std::string *r) {
case SLASH_SLASH:
case SLASH_STAR:
case SLASH_STAR_STAR:
- *r += RESET;
+ *r += HI_RESET;
break;
default:
break;
diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight_c_test.cpp
index 3b336dbe..5ca13e43 100644
--- a/llamafile/highlight_c_test.cpp
+++ b/llamafile/highlight_c_test.cpp
@@ -25,49 +25,53 @@ using namespace std::string_literals;
HighlightC g_highlight;
void test(std::string input, std::string expect) {
- std::string got;
- g_highlight.feed(&got, input);
- if (got != expect) {
- fprintf(stderr,
- "error: highlighting failed\n"
- "\t input: %`'s\n"
- "\texpected: %`'s\n"
- "\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
- exit(1);
- }
+ std::string got;
+ g_highlight.feed(&got, input);
+ if (got != expect) {
+ fprintf(stderr,
+ "error: highlighting failed\n"
+ "\t input: %`'s\n"
+ "\texpected: %`'s\n"
+ "\t got: %`'s\n",
+ input.c_str(), expect.c_str(), got.c_str());
+ exit(1);
+ }
}
void testf(std::string input, std::string expect) {
- std::string got;
- g_highlight.feed(&got, input);
- g_highlight.flush(&got);
- if (got != expect) {
- fprintf(stderr,
- "error: highlighting failed\n"
- "\t input: %`'s\n"
- "\texpected: %`'s\n"
- "\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
- exit(1);
- }
+ std::string got;
+ g_highlight.feed(&got, input);
+ g_highlight.flush(&got);
+ if (got != expect) {
+ fprintf(stderr,
+ "error: highlighting failed\n"
+ "\t input: %`'s\n"
+ "\texpected: %`'s\n"
+ "\t got: %`'s\n",
+ input.c_str(), expect.c_str(), got.c_str());
+ exit(1);
+ }
}
int main(int argc, char *argv[]) {
- test("", "");
- test("vola", "");
- test("tile ", ""s + KEYWORD + "volatile" + RESET + " ");
- test("yo // hi\nthere ", "yo "s + COMMENT + "// hi" + RESET + "\nthere ");
- testf("// sup", ""s + COMMENT + "// sup" + RESET);
- test("yo /* hi */ there ", "yo "s + COMMENT + "/* hi */" + RESET + " there ");
- testf("yo /* hi */ there", "yo "s + COMMENT + "/* hi */" + RESET + " there");
- testf("yo /* h*i */ there", "yo "s + COMMENT + "/* h*i */" + RESET + " there");
- testf("vola", "vola");
- testf("volatile", ""s + KEYWORD + "volatile" + RESET);
- testf("/hi/", "/hi/");
- testf("x '' y", "x "s + STRING + "''" + RESET + " y");
- testf("x 'c' y", "x "s + STRING + "'c'" + RESET + " y");
- testf("x '\\'' y", "x "s + STRING + "'\\''" + RESET + " y");
- testf("x \"c\" y", "x "s + STRING + "\"c\"" + RESET + " y");
- testf("x \"\\\"\" y", "x "s + STRING + "\"\\\"\"" + RESET + " y");
+ test("", "");
+ test("vola", "");
+ test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");
+ test("yo // hi\nthere ",
+ "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");
+ testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);
+ test("yo /* hi */ there ",
+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");
+ testf("yo /* hi */ there",
+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");
+ testf("yo /* h*i */ there",
+ "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");
+ testf("vola", "vola");
+ testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);
+ testf("/hi/", "/hi/");
+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");
+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");
+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");
+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");
+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");
}
diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cpp
index c77b6611..63f3c9e3 100644
--- a/llamafile/highlight_markdown.cpp
+++ b/llamafile/highlight_markdown.cpp
@@ -64,7 +64,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
case STAR:
if (c == '*') {
t_ = STRONG;
- *r += BOLDE;
+ *r += HI_BOLD;
} else {
t_ = NORMAL;
}
@@ -82,7 +82,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
*r += c;
if (c == '*') {
t_ = NORMAL;
- *r += RESET;
+ *r += HI_RESET;
} else {
t_ = STRONG;
}
@@ -170,7 +170,7 @@ void HighlightMarkdown::flush(std::string *r) {
break;
case STRONG:
case STRONG_STAR:
- *r += RESET;
+ *r += HI_RESET;
break;
case CODE:
case CODE_TICK:
diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cpp
index ec0f4058..c59890d3 100644
--- a/llamafile/highlight_python.cpp
+++ b/llamafile/highlight_python.cpp
@@ -69,15 +69,15 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
goto Word;
} else if (c == '#') {
t_ = COM;
- *r += COMMENT;
+ *r += HI_COMMENT;
*r += c;
} else if (c == '\'') {
t_ = SQUOTE;
- *r += STRING;
+ *r += HI_STRING;
*r += c;
} else if (c == '"') {
t_ = DQUOTE;
- *r += STRING;
+ *r += HI_STRING;
*r += c;
} else {
*r += c;
@@ -90,9 +90,9 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
word_ += c;
} else {
if (is_keyword_python(word_.data(), word_.size())) {
- *r += KEYWORD;
+ *r += HI_KEYWORD;
*r += word_;
- *r += RESET;
+ *r += HI_RESET;
} else {
*r += word_;
}
@@ -104,7 +104,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
case COM:
if (c == '\n') {
- *r += RESET;
+ *r += HI_RESET;
*r += c;
t_ = NORMAL;
} else {
@@ -124,7 +124,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
case SQUOTESTR:
*r += c;
if (c == '\'') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
}
break;
@@ -135,7 +135,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
*r += c;
t_ = SQUOTE3;
} else {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
goto Normal;
}
@@ -156,7 +156,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
case SQUOTE32:
*r += c;
if (c == '\'') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
} else {
t_ = SQUOTE3;
@@ -175,7 +175,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
case DQUOTESTR:
*r += c;
if (c == '"') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
}
break;
@@ -186,7 +186,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
*r += c;
t_ = DQUOTE3;
} else {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
goto Normal;
}
@@ -207,7 +207,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
case DQUOTE32:
*r += c;
if (c == '"') {
- *r += RESET;
+ *r += HI_RESET;
t_ = NORMAL;
} else {
t_ = DQUOTE3;
@@ -224,9 +224,9 @@ void HighlightPython::flush(std::string *r) {
switch (t_) {
case WORD:
if (is_keyword_python(word_.data(), word_.size())) {
- *r += KEYWORD;
+ *r += HI_KEYWORD;
*r += word_;
- *r += RESET;
+ *r += HI_RESET;
} else {
*r += word_;
}
@@ -245,7 +245,7 @@ void HighlightPython::flush(std::string *r) {
case DQUOTE3:
case DQUOTE31:
case DQUOTE32:
- *r += RESET;
+ *r += HI_RESET;
break;
default:
break;
diff --git a/llamafile/highlight_python_test.cpp b/llamafile/highlight_python_test.cpp
index 7510ec46..143bb8b9 100644
--- a/llamafile/highlight_python_test.cpp
+++ b/llamafile/highlight_python_test.cpp
@@ -56,22 +56,22 @@ void testf(std::string input, std::string expect) {
int main(int argc, char *argv[]) {
test("", "");
test("d", "");
- test("ef ", ""s + KEYWORD + "def" + RESET + " ");
- test("yo # hi\nthere ", "yo "s + COMMENT + "# hi" + RESET + "\nthere ");
- testf("# sup", ""s + COMMENT + "# sup" + RESET);
+ test("ef ", ""s + HI_KEYWORD + "def" + HI_RESET + " ");
+ test("yo # hi\nthere ", "yo "s + HI_COMMENT + "# hi" + HI_RESET + "\nthere ");
+ testf("# sup", ""s + HI_COMMENT + "# sup" + HI_RESET);
testf("de", "de");
- testf("def", ""s + KEYWORD + "def" + RESET);
+ testf("def", ""s + HI_KEYWORD + "def" + HI_RESET);
testf("/hi/", "/hi/");
- testf("x '' y", "x "s + STRING + "''" + RESET + " y");
- testf("x '''''' y", "x "s + STRING + "''''''" + RESET + " y");
- testf("x '''xxx''' y", "x "s + STRING + "'''xxx'''" + RESET + " y");
- testf("x '''xx'x''' y", "x "s + STRING + "'''xx'x'''" + RESET + " y");
- testf("x '''xx''x''' y", "x "s + STRING + "'''xx''x'''" + RESET + " y");
- testf("x \"\" y", "x "s + STRING + "\"\"" + RESET + " y");
- testf("x \"\"\"\"\"\" y", "x "s + STRING + "\"\"\"\"\"\"" + RESET + " y");
- testf("x \"\"\"xxx\"\"\" y", "x "s + STRING + "\"\"\"xxx\"\"\"" + RESET + " y");
- testf("x 'c' y", "x "s + STRING + "'c'" + RESET + " y");
- testf("x '\\'' y", "x "s + STRING + "'\\''" + RESET + " y");
- testf("x \"c\" y", "x "s + STRING + "\"c\"" + RESET + " y");
- testf("x \"\\\"\" y", "x "s + STRING + "\"\\\"\"" + RESET + " y");
+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");
+ testf("x '''''' y", "x "s + HI_STRING + "''''''" + HI_RESET + " y");
+ testf("x '''xxx''' y", "x "s + HI_STRING + "'''xxx'''" + HI_RESET + " y");
+ testf("x '''xx'x''' y", "x "s + HI_STRING + "'''xx'x'''" + HI_RESET + " y");
+ testf("x '''xx''x''' y", "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");
+ testf("x \"\" y", "x "s + HI_STRING + "\"\"" + HI_RESET + " y");
+ testf("x \"\"\"\"\"\" y", "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");
+ testf("x \"\"\"xxx\"\"\" y", "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");
+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");
+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");
+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");
+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");
}
diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight_rust.cpp
new file mode 100644
index 00000000..5ccd85ca
--- /dev/null
+++ b/llamafile/highlight_rust.cpp
@@ -0,0 +1,262 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+#define NORMAL 0
+#define WORD 1
+#define QUOTE 2
+#define QUOTE2 3
+#define DQUOTE 4
+#define SLASH 5
+#define SLASH_SLASH 6
+#define SLASH_STAR 7
+#define SLASH_STAR_STAR 8
+#define HASH 9
+#define HASH_EXCLAIM 10
+#define ATTRIB 11
+#define BACKSLASH 64
+
+HighlightRust::HighlightRust() {
+}
+
+HighlightRust::~HighlightRust() {
+}
+
+void HighlightRust::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+
+ if (t_ & BACKSLASH) {
+ t_ &= ~BACKSLASH;
+ *r += c;
+ if (t_ == QUOTE)
+ t_ = QUOTE2;
+ continue;
+ } else if (c == '\\') {
+ *r += c;
+ t_ |= BACKSLASH;
+ continue;
+ }
+
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ goto Word;
+ } else if (c == '/') {
+ t_ = SLASH;
+ } else if (c == '#') {
+ t_ = HASH;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else {
+ *r += c;
+ }
+ break;
+
+ Word:
+ case WORD:
+ if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_' || c == '!') {
+ word_ += c;
+ } else {
+ if (is_keyword_rust(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (!word_.empty() && word_[word_.size() - 1] == '!') {
+ *r += HI_MACRO;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH:
+ if (c == '/') {
+ *r += HI_COMMENT;
+ *r += "//";
+ t_ = SLASH_SLASH;
+ } else if (c == '*') {
+ *r += HI_COMMENT;
+ *r += "/*";
+ t_ = SLASH_STAR;
+ } else {
+ *r += '/';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH_SLASH:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case SLASH_STAR:
+ *r += c;
+ if (c == '*')
+ t_ = SLASH_STAR_STAR;
+ break;
+
+ case SLASH_STAR_STAR:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ t_ = SLASH_STAR;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ t_ = QUOTE2;
+ }
+ break;
+
+ case QUOTE2:
+ if (c == '\'') {
+ *r += c;
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ }
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ break;
+
+ case HASH:
+ if (c == '!') {
+ t_ = HASH_EXCLAIM;
+ } else if (c == '[') {
+ *r += HI_ATTRIB;
+ *r += "#[";
+ t_ = ATTRIB;
+ } else {
+ *r += '#';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case HASH_EXCLAIM:
+ if (c == '[') {
+ *r += HI_ATTRIB;
+ *r += "#![";
+ t_ = ATTRIB;
+ } else {
+ *r += "#!";
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case ATTRIB:
+ *r += c;
+ if (c == '[') {
+ ++nest_;
+ } else if (c == ']') {
+ if (nest_) {
+ --nest_;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightRust::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_rust(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (!word_.empty() && word_[word_.size() - 1] == '!') {
+ *r += HI_MACRO;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case SLASH:
+ *r += '/';
+ break;
+ case QUOTE:
+ case QUOTE2:
+ case DQUOTE:
+ case ATTRIB:
+ case SLASH_SLASH:
+ case SLASH_STAR:
+ case SLASH_STAR_STAR:
+ *r += HI_RESET;
+ break;
+ case HASH:
+ *r += '#';
+ break;
+ case HASH_EXCLAIM:
+ *r += "#!";
+ break;
+ default:
+ break;
+ }
+ nest_ = 0;
+ t_ = NORMAL;
+}
diff --git a/llamafile/is_keyword_rust.c b/llamafile/is_keyword_rust.c
new file mode 100644
index 00000000..7225ba86
--- /dev/null
+++ b/llamafile/is_keyword_rust.c
@@ -0,0 +1,187 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf is_keyword_rust.gperf */
+/* Computed positions: -k'1-3' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "is_keyword_rust.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 52
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 8
+#define MIN_HASH_VALUE 2
+#define MAX_HASH_VALUE 85
+/* maximum key range = 84, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 10, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 0, 15, 40,
+ 40, 20, 10, 15, 10, 15, 15, 86, 25, 0,
+ 5, 5, 10, 35, 0, 0, 0, 0, 10, 55,
+ 5, 20, 5, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86, 86, 86, 86,
+ 86, 86, 86, 86, 86, 86, 86
+ };
+ register unsigned int hval = len;
+
+ switch (hval)
+ {
+ default:
+ hval += asso_values[(unsigned char)str[2]+1];
+ /*FALLTHROUGH*/
+ case 2:
+ hval += asso_values[(unsigned char)str[1]];
+ /*FALLTHROUGH*/
+ case 1:
+ hval += asso_values[(unsigned char)str[0]];
+ break;
+ }
+ return hval;
+}
+
+const char *
+is_keyword_rust (register const char *str, register size_t len)
+{
+ static const char * const wordlist[] =
+ {
+ "", "",
+ "as",
+ "mut",
+ "",
+ "match",
+ "struct",
+ "",
+ "try",
+ "",
+ "async",
+ "unsafe",
+ "unsized",
+ "use",
+ "true",
+ "false",
+ "",
+ "fn",
+ "for",
+ "",
+ "trait",
+ "static",
+ "in",
+ "abstract",
+ "self",
+ "union",
+ "return",
+ "if",
+ "mod",
+ "priv",
+ "break",
+ "extern",
+ "virtual",
+ "override",
+ "Self",
+ "final",
+ "", "",
+ "ref",
+ "enum",
+ "super",
+ "", "",
+ "box",
+ "loop",
+ "macro",
+ "",
+ "do",
+ "let",
+ "else",
+ "yield",
+ "", "",
+ "pub",
+ "impl",
+ "const",
+ "", "",
+ "continue",
+ "type",
+ "crate",
+ "typeof",
+ "", "",
+ "move",
+ "", "", "",
+ "dyn",
+ "", "", "", "", "", "",
+ "await",
+ "", "", "", "",
+ "where",
+ "become",
+ "", "", "",
+ "while"
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register const char *s = wordlist[key];
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_rust.gperf b/llamafile/is_keyword_rust.gperf
new file mode 100644
index 00000000..35841cf5
--- /dev/null
+++ b/llamafile/is_keyword_rust.gperf
@@ -0,0 +1,60 @@
+%{
+#include <string.h>
+%}
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_rust
+%%
+Self
+abstract
+as
+async
+await
+become
+box
+break
+const
+continue
+crate
+do
+dyn
+else
+enum
+extern
+false
+final
+fn
+for
+if
+impl
+in
+let
+loop
+macro
+match
+mod
+move
+mut
+override
+priv
+pub
+ref
+return
+self
+static
+struct
+super
+trait
+true
+try
+type
+typeof
+union
+unsafe
+unsized
+use
+virtual
+where
+while
+yield