Skip to content

Distribute and run LLMs with a single file.

Add TypeScript syntax highlighting

Justine Tunney7db4b98cadae
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 4adcb3e0..8c266bb4 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -175,7 +175,6 @@ o/$(MODE)/llamafile/highlight_c_test: \
o/$(MODE)/llamafile/highlight_c.o \
o/$(MODE)/llamafile/is_keyword_c.o \
o/$(MODE)/llamafile/is_keyword_c_type.o \
- o/$(MODE)/llamafile/is_keyword_js.o \
o/$(MODE)/llamafile/highlight_python_test: \
o/$(MODE)/llamafile/highlight_python_test.o \
diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cpp
index eecfe7d1..5105840d 100644
--- a/llamafile/chatbot.cpp
+++ b/llamafile/chatbot.cpp
@@ -304,6 +304,7 @@ int chatbot_main(int argc, char **argv) {
// perform important setup
HighlightMarkdown highlighter;
+ ColorBleeder bleeder(&highlighter);
struct llama_sampling_context *sampler = llama_sampling_init(params.sparams);
signal(SIGINT, on_sigint);
@@ -336,7 +337,7 @@ int chatbot_main(int argc, char **argv) {
if (llama_token_is_eog(g_model, id))
break;
std::string s;
- highlighter.feed(&s, llama_token_to_piece(g_ctx, id, params.special));
+ bleeder.feed(&s, llama_token_to_piece(g_ctx, id, params.special));
printf("%s", s.c_str());
fflush(stdout);
eval_id(id);
@@ -344,7 +345,7 @@ int chatbot_main(int argc, char **argv) {
g_got_sigint = 0;
free(line);
std::string s;
- highlighter.flush(&s);
+ bleeder.flush(&s);
printf("%s\n", s.c_str());
}
diff --git a/llamafile/color_bleeder.cpp b/llamafile/color_bleeder.cpp
new file mode 100644
index 00000000..d7ca5a52
--- /dev/null
+++ b/llamafile/color_bleeder.cpp
@@ -0,0 +1,160 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+// the `less` command was intentionally designed to prevent ansi sgr
+// codes from bleeding across lines. for example, saying:
+//
+// printf '\e[1m line 1 \n line 2 \e[0m \n'
+//
+// both lines will appear bold in the terminal. however with:
+//
+// printf '\e[1m line 1 \n line 2 \e[0m \n' | less
+//
+// only the first line will appear bold. this is because less has an
+// implicit \e[0m at the end of each line. this is bad, because our
+// highlighting code assumes that, when emitting things like comments on
+// multiple lines, we only need to set the color at the beginning.
+//
+// this class may be used to compose a Highlighter object. it'll relay
+// the emitted bytes through, taking special care to restore the ansi
+// sgr state at the beginning of each line. thus less will now work.
+//
+// only the subset of ansi escape codes that we use are supported.
+
+enum {
+ NORMAL,
+ ESC,
+ CSI,
+};
+
+ColorBleeder::ColorBleeder(Highlight *h) : h_(h) {
+}
+
+ColorBleeder::~ColorBleeder() {
+}
+
+void ColorBleeder::relay(std::string *r, const std::string &s) {
+ for (char c : s) {
+ *r += c;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ switch (c) {
+ case 033:
+ t_ = ESC;
+ break;
+ case '\n':
+ if (intensity_ || foreground_ || background_) {
+ bool got_some = false;
+ *r += "\033[";
+ if (intensity_) {
+ *r += std::to_string(intensity_);
+ got_some = true;
+ }
+ if (foreground_) {
+ if (got_some)
+ *r += ';';
+ *r += std::to_string(foreground_);
+ got_some = true;
+ }
+ if (background_) {
+ if (got_some)
+ *r += ';';
+ *r += std::to_string(background_);
+ }
+ *r += 'm';
+ }
+ break;
+ }
+ break;
+
+ case ESC:
+ switch (c) {
+ case '[':
+ t_ = CSI;
+ n_ = 0;
+ x_ = 0;
+ break;
+ default:
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case CSI:
+ if (isdigit(c)) {
+ x_ *= 10;
+ x_ += c - '0';
+ } else if (c == ';') {
+ if (n_ < sizeof(sgr_codes_)) {
+ sgr_codes_[n_++] = x_;
+ x_ = 0;
+ }
+ } else if (c == 'm') {
+ if (n_ < sizeof(sgr_codes_)) {
+ sgr_codes_[n_++] = x_;
+ x_ = 0;
+ }
+ for (int i = 0; i < n_; ++i) {
+ int g = sgr_codes_[i];
+ if (g == 0) {
+ intensity_ = 0;
+ foreground_ = 0;
+ background_ = 0;
+ } else if (g == 1 || g == 2) {
+ intensity_ = g;
+ } else if (g == 22) {
+ intensity_ = 0;
+ } else if ((30 <= g && g <= 37) || //
+ (90 <= g && g <= 97)) {
+ foreground_ = g;
+ } else if (g == 39) {
+ foreground_ = 0;
+ } else if ((40 <= g && g <= 47) || //
+ (100 <= g && g <= 107)) {
+ background_ = g;
+ } else if (g == 49) {
+ background_ = 0;
+ }
+ }
+ t_ = NORMAL;
+ } else {
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void ColorBleeder::feed(std::string *r, std::string_view input) {
+ std::string s;
+ h_->feed(&s, input);
+ relay(r, s);
+}
+
+void ColorBleeder::flush(std::string *r) {
+ std::string s;
+ h_->flush(&s);
+ relay(r, s);
+}
diff --git a/llamafile/high.cpp b/llamafile/high.cpp
index eaf77775..6e140954 100644
--- a/llamafile/high.cpp
+++ b/llamafile/high.cpp
@@ -72,6 +72,7 @@ int main(int argc, char *argv[]) {
// process input
std::string res;
+ ColorBleeder H(h);
for (;;) {
// read input chunk
@@ -87,7 +88,7 @@ int main(int argc, char *argv[]) {
// highlight chunk
res.clear();
- h->feed(&res, std::string_view(buf, got));
+ H.feed(&res, std::string_view(buf, got));
// write highlighted output chunk
write(outfd, res.data(), res.size());
@@ -95,6 +96,6 @@ int main(int argc, char *argv[]) {
// flush highlighter
res.clear();
- h->flush(&res);
+ H.flush(&res);
write(outfd, res.data(), res.size());
}
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 233b3b7d..450fa47e 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -40,7 +40,10 @@ Highlight *Highlight::create(const std::string_view &lang) {
} else if (lang == "js" || //
lang == "json" || //
lang == "javascript") {
- return new HighlightC(is_keyword_js);
+ return new HighlightJs(is_keyword_js, nullptr);
+ } else if (lang == "ts" || //
+ lang == "typescript") {
+ return new HighlightJs(is_keyword_typescript, is_keyword_typescript_type);
} else if (lang == "java") {
return new HighlightC(is_keyword_java);
} else if (lang == "py" || //
@@ -60,7 +63,7 @@ Highlight *Highlight::create(const std::string_view &lang) {
lang == "pascal") {
return new HighlightPascal;
} else if (lang == "go") {
- return new HighlightC(is_keyword_go, is_keyword_go_type);
+ return new HighlightGo;
} else if (lang == "sql") {
return new HighlightSql;
} else if (lang == "css") {
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index ca6f88b0..132c92d1 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -51,6 +51,8 @@ is_keyword_f is_keyword_c_builtin;
is_keyword_f is_keyword_c_constant;
is_keyword_f is_keyword_cxx;
is_keyword_f is_keyword_js;
+is_keyword_f is_keyword_js_builtin;
+is_keyword_f is_keyword_js_constant;
is_keyword_f is_keyword_java;
is_keyword_f is_keyword_python;
is_keyword_f is_keyword_rust;
@@ -90,6 +92,9 @@ is_keyword_f is_keyword_tcl_builtin;
is_keyword_f is_keyword_ruby;
is_keyword_f is_keyword_ruby_builtin;
is_keyword_f is_keyword_ruby_constant;
+is_keyword_f is_keyword_typescript;
+is_keyword_f is_keyword_typescript_type;
+is_keyword_f is_keyword_typescript_constant;
}
class Highlight {
@@ -108,10 +113,29 @@ class HighlightPlain : public Highlight {
void flush(std::string *result) override;
};
+class ColorBleeder : public Highlight {
+ public:
+ ColorBleeder(Highlight *h);
+ ~ColorBleeder() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ void relay(std::string *r, const std::string &s);
+ unsigned char t_ = 0;
+ unsigned char x_ = 0;
+ unsigned char n_ = 0;
+ unsigned char intensity_ = 0;
+ unsigned char foreground_ = 0;
+ unsigned char background_ = 0;
+ unsigned char sgr_codes_[8];
+ Highlight *h_;
+};
+
class HighlightC : public Highlight {
public:
- HighlightC(is_keyword_f is_keyword = is_keyword_c, is_keyword_f is_type = nullptr,
- is_keyword_f is_builtin = nullptr, is_keyword_f is_constant = nullptr);
+ HighlightC(is_keyword_f *is_keyword = is_keyword_c, is_keyword_f *is_type = nullptr,
+ is_keyword_f *is_builtin = nullptr, is_keyword_f *is_constant = nullptr);
~HighlightC() override;
void feed(std::string *result, std::string_view input) override;
void flush(std::string *result) override;
@@ -127,6 +151,33 @@ class HighlightC : public Highlight {
is_keyword_f *is_constant_;
};
+class HighlightGo : public Highlight {
+ public:
+ HighlightGo();
+ ~HighlightGo() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightJs : public Highlight {
+ public:
+ HighlightJs(is_keyword_f *is_keyword, is_keyword_f *is_type);
+ ~HighlightJs() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ int expect_;
+ std::string word_;
+ is_keyword_f *is_type_;
+ is_keyword_f *is_keyword_;
+};
+
class HighlightPython : public Highlight {
public:
HighlightPython();
@@ -302,6 +353,7 @@ class HighlightHaskell : public Highlight {
private:
int t_ = 0;
+ int level_ = 0;
std::string symbol_;
};
@@ -315,6 +367,11 @@ class HighlightPerl : public Highlight {
private:
int t_ = 0;
int i_ = 0;
+ int c_ = 0;
+ int last_ = 0;
+ int expect_ = 0;
+ unsigned char opener_ = 0;
+ unsigned char closer_ = 0;
bool pending_heredoc_ = false;
bool indented_heredoc_ = false;
std::string word_;
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cpp
index 1be0706c..2175485f 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight_c.cpp
@@ -30,8 +30,6 @@ enum {
SLASH_SLASH,
SLASH_STAR,
SLASH_STAR_STAR,
- TICK,
- TICK_BACKSLASH,
R,
R_DQUOTE,
RAW,
@@ -39,8 +37,8 @@ enum {
HighlightC::HighlightC(is_keyword_f *is_keyword, //
is_keyword_f *is_type, //
- is_keyword_f is_builtin, //
- is_keyword_f is_constant)
+ is_keyword_f *is_builtin, //
+ is_keyword_f *is_constant)
: is_keyword_(is_keyword),
is_type_(is_type),
is_builtin_(is_builtin),
@@ -73,10 +71,6 @@ void HighlightC::feed(std::string *r, std::string_view input) {
t_ = DQUOTE;
*r += HI_STRING;
*r += c;
- } else if (c == '`' && is_keyword_ == is_keyword_js) {
- t_ = TICK;
- *r += HI_STRING;
- *r += c;
} else {
*r += c;
}
@@ -184,21 +178,6 @@ void HighlightC::feed(std::string *r, std::string_view input) {
t_ = DQUOTE;
break;
- case TICK:
- *r += c;
- if (c == '`') {
- *r += HI_RESET;
- t_ = NORMAL;
- } else if (c == '\\') {
- t_ = TICK_BACKSLASH;
- }
- break;
-
- case TICK_BACKSLASH:
- *r += c;
- t_ = TICK;
- break;
-
case R:
if (c == '"') {
t_ = R_DQUOTE;
@@ -272,8 +251,6 @@ void HighlightC::flush(std::string *r) {
case R:
*r += 'R';
break;
- case TICK:
- case TICK_BACKSLASH:
case QUOTE:
case QUOTE_BACKSLASH:
case DQUOTE:
diff --git a/llamafile/highlight_go.cpp b/llamafile/highlight_go.cpp
new file mode 100644
index 00000000..64aa5612
--- /dev/null
+++ b/llamafile/highlight_go.cpp
@@ -0,0 +1,222 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ DQUOTE,
+ DQUOTE_BACKSLASH,
+ SLASH,
+ SLASH_SLASH,
+ SLASH_STAR,
+ SLASH_STAR_STAR,
+ TICK,
+ TICK_BACKSLASH,
+};
+
+HighlightGo::HighlightGo() {
+}
+
+HighlightGo::~HighlightGo() {
+}
+
+void HighlightGo::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '/') {
+ t_ = SLASH;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '`') {
+ t_ = TICK;
+ *r += HI_STRING;
+ *r += c;
+ } else {
+ *r += c;
+ }
+ break;
+
+ Word:
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_go(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_go_type(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH:
+ if (c == '/') {
+ *r += HI_COMMENT;
+ *r += "//";
+ t_ = SLASH_SLASH;
+ } else if (c == '*') {
+ *r += HI_COMMENT;
+ *r += "/*";
+ t_ = SLASH_STAR;
+ } else {
+ *r += '/';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH_SLASH:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case SLASH_STAR:
+ *r += c;
+ if (c == '*')
+ t_ = SLASH_STAR_STAR;
+ break;
+
+ case SLASH_STAR_STAR:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c != '*') {
+ t_ = SLASH_STAR;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE;
+ break;
+
+ case TICK:
+ *r += c;
+ if (c == '`') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = TICK_BACKSLASH;
+ }
+ break;
+
+ case TICK_BACKSLASH:
+ *r += c;
+ t_ = TICK;
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightGo::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_go(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_go_type(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case SLASH:
+ *r += '/';
+ break;
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case DQUOTE:
+ case DQUOTE_BACKSLASH:
+ case SLASH_SLASH:
+ case SLASH_STAR:
+ case SLASH_STAR_STAR:
+ case TICK:
+ case TICK_BACKSLASH:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_haskell.cpp b/llamafile/highlight_haskell.cpp
index 59b21509..1120b122 100644
--- a/llamafile/highlight_haskell.cpp
+++ b/llamafile/highlight_haskell.cpp
@@ -22,8 +22,6 @@
enum {
NORMAL,
WORD,
- QUOTE,
- QUOTE_BACKSLASH,
DQUOTE,
DQUOTE_BACKSLASH,
TICK,
@@ -31,6 +29,7 @@ enum {
CURL,
CURL_HYPHEN,
CURL_HYPHEN_HYPHEN,
+ CURL_HYPHEN_CURL,
HYPHEN,
HYPHEN_HYPHEN,
HYPHEN_LT,
@@ -60,10 +59,6 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {
t_ = HYPHEN;
} else if (c == '{') {
t_ = CURL;
- } else if (c == '\'') {
- t_ = QUOTE;
- *r += HI_STRING;
- *r += c;
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
@@ -204,6 +199,7 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {
*r += HI_COMMENT;
*r += "{-";
t_ = CURL_HYPHEN;
+ level_ = 1;
} else {
*r += '{';
t_ = NORMAL;
@@ -213,35 +209,35 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {
case CURL_HYPHEN:
*r += c;
- if (c == '-')
+ if (c == '-') {
t_ = CURL_HYPHEN_HYPHEN;
+ } else if (c == '{') {
+ t_ = CURL_HYPHEN_CURL;
+ }
break;
- case CURL_HYPHEN_HYPHEN:
+ case CURL_HYPHEN_CURL:
*r += c;
- if (c == '}') {
- *r += HI_RESET;
- t_ = NORMAL;
- } else if (c != '-') {
+ if (c == '-') {
+ t_ = CURL_HYPHEN;
+ ++level_;
+ } else if (c != '{') {
t_ = CURL_HYPHEN;
}
break;
- case QUOTE:
+ case CURL_HYPHEN_HYPHEN:
*r += c;
- if (c == '\'') {
- *r += HI_RESET;
- t_ = NORMAL;
- } else if (c == '\\') {
- t_ = QUOTE_BACKSLASH;
+ if (c == '}') {
+ if (!--level_) {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ } else if (c != '-') {
+ t_ = CURL_HYPHEN;
}
break;
- case QUOTE_BACKSLASH:
- *r += c;
- t_ = QUOTE;
- break;
-
case DQUOTE:
*r += c;
if (c == '"') {
@@ -312,13 +308,12 @@ void HighlightHaskell::flush(std::string *r) {
break;
case TICK:
case TICK_BACKSLASH:
- case QUOTE:
- case QUOTE_BACKSLASH:
case DQUOTE:
case DQUOTE_BACKSLASH:
case HYPHEN_HYPHEN:
case CURL_HYPHEN:
case CURL_HYPHEN_HYPHEN:
+ case CURL_HYPHEN_CURL:
*r += HI_RESET;
break;
default:
diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight_js.cpp
new file mode 100644
index 00000000..fa4d236a
--- /dev/null
+++ b/llamafile/highlight_js.cpp
@@ -0,0 +1,290 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ DQUOTE,
+ DQUOTE_BACKSLASH,
+ SLASH,
+ SLASH_SLASH,
+ SLASH_STAR,
+ SLASH_STAR_STAR,
+ TICK,
+ TICK_BACKSLASH,
+ REGEX,
+ REGEX_BACKSLASH,
+};
+
+enum {
+ EXPECT_VALUE,
+ EXPECT_OPERATOR,
+};
+
+HighlightJs::HighlightJs(is_keyword_f *is_keyword, is_keyword_f *is_type)
+ : expect_(EXPECT_VALUE), is_keyword_(is_keyword), is_type_(is_type) {
+}
+
+HighlightJs::~HighlightJs() {
+}
+
+void HighlightJs::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '/') {
+ t_ = SLASH;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ expect_ = EXPECT_OPERATOR;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ expect_ = EXPECT_OPERATOR;
+ } else if (c == '`') {
+ t_ = TICK;
+ *r += HI_STRING;
+ *r += c;
+ expect_ = EXPECT_OPERATOR;
+ } else if (c == ')' || c == '}' || c == ']') {
+ expect_ = EXPECT_OPERATOR;
+ *r += c;
+ } else if (ispunct(c)) {
+ expect_ = EXPECT_VALUE;
+ *r += c;
+ } else if (isdigit(c)) {
+ expect_ = EXPECT_OPERATOR;
+ *r += c;
+ } else {
+ *r += c;
+ }
+ break;
+
+ Word:
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ expect_ = EXPECT_VALUE;
+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
+ expect_ = EXPECT_VALUE;
+ } else if (is_keyword_js_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ expect_ = EXPECT_OPERATOR;
+ } else if (is_keyword_js_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ expect_ = EXPECT_OPERATOR;
+ } else {
+ *r += word_;
+ expect_ = EXPECT_OPERATOR;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH:
+ if (c == '/') {
+ *r += HI_COMMENT;
+ *r += "//";
+ t_ = SLASH_SLASH;
+ } else if (c == '*') {
+ *r += HI_COMMENT;
+ *r += "/*";
+ t_ = SLASH_STAR;
+ } else if (expect_ == EXPECT_VALUE) {
+ expect_ = EXPECT_OPERATOR;
+ *r += HI_STRING;
+ *r += '/';
+ *r += c;
+ if (c == '\\') {
+ t_ = REGEX_BACKSLASH;
+ } else {
+ t_ = REGEX;
+ }
+ } else {
+ *r += '/';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH_SLASH:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case SLASH_STAR:
+ *r += c;
+ if (c == '*')
+ t_ = SLASH_STAR_STAR;
+ break;
+
+ case SLASH_STAR_STAR:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c != '*') {
+ t_ = SLASH_STAR;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE;
+ break;
+
+ case TICK:
+ *r += c;
+ if (c == '`') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = TICK_BACKSLASH;
+ }
+ break;
+
+ case TICK_BACKSLASH:
+ *r += c;
+ t_ = TICK;
+ break;
+
+ case REGEX:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = REGEX_BACKSLASH;
+ }
+ break;
+
+ case REGEX_BACKSLASH:
+ *r += c;
+ t_ = REGEX;
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightJs::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {
+ *r += HI_TYPE;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_js_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_js_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case SLASH:
+ *r += '/';
+ break;
+ case TICK:
+ case TICK_BACKSLASH:
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case DQUOTE:
+ case DQUOTE_BACKSLASH:
+ case SLASH_SLASH:
+ case SLASH_STAR:
+ case SLASH_STAR_STAR:
+ case REGEX:
+ case REGEX_BACKSLASH:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_lua.cpp b/llamafile/highlight_lua.cpp
index d393f3ed..fa95127b 100644
--- a/llamafile/highlight_lua.cpp
+++ b/llamafile/highlight_lua.cpp
@@ -19,6 +19,8 @@
#include <ctype.h>
+// TODO: Support --[[ syntax
+
enum {
NORMAL,
WORD,
diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cpp
index fe2465c5..98559c1a 100644
--- a/llamafile/highlight_perl.cpp
+++ b/llamafile/highlight_perl.cpp
@@ -37,8 +37,108 @@ enum {
LT_LT_QNAME,
HEREDOC_BOL,
HEREDOC,
+ REGEX,
+ REGEX_BACKSLASH,
+ S_REGEX,
+ S_REGEX_BACKSLASH,
+ S_REGEX_S,
+ S_REGEX_S_BACKSLASH,
+ EQUAL,
};
+enum {
+ EXPECT_VALUE,
+ EXPECT_OPERATOR,
+};
+
+static int mirror(int c) {
+ switch (c) {
+ case '(':
+ return ')';
+ case '{':
+ return '}';
+ case '[':
+ return ']';
+ case '<':
+ return '>';
+ default:
+ return c;
+ }
+}
+
+static bool is_magic_var(int c) {
+ switch (c) {
+ case '!':
+ case '"':
+ case '#':
+ case '&':
+ case '-':
+ case '/':
+ case '<':
+ case '=':
+ case '>':
+ case '?':
+ case '@':
+ case '\'':
+ case '\\':
+ case '^':
+ case '_':
+ case '`':
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_regex_punct(int c) {
+ switch (c) {
+ case '!':
+ case '"':
+ case '#':
+ case '%':
+ case '&':
+ case '(':
+ case '*':
+ case ',':
+ case '-':
+ case '.':
+ case '/':
+ case ':':
+ case ';':
+ case '<':
+ case '=':
+ case '@':
+ case '[':
+ case '\'':
+ case '^':
+ case '`':
+ case '{':
+ case '|':
+ case '~':
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_regex_prefix(const std::string_view &s) {
+ return s == "m" || //
+ s == "s" || //
+ s == "y" || //
+ s == "q" || //
+ s == "tr" || //
+ s == "qq" || //
+ s == "qw" || //
+ s == "qx" || //
+ s == "qr";
+}
+
+static bool is_double_regex(const std::string_view &s) {
+ return s == "s" || //
+ s == "y" || //
+ s == "tr";
+}
+
HighlightPerl::HighlightPerl() {
}
@@ -46,9 +146,10 @@ HighlightPerl::~HighlightPerl() {
}
void HighlightPerl::feed(std::string *r, std::string_view input) {
- int c;
+ int c = 0;
for (size_t i = 0; i < input.size(); ++i) {
- c = input[i] & 255;
+ last_ = c_;
+ c_ = c = input[i] & 255;
switch (t_) {
Normal:
@@ -60,21 +161,28 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
t_ = QUOTE;
*r += HI_STRING;
*r += c;
+ expect_ = EXPECT_OPERATOR;
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
*r += c;
+ expect_ = EXPECT_OPERATOR;
+ } else if (c == '=' && (!last_ || last_ == '\n')) {
+ t_ = EQUAL;
} else if (c == '`') {
t_ = TICK;
*r += HI_STRING;
*r += c;
+ expect_ = EXPECT_OPERATOR;
} else if (c == '$') {
*r += c;
t_ = VAR;
+ expect_ = EXPECT_OPERATOR;
} else if (c == '@' || c == '%') {
*r += c;
*r += HI_VAR;
t_ = VAR2;
+ expect_ = EXPECT_OPERATOR;
} else if (c == '#') {
*r += HI_COMMENT;
*r += c;
@@ -82,6 +190,14 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
} else if (c == '<') {
*r += c;
t_ = LT;
+ expect_ = EXPECT_VALUE;
+ } else if (c == '/' && expect_ == EXPECT_VALUE && last_ != '/') {
+ opener_ = '/';
+ closer_ = '/';
+ expect_ = EXPECT_OPERATOR;
+ *r += HI_STRING;
+ *r += c;
+ t_ = REGEX;
} else if (c == '\n') {
*r += c;
if (pending_heredoc_) {
@@ -90,11 +206,21 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
t_ = HEREDOC_BOL;
i_ = 0;
}
+ } else if (c == ')' || c == '}' || c == ']') {
+ expect_ = EXPECT_OPERATOR;
+ *r += c;
+ } else if (ispunct(c)) {
+ expect_ = EXPECT_VALUE;
+ *r += c;
+ } else if (isdigit(c)) {
+ expect_ = EXPECT_OPERATOR;
+ *r += c;
} else {
*r += c;
}
break;
+ Word:
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
word_ += c;
@@ -103,8 +229,27 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
+ if (word_ == "shift") {
+ expect_ = EXPECT_OPERATOR;
+ } else {
+ expect_ = EXPECT_VALUE;
+ }
} else {
*r += word_;
+ expect_ = EXPECT_VALUE;
+ if (is_regex_punct(c) && is_regex_prefix(word_)) {
+ opener_ = c;
+ closer_ = mirror(c);
+ *r += HI_STRING;
+ *r += c;
+ if (is_double_regex(word_)) {
+ t_ = S_REGEX;
+ } else {
+ t_ = REGEX;
+ }
+ word_.clear();
+ break;
+ }
}
word_.clear();
t_ = NORMAL;
@@ -113,13 +258,20 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case VAR:
- t_ = VAR2;
- if (c == '{') {
+ if (isdigit(c) || is_magic_var(c)) {
+ *r += HI_VAR;
+ *r += c;
+ *r += HI_RESET;
+ t_ = NORMAL;
+ break;
+ } else if (c == '{') {
+ t_ = VAR2;
*r += c;
*r += HI_VAR;
break;
} else {
*r += HI_VAR;
+ t_ = VAR2;
}
// fallthrough
@@ -143,6 +295,50 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
}
break;
+ case REGEX:
+ *r += c;
+ if (c == closer_) {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = REGEX_BACKSLASH;
+ }
+ break;
+
+ case REGEX_BACKSLASH:
+ *r += c;
+ t_ = REGEX;
+ break;
+
+ case S_REGEX:
+ *r += c;
+ if (c == opener_) {
+ t_ = S_REGEX_S;
+ } else if (c == '\\') {
+ t_ = S_REGEX_BACKSLASH;
+ }
+ break;
+
+ case S_REGEX_BACKSLASH:
+ *r += c;
+ t_ = S_REGEX;
+ break;
+
+ case S_REGEX_S:
+ *r += c;
+ if (c == closer_) {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = S_REGEX_S_BACKSLASH;
+ }
+ break;
+
+ case S_REGEX_S_BACKSLASH:
+ *r += c;
+ t_ = S_REGEX_S;
+ break;
+
case QUOTE:
*r += c;
if (c == '\'') {
@@ -188,6 +384,21 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
t_ = TICK;
break;
+ case EQUAL:
+ if (isalpha(c)) {
+ *r += HI_COMMENT;
+ *r += '=';
+ *r += c;
+ heredoc_ = "=cut";
+ t_ = HEREDOC;
+ i_ = 0;
+ } else {
+ *r += '=';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
case LT:
if (c == '<') {
*r += c;
@@ -205,7 +416,8 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
if (c == '-') {
indented_heredoc_ = true;
*r += c;
- } else if (c == '\'') {
+ } else if (c == '"' || c == '\'') {
+ closer_ = c;
t_ = LT_LT_QNAME;
*r += HI_STRING;
*r += c;
@@ -237,7 +449,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case LT_LT_QNAME:
*r += c;
- if (c == '\'') {
+ if (c == closer_) {
*r += HI_RESET;
t_ = HEREDOC_BOL;
pending_heredoc_ = true;
@@ -267,8 +479,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case HEREDOC:
*r += c;
- if (c == '\n')
+ if (c == '\n') {
t_ = HEREDOC_BOL;
+ i_ = 0;
+ }
break;
default:
@@ -289,6 +503,9 @@ void HighlightPerl::flush(std::string *r) {
}
word_.clear();
break;
+ case EQUAL:
+ *r += '=';
+ break;
case VAR2:
case TICK:
case TICK_BACKSLASH:
@@ -300,6 +517,12 @@ void HighlightPerl::flush(std::string *r) {
case HEREDOC_BOL:
case HEREDOC:
case LT_LT_QNAME:
+ case REGEX:
+ case REGEX_BACKSLASH:
+ case S_REGEX:
+ case S_REGEX_BACKSLASH:
+ case S_REGEX_S:
+ case S_REGEX_S_BACKSLASH:
*r += HI_RESET;
break;
default:
diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cpp
index 75f72e5b..82a8bed7 100644
--- a/llamafile/highlight_ruby.cpp
+++ b/llamafile/highlight_ruby.cpp
@@ -366,6 +366,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case DOLLAR:
if (isdigit(c) || //
c == '!' || //
+ c == '"' || //
c == '#' || //
c == '$' || //
c == '&' || //
@@ -375,6 +376,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
c == '=' || //
c == '>' || //
c == '@' || //
+ c == '\'' || //
c == '\\' || //
c == '^' || //
c == '_' || //
@@ -383,7 +385,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
*r += '$';
*r += c;
*r += HI_RESET;
- t_ = AT_WORD;
+ t_ = NORMAL;
} else if (isalpha(c)) {
*r += HI_VAR;
*r += '$';
@@ -487,7 +489,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
t_ = LT_LT_NAME;
heredoc_ += c;
*r += c;
- } else if (!isblank(c)) {
+ } else {
t_ = NORMAL;
goto Normal;
}
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index 0ab73bac..ce172712 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -56,6 +56,7 @@ const char *const kLanguages[] = {
"sql", //
"swift", //
"tcl", //
+ "typescript", //
"zig", //
};
diff --git a/llamafile/is_keyword_cxx.c b/llamafile/is_keyword_cxx.c
index 122b1d6c..65a2eada 100644
--- a/llamafile/is_keyword_cxx.c
+++ b/llamafile/is_keyword_cxx.c
@@ -33,7 +33,7 @@
#include <string.h>
-#define TOTAL_KEYWORDS 167
+#define TOTAL_KEYWORDS 165
#define MIN_WORD_LENGTH 2
#define MAX_WORD_LENGTH 27
#define MIN_HASH_VALUE 3
@@ -55,16 +55,16 @@ hash (register const char *str, register size_t len)
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
- 451, 451, 25, 451, 451, 451, 451, 451, 451, 451,
- 451, 451, 451, 451, 451, 451, 451, 451, 0, 60,
- 50, 30, 35, 451, 25, 451, 5, 451, 451, 451,
+ 451, 451, 30, 451, 451, 451, 451, 451, 451, 451,
+ 451, 451, 451, 451, 451, 451, 451, 451, 10, 60,
+ 50, 30, 35, 451, 25, 451, 0, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 0, 10,
- 5, 451, 451, 451, 451, 451, 451, 451, 5, 451,
+ 10, 451, 451, 451, 451, 451, 451, 451, 5, 451,
5, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 0, 451, 15, 130, 0,
20, 10, 80, 145, 140, 45, 451, 30, 70, 35,
- 0, 65, 95, 125, 5, 15, 0, 110, 105, 140,
- 85, 5, 5, 451, 451, 451, 451, 451, 451, 451,
+ 0, 65, 95, 120, 5, 15, 0, 110, 85, 140,
+ 85, 5, 10, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
451, 451, 451, 451, 451, 451, 451, 451, 451, 451,
@@ -116,11 +116,11 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str18[sizeof("noexcept")];
char stringpool_str19[sizeof("co_return")];
char stringpool_str21[sizeof("extern")];
- char stringpool_str22[sizeof("__FUNCTION__")];
+ char stringpool_str22[sizeof("char8_t")];
char stringpool_str23[sizeof("__extension__")];
char stringpool_str24[sizeof("char")];
char stringpool_str26[sizeof("struct")];
- char stringpool_str27[sizeof("char8_t")];
+ char stringpool_str27[sizeof("__FUNCTION__")];
char stringpool_str28[sizeof("__attribute__")];
char stringpool_str29[sizeof("case")];
char stringpool_str30[sizeof("__restrict")];
@@ -140,9 +140,9 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str51[sizeof("strlenesque")];
char stringpool_str52[sizeof("returnspointerwithnoaliases")];
char stringpool_str53[sizeof("char32_t")];
- char stringpool_str54[sizeof("__real__ ")];
char stringpool_str57[sizeof("__asm__")];
char stringpool_str58[sizeof("and")];
+ char stringpool_str59[sizeof("__real__ ")];
char stringpool_str61[sizeof("interruptfn")];
char stringpool_str62[sizeof("dontcallback")];
char stringpool_str63[sizeof("continue")];
@@ -165,10 +165,10 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str90[sizeof("__asm")];
char stringpool_str92[sizeof("thatispacked")];
char stringpool_str93[sizeof("for")];
- char stringpool_str94[sizeof("__imag__ ")];
+ char stringpool_str94[sizeof("_Float128")];
char stringpool_str96[sizeof("__real")];
char stringpool_str98[sizeof("xor")];
- char stringpool_str99[sizeof("_Float128")];
+ char stringpool_str99[sizeof("__imag__ ")];
char stringpool_str100[sizeof("__forceinline__")];
char stringpool_str102[sizeof("thread_local")];
char stringpool_str103[sizeof("operator")];
@@ -180,11 +180,11 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str112[sizeof("reallocesque")];
char stringpool_str113[sizeof("typename")];
char stringpool_str114[sizeof("threadIdx")];
+ char stringpool_str115[sizeof("__device__")];
char stringpool_str116[sizeof("export")];
char stringpool_str118[sizeof("_Float16")];
char stringpool_str120[sizeof("forcealignargpointer")];
char stringpool_str123[sizeof("template")];
- char stringpool_str124[sizeof("true")];
char stringpool_str125[sizeof("__inline__")];
char stringpool_str127[sizeof("if")];
char stringpool_str128[sizeof("_Float64")];
@@ -192,10 +192,9 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str130[sizeof("scanfesque")];
char stringpool_str131[sizeof("inline")];
char stringpool_str133[sizeof("__inline")];
- char stringpool_str135[sizeof("__device__")];
+ char stringpool_str136[sizeof("not_eq")];
char stringpool_str138[sizeof("atomic_commit")];
char stringpool_str140[sizeof("atomic_noexcept")];
- char stringpool_str141[sizeof("not_eq")];
char stringpool_str143[sizeof("_Float32")];
char stringpool_str145[sizeof("bitor")];
char stringpool_str146[sizeof("__null")];
@@ -204,7 +203,7 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str149[sizeof("__alignof")];
char stringpool_str150[sizeof("float")];
char stringpool_str151[sizeof("friend")];
- char stringpool_str154[sizeof("__float80")];
+ char stringpool_str154[sizeof("void")];
char stringpool_str155[sizeof("wontreturn")];
char stringpool_str156[sizeof("bitand")];
char stringpool_str158[sizeof("explicit")];
@@ -212,14 +211,15 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str160[sizeof("union")];
char stringpool_str161[sizeof("printfesque")];
char stringpool_str163[sizeof("__host__")];
+ char stringpool_str164[sizeof("__float80")];
char stringpool_str166[sizeof("typeid")];
char stringpool_str169[sizeof("dontthrow")];
char stringpool_str170[sizeof("__signed__")];
- char stringpool_str171[sizeof("sizeof")];
+ char stringpool_str171[sizeof("and_eq")];
char stringpool_str172[sizeof("private")];
- char stringpool_str174[sizeof("void")];
- char stringpool_str175[sizeof("false")];
- char stringpool_str176[sizeof("and_eq")];
+ char stringpool_str173[sizeof("volatile")];
+ char stringpool_str175[sizeof("__volatile")];
+ char stringpool_str176[sizeof("sizeof")];
char stringpool_str177[sizeof("nullptr")];
char stringpool_str178[sizeof("register")];
char stringpool_str180[sizeof("compl")];
@@ -229,11 +229,10 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str187[sizeof("synchronized")];
char stringpool_str188[sizeof("__signed")];
char stringpool_str191[sizeof("mallocesque")];
- char stringpool_str193[sizeof("volatile")];
- char stringpool_str195[sizeof("__volatile")];
+ char stringpool_str193[sizeof("requires")];
char stringpool_str196[sizeof("signed")];
char stringpool_str197[sizeof("__launch_bounds__")];
- char stringpool_str198[sizeof("requires")];
+ char stringpool_str198[sizeof("__volatile__ ")];
char stringpool_str199[sizeof("protected")];
char stringpool_str202[sizeof("typedef")];
char stringpool_str205[sizeof("break")];
@@ -242,7 +241,6 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str209[sizeof("__label__")];
char stringpool_str210[sizeof("while")];
char stringpool_str211[sizeof("__imag")];
- char stringpool_str213[sizeof("__volatile__ ")];
char stringpool_str214[sizeof("goto")];
char stringpool_str215[sizeof("privileged")];
char stringpool_str216[sizeof("double")];
@@ -251,23 +249,23 @@ is_keyword_cxx (register const char *str, register size_t len)
char stringpool_str219[sizeof("long")];
char stringpool_str220[sizeof("__global__")];
char stringpool_str221[sizeof("__bf16")];
+ char stringpool_str226[sizeof("xor_eq")];
char stringpool_str229[sizeof("libcesque")];
- char stringpool_str231[sizeof("xor_eq")];
char stringpool_str232[sizeof("gridDim")];
+ char stringpool_str241[sizeof("vallocesque")];
char stringpool_str251[sizeof("textwindows")];
- char stringpool_str261[sizeof("vallocesque")];
char stringpool_str268[sizeof("blockDim")];
char stringpool_str269[sizeof("bool")];
char stringpool_str273[sizeof("__builtin_offsetof")];
char stringpool_str276[sizeof("public")];
+ char stringpool_str277[sizeof("virtual")];
char stringpool_str283[sizeof("new")];
char stringpool_str285[sizeof("catch")];
char stringpool_str290[sizeof("throw")];
char stringpool_str292[sizeof("wchar_t")];
- char stringpool_str297[sizeof("virtual")];
char stringpool_str298[sizeof("unsigned")];
+ char stringpool_str310[sizeof("or_eq")];
char stringpool_str318[sizeof("blockIdx")];
- char stringpool_str320[sizeof("or_eq")];
char stringpool_str336[sizeof("__builtin_va_arg")];
char stringpool_str450[sizeof("using")];
};
@@ -286,11 +284,11 @@ is_keyword_cxx (register const char *str, register size_t len)
"noexcept",
"co_return",
"extern",
- "__FUNCTION__",
+ "char8_t",
"__extension__",
"char",
"struct",
- "char8_t",
+ "__FUNCTION__",
"__attribute__",
"case",
"__restrict",
@@ -310,9 +308,9 @@ is_keyword_cxx (register const char *str, register size_t len)
"strlenesque",
"returnspointerwithnoaliases",
"char32_t",
- "__real__ ",
"__asm__",
"and",
+ "__real__ ",
"interruptfn",
"dontcallback",
"continue",
@@ -335,10 +333,10 @@ is_keyword_cxx (register const char *str, register size_t len)
"__asm",
"thatispacked",
"for",
- "__imag__ ",
+ "_Float128",
"__real",
"xor",
- "_Float128",
+ "__imag__ ",
"__forceinline__",
"thread_local",
"operator",
@@ -350,11 +348,11 @@ is_keyword_cxx (register const char *str, register size_t len)
"reallocesque",
"typename",
"threadIdx",
+ "__device__",
"export",
"_Float16",
"forcealignargpointer",
"template",
- "true",
"__inline__",
"if",
"_Float64",
@@ -362,10 +360,9 @@ is_keyword_cxx (register const char *str, register size_t len)
"scanfesque",
"inline",
"__inline",
- "__device__",
+ "not_eq",
"atomic_commit",
"atomic_noexcept",
- "not_eq",
"_Float32",
"bitor",
"__null",
@@ -374,7 +371,7 @@ is_keyword_cxx (register const char *str, register size_t len)
"__alignof",
"float",
"friend",
- "__float80",
+ "void",
"wontreturn",
"bitand",
"explicit",
@@ -382,14 +379,15 @@ is_keyword_cxx (register const char *str, register size_t len)
"union",
"printfesque",
"__host__",
+ "__float80",
"typeid",
"dontthrow",
"__signed__",
- "sizeof",
- "private",
- "void",
- "false",
"and_eq",
+ "private",
+ "volatile",
+ "__volatile",
+ "sizeof",
"nullptr",
"register",
"compl",
@@ -399,11 +397,10 @@ is_keyword_cxx (register const char *str, register size_t len)
"synchronized",
"__signed",
"mallocesque",
- "volatile",
- "__volatile",
+ "requires",
"signed",
"__launch_bounds__",
- "requires",
+ "__volatile__ ",
"protected",
"typedef",
"break",
@@ -412,7 +409,6 @@ is_keyword_cxx (register const char *str, register size_t len)
"__label__",
"while",
"__imag",
- "__volatile__ ",
"goto",
"privileged",
"double",
@@ -421,23 +417,23 @@ is_keyword_cxx (register const char *str, register size_t len)
"long",
"__global__",
"__bf16",
- "libcesque",
"xor_eq",
+ "libcesque",
"gridDim",
- "textwindows",
"vallocesque",
+ "textwindows",
"blockDim",
"bool",
"__builtin_offsetof",
"public",
+ "virtual",
"new",
"catch",
"throw",
"wchar_t",
- "virtual",
"unsigned",
- "blockIdx",
"or_eq",
+ "blockIdx",
"__builtin_va_arg",
"using"
};
@@ -495,11 +491,11 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,
- -1, -1,
+ -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,
- -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,
@@ -549,7 +545,7 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str112,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str113,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str114,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str118,
@@ -557,7 +553,7 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str120,
-1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str123,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str124,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str127,
@@ -567,14 +563,13 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str131,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str133,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str135,
-1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str138,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str140,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str141,
- -1,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str143,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str145,
@@ -595,15 +590,16 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str161,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str163,
- -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str164,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str166,
-1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str169,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str170,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str171,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str172,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str173,
-1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str174,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str175,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str176,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str177,
@@ -621,8 +617,7 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str191,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str193,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str195,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str196,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str197,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str198,
@@ -637,8 +632,7 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str209,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str210,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str211,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str213,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str214,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str215,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str216,
@@ -647,24 +641,26 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str219,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str220,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str221,
- -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str226,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str229,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str231,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str232,
- -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str241,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str251,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str261,
- -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str268,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str269,
-1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str273,
-1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str276,
- -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str277,
+ -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str283,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str285,
@@ -672,17 +668,15 @@ is_keyword_cxx (register const char *str, register size_t len)
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str290,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str292,
- -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str297,
+ -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str298,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str310,
+ -1, -1, -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str318,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str320,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str336,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1,
diff --git a/llamafile/is_keyword_cxx.gperf b/llamafile/is_keyword_cxx.gperf
index 00969515..099ba0cb 100644
--- a/llamafile/is_keyword_cxx.gperf
+++ b/llamafile/is_keyword_cxx.gperf
@@ -96,7 +96,6 @@ enum
explicit
export
extern
-false
float
for
forcealign
@@ -157,7 +156,6 @@ this
threadIdx
thread_local
throw
-true
try
typedef
typeid
diff --git a/llamafile/is_keyword_go_type.c b/llamafile/is_keyword_go_type.c
index a42b8ac2..a991d39c 100644
--- a/llamafile/is_keyword_go_type.c
+++ b/llamafile/is_keyword_go_type.c
@@ -33,7 +33,7 @@
#include <string.h>
-#define TOTAL_KEYWORDS 17
+#define TOTAL_KEYWORDS 18
#define MIN_WORD_LENGTH 3
#define MAX_WORD_LENGTH 10
#define MIN_HASH_VALUE 4
@@ -62,8 +62,8 @@ hash (register const char *str, register size_t len)
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 5, 10,
- 27, 10, 0, 27, 27, 0, 27, 27, 27, 27,
- 27, 27, 27, 27, 0, 27, 5, 0, 27, 27,
+ 27, 10, 0, 0, 27, 0, 27, 27, 27, 27,
+ 27, 27, 27, 27, 0, 0, 5, 0, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
@@ -89,6 +89,7 @@ is_keyword_go_type (register const char *str, register size_t len)
{
char stringpool_str4[sizeof("int8")];
char stringpool_str5[sizeof("uint8")];
+ char stringpool_str6[sizeof("string")];
char stringpool_str7[sizeof("uintptr")];
char stringpool_str8[sizeof("int")];
char stringpool_str9[sizeof("uint")];
@@ -109,6 +110,7 @@ is_keyword_go_type (register const char *str, register size_t len)
{
"int8",
"uint8",
+ "string",
"uintptr",
"int",
"uint",
@@ -131,7 +133,7 @@ is_keyword_go_type (register const char *str, register size_t len)
-1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
diff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/is_keyword_go_type.gperf
index ec5eada3..2667eab1 100644
--- a/llamafile/is_keyword_go_type.gperf
+++ b/llamafile/is_keyword_go_type.gperf
@@ -18,6 +18,7 @@ int32
int64
int8
rune
+string
uint
uint16
uint32
diff --git a/llamafile/is_keyword_js.c b/llamafile/is_keyword_js.c
index 3a4fd745..8e186680 100644
--- a/llamafile/is_keyword_js.c
+++ b/llamafile/is_keyword_js.c
@@ -33,12 +33,12 @@
#include <string.h>
-#define TOTAL_KEYWORDS 39
+#define TOTAL_KEYWORDS 36
#define MIN_WORD_LENGTH 2
#define MAX_WORD_LENGTH 10
-#define MIN_HASH_VALUE 3
-#define MAX_HASH_VALUE 55
-/* maximum key range = 53, duplicates = 0 */
+#define MIN_HASH_VALUE 2
+#define MAX_HASH_VALUE 61
+/* maximum key range = 60, duplicates = 0 */
#ifdef __GNUC__
__inline
@@ -52,32 +52,32 @@ hash (register const char *str, register size_t len)
{
static const unsigned char asso_values[] =
{
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 5, 45, 0,
- 5, 0, 15, 56, 40, 0, 56, 56, 35, 15,
- 25, 0, 56, 56, 0, 10, 0, 0, 45, 10,
- 30, 25, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,
- 56, 56, 56, 56, 56, 56
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 0, 15, 0,
+ 0, 0, 10, 62, 10, 0, 62, 62, 10, 5,
+ 0, 20, 62, 62, 30, 0, 20, 40, 15, 15,
+ 25, 35, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,
+ 62, 62, 62, 62, 62, 62
};
return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];
}
@@ -87,138 +87,133 @@ is_keyword_js (register const char *str, register size_t len)
{
struct stringpool_t
{
- char stringpool_str3[sizeof("try")];
- char stringpool_str4[sizeof("true")];
- char stringpool_str5[sizeof("const")];
- char stringpool_str6[sizeof("return")];
- char stringpool_str7[sizeof("do")];
- char stringpool_str8[sizeof("continue")];
- char stringpool_str9[sizeof("case")];
- char stringpool_str10[sizeof("catch")];
- char stringpool_str11[sizeof("delete")];
- char stringpool_str12[sizeof("default")];
- char stringpool_str13[sizeof("debugger")];
- char stringpool_str14[sizeof("with")];
- char stringpool_str15[sizeof("super")];
- char stringpool_str16[sizeof("static")];
- char stringpool_str17[sizeof("if")];
- char stringpool_str18[sizeof("for")];
+ char stringpool_str2[sizeof("in")];
+ char stringpool_str3[sizeof("new")];
+ char stringpool_str4[sizeof("case")];
+ char stringpool_str5[sizeof("catch")];
+ char stringpool_str6[sizeof("delete")];
+ char stringpool_str7[sizeof("default")];
+ char stringpool_str8[sizeof("debugger")];
+ char stringpool_str10[sizeof("instanceof")];
+ char stringpool_str11[sizeof("import")];
+ char stringpool_str12[sizeof("if")];
+ char stringpool_str13[sizeof("let")];
+ char stringpool_str14[sizeof("else")];
+ char stringpool_str15[sizeof("class")];
+ char stringpool_str17[sizeof("finally")];
+ char stringpool_str18[sizeof("var")];
+ char stringpool_str19[sizeof("with")];
char stringpool_str20[sizeof("await")];
- char stringpool_str21[sizeof("import")];
- char stringpool_str22[sizeof("finally")];
- char stringpool_str23[sizeof("function")];
- char stringpool_str25[sizeof("false")];
- char stringpool_str26[sizeof("switch")];
- char stringpool_str27[sizeof("in")];
- char stringpool_str28[sizeof("new")];
- char stringpool_str29[sizeof("null")];
- char stringpool_str30[sizeof("yield")];
- char stringpool_str31[sizeof("typeof")];
- char stringpool_str35[sizeof("instanceof")];
- char stringpool_str36[sizeof("export")];
- char stringpool_str37[sizeof("extends")];
- char stringpool_str38[sizeof("let")];
- char stringpool_str39[sizeof("else")];
- char stringpool_str40[sizeof("class")];
- char stringpool_str44[sizeof("this")];
- char stringpool_str45[sizeof("throw")];
- char stringpool_str49[sizeof("void")];
+ char stringpool_str21[sizeof("switch")];
+ char stringpool_str22[sizeof("do")];
+ char stringpool_str25[sizeof("const")];
+ char stringpool_str26[sizeof("static")];
+ char stringpool_str28[sizeof("continue")];
+ char stringpool_str30[sizeof("while")];
+ char stringpool_str31[sizeof("export")];
+ char stringpool_str32[sizeof("extends")];
+ char stringpool_str33[sizeof("for")];
+ char stringpool_str34[sizeof("this")];
+ char stringpool_str35[sizeof("throw")];
+ char stringpool_str36[sizeof("return")];
+ char stringpool_str39[sizeof("void")];
+ char stringpool_str40[sizeof("yield")];
+ char stringpool_str45[sizeof("super")];
char stringpool_str50[sizeof("break")];
- char stringpool_str53[sizeof("var")];
- char stringpool_str55[sizeof("while")];
+ char stringpool_str53[sizeof("try")];
+ char stringpool_str58[sizeof("function")];
+ char stringpool_str61[sizeof("typeof")];
};
static const struct stringpool_t stringpool_contents =
{
- "try",
- "true",
- "const",
- "return",
- "do",
- "continue",
+ "in",
+ "new",
"case",
"catch",
"delete",
"default",
"debugger",
- "with",
- "super",
- "static",
- "if",
- "for",
- "await",
+ "instanceof",
"import",
+ "if",
+ "let",
+ "else",
+ "class",
"finally",
- "function",
- "false",
+ "var",
+ "with",
+ "await",
"switch",
- "in",
- "new",
- "null",
- "yield",
- "typeof",
- "instanceof",
+ "do",
+ "const",
+ "static",
+ "continue",
+ "while",
"export",
"extends",
- "let",
- "else",
- "class",
+ "for",
"this",
"throw",
+ "return",
"void",
+ "yield",
+ "super",
"break",
- "var",
- "while"
+ "try",
+ "function",
+ "typeof"
};
#define stringpool ((const char *) &stringpool_contents)
static const int wordlist[] =
{
- -1, -1, -1,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str2,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
- -1,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,
- -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
- -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
- -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
+ -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
-1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
diff --git a/llamafile/is_keyword_js.gperf b/llamafile/is_keyword_js.gperf
index 00beff2a..c59e6580 100644
--- a/llamafile/is_keyword_js.gperf
+++ b/llamafile/is_keyword_js.gperf
@@ -21,7 +21,6 @@ do
else
export
extends
-false
finally
for
function
@@ -31,14 +30,12 @@ in
instanceof
let
new
-null
return
static
super
switch
this
throw
-true
try
typeof
var
diff --git a/llamafile/is_keyword_js_builtin.c b/llamafile/is_keyword_js_builtin.c
new file mode 100644
index 00000000..1ca65467
--- /dev/null
+++ b/llamafile/is_keyword_js_builtin.c
@@ -0,0 +1,341 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_js_builtin.gperf */
+/* Computed positions: -k'1,5-6' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_js_builtin.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 65
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 22
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 102
+/* maximum key range = 100, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 25,
+ 15, 15, 103, 103, 10, 103, 5, 103, 103, 103,
+ 103, 103, 103, 103, 103, 10, 25, 25, 50, 15,
+ 25, 0, 103, 0, 70, 103, 103, 60, 5, 10,
+ 5, 103, 25, 0, 40, 25, 15, 20, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 15, 103, 10,
+ 0, 5, 103, 30, 103, 0, 103, 103, 0, 103,
+ 0, 0, 0, 103, 5, 0, 15, 35, 103, 103,
+ 0, 5, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,
+ 103, 103, 103, 103, 103, 103
+ };
+ register unsigned int hval = len;
+
+ switch (hval)
+ {
+ default:
+ hval += asso_values[(unsigned char)str[5]];
+ /*FALLTHROUGH*/
+ case 5:
+ hval += asso_values[(unsigned char)str[4]];
+ /*FALLTHROUGH*/
+ case 4:
+ case 3:
+ case 2:
+ case 1:
+ hval += asso_values[(unsigned char)str[0]];
+ break;
+ }
+ return hval;
+}
+
+const char *
+is_keyword_js_builtin (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("Set")];
+ char stringpool_str4[sizeof("Intl")];
+ char stringpool_str6[sizeof("Symbol")];
+ char stringpool_str8[sizeof("isFinite")];
+ char stringpool_str9[sizeof("eval")];
+ char stringpool_str10[sizeof("isNaN")];
+ char stringpool_str12[sizeof("Promise")];
+ char stringpool_str13[sizeof("parseInt")];
+ char stringpool_str14[sizeof("decodeURI")];
+ char stringpool_str15[sizeof("Proxy")];
+ char stringpool_str16[sizeof("escape")];
+ char stringpool_str18[sizeof("InternalError")];
+ char stringpool_str19[sizeof("encodeURI")];
+ char stringpool_str20[sizeof("Array")];
+ char stringpool_str21[sizeof("Number")];
+ char stringpool_str22[sizeof("SharedArrayBuffer")];
+ char stringpool_str23[sizeof("decodeURIComponent")];
+ char stringpool_str24[sizeof("Int8Array")];
+ char stringpool_str25[sizeof("Error")];
+ char stringpool_str26[sizeof("SyntaxError")];
+ char stringpool_str27[sizeof("Atomics")];
+ char stringpool_str28[sizeof("encodeURIComponent")];
+ char stringpool_str29[sizeof("Generator")];
+ char stringpool_str30[sizeof("Int16Array")];
+ char stringpool_str31[sizeof("RegExp")];
+ char stringpool_str32[sizeof("WeakSet")];
+ char stringpool_str33[sizeof("AsyncIterator")];
+ char stringpool_str34[sizeof("AsyncGenerator")];
+ char stringpool_str35[sizeof("Int32Array")];
+ char stringpool_str36[sizeof("String")];
+ char stringpool_str37[sizeof("GeneratorFunction")];
+ char stringpool_str38[sizeof("Iterator")];
+ char stringpool_str39[sizeof("BigUint64Array")];
+ char stringpool_str40[sizeof("parseFloat")];
+ char stringpool_str41[sizeof("Object")];
+ char stringpool_str42[sizeof("AsyncGeneratorFunction")];
+ char stringpool_str43[sizeof("URIError")];
+ char stringpool_str44[sizeof("EvalError")];
+ char stringpool_str45[sizeof("FinalizationRegistry")];
+ char stringpool_str46[sizeof("BigInt")];
+ char stringpool_str47[sizeof("Reflect")];
+ char stringpool_str48[sizeof("Function")];
+ char stringpool_str49[sizeof("ReferenceError")];
+ char stringpool_str50[sizeof("Uint8Array")];
+ char stringpool_str51[sizeof("ArrayBuffer")];
+ char stringpool_str52[sizeof("Boolean")];
+ char stringpool_str53[sizeof("BigInt64Array")];
+ char stringpool_str54[sizeof("Date")];
+ char stringpool_str55[sizeof("RangeError")];
+ char stringpool_str57[sizeof("WeakRef")];
+ char stringpool_str58[sizeof("AsyncFunction")];
+ char stringpool_str59[sizeof("AggregateError")];
+ char stringpool_str62[sizeof("Float64Array")];
+ char stringpool_str63[sizeof("Map")];
+ char stringpool_str64[sizeof("Math")];
+ char stringpool_str66[sizeof("Uint32Array")];
+ char stringpool_str67[sizeof("Float32Array")];
+ char stringpool_str68[sizeof("unescape")];
+ char stringpool_str69[sizeof("TypeError")];
+ char stringpool_str71[sizeof("Uint16Array")];
+ char stringpool_str72[sizeof("Uint8ClampedArray")];
+ char stringpool_str73[sizeof("DataView")];
+ char stringpool_str74[sizeof("JSON")];
+ char stringpool_str77[sizeof("Float16Array")];
+ char stringpool_str102[sizeof("WeakMap")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "Set",
+ "Intl",
+ "Symbol",
+ "isFinite",
+ "eval",
+ "isNaN",
+ "Promise",
+ "parseInt",
+ "decodeURI",
+ "Proxy",
+ "escape",
+ "InternalError",
+ "encodeURI",
+ "Array",
+ "Number",
+ "SharedArrayBuffer",
+ "decodeURIComponent",
+ "Int8Array",
+ "Error",
+ "SyntaxError",
+ "Atomics",
+ "encodeURIComponent",
+ "Generator",
+ "Int16Array",
+ "RegExp",
+ "WeakSet",
+ "AsyncIterator",
+ "AsyncGenerator",
+ "Int32Array",
+ "String",
+ "GeneratorFunction",
+ "Iterator",
+ "BigUint64Array",
+ "parseFloat",
+ "Object",
+ "AsyncGeneratorFunction",
+ "URIError",
+ "EvalError",
+ "FinalizationRegistry",
+ "BigInt",
+ "Reflect",
+ "Function",
+ "ReferenceError",
+ "Uint8Array",
+ "ArrayBuffer",
+ "Boolean",
+ "BigInt64Array",
+ "Date",
+ "RangeError",
+ "WeakRef",
+ "AsyncFunction",
+ "AggregateError",
+ "Float64Array",
+ "Map",
+ "Math",
+ "Uint32Array",
+ "Float32Array",
+ "unescape",
+ "TypeError",
+ "Uint16Array",
+ "Uint8ClampedArray",
+ "DataView",
+ "JSON",
+ "Float16Array",
+ "WeakMap"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str72,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_js_builtin.gperf b/llamafile/is_keyword_js_builtin.gperf
new file mode 100644
index 00000000..3d273221
--- /dev/null
+++ b/llamafile/is_keyword_js_builtin.gperf
@@ -0,0 +1,74 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_js_builtin
+%%
+AggregateError
+Array
+ArrayBuffer
+AsyncFunction
+AsyncGenerator
+AsyncGeneratorFunction
+AsyncIterator
+Atomics
+BigInt
+BigInt64Array
+BigUint64Array
+Boolean
+DataView
+Date
+Error
+EvalError
+FinalizationRegistry
+Float16Array
+Float32Array
+Float64Array
+Function
+Generator
+GeneratorFunction
+Int16Array
+Int32Array
+Int8Array
+InternalError
+Intl
+Iterator
+JSON
+Map
+Math
+Number
+Object
+Promise
+Proxy
+RangeError
+ReferenceError
+Reflect
+RegExp
+Set
+SharedArrayBuffer
+String
+Symbol
+SyntaxError
+TypeError
+URIError
+Uint16Array
+Uint32Array
+Uint8Array
+Uint8ClampedArray
+WeakMap
+WeakRef
+WeakSet
+decodeURI
+decodeURIComponent
+encodeURI
+encodeURIComponent
+escape
+eval
+isFinite
+isNaN
+parseFloat
+parseInt
+unescape
diff --git a/llamafile/is_keyword_js_constant.c b/llamafile/is_keyword_js_constant.c
new file mode 100644
index 00000000..5ce7694d
--- /dev/null
+++ b/llamafile/is_keyword_js_constant.c
@@ -0,0 +1,139 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_js_constant.gperf */
+/* Computed positions: -k'1' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_js_constant.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 7
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 10
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 10
+/* maximum key range = 8, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 0, 11, 11, 11, 11, 0, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 0, 0, 11, 11, 11, 11, 11, 11,
+ 2, 11, 11, 11, 11, 11, 0, 0, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11
+ };
+ return len + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_js_constant (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("NaN")];
+ char stringpool_str4[sizeof("true")];
+ char stringpool_str5[sizeof("false")];
+ char stringpool_str6[sizeof("null")];
+ char stringpool_str8[sizeof("Infinity")];
+ char stringpool_str9[sizeof("undefined")];
+ char stringpool_str10[sizeof("globalThis")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "NaN",
+ "true",
+ "false",
+ "null",
+ "Infinity",
+ "undefined",
+ "globalThis"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_js_constant.gperf b/llamafile/is_keyword_js_constant.gperf
new file mode 100644
index 00000000..26e8a0d3
--- /dev/null
+++ b/llamafile/is_keyword_js_constant.gperf
@@ -0,0 +1,16 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_js_constant
+%%
+Infinity
+NaN
+false
+globalThis
+null
+true
+undefined
diff --git a/llamafile/is_keyword_typescript.c b/llamafile/is_keyword_typescript.c
new file mode 100644
index 00000000..9eb5270c
--- /dev/null
+++ b/llamafile/is_keyword_typescript.c
@@ -0,0 +1,302 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_typescript.gperf */
+/* Computed positions: -k'1,$' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_typescript.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 55
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 10
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 99
+/* maximum key range = 97, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100, 40, 0, 15,
+ 25, 5, 10, 10, 50, 15,100, 0, 0, 35,
+ 25, 50, 45,100, 20, 5, 0,100, 0, 45,
+ 100, 35,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100,100,100,100,100,
+ 100,100,100,100,100,100
+ };
+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_typescript (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("let")];
+ char stringpool_str5[sizeof("break")];
+ char stringpool_str6[sizeof("target")];
+ char stringpool_str8[sizeof("set")];
+ char stringpool_str9[sizeof("type")];
+ char stringpool_str11[sizeof("export")];
+ char stringpool_str13[sizeof("get")];
+ char stringpool_str14[sizeof("else")];
+ char stringpool_str15[sizeof("keyof")];
+ char stringpool_str16[sizeof("typeof")];
+ char stringpool_str17[sizeof("extends")];
+ char stringpool_str19[sizeof("satisfies")];
+ char stringpool_str20[sizeof("const")];
+ char stringpool_str21[sizeof("import")];
+ char stringpool_str22[sizeof("is")];
+ char stringpool_str23[sizeof("var")];
+ char stringpool_str24[sizeof("case")];
+ char stringpool_str25[sizeof("class")];
+ char stringpool_str26[sizeof("static")];
+ char stringpool_str27[sizeof("if")];
+ char stringpool_str28[sizeof("continue")];
+ char stringpool_str29[sizeof("interface")];
+ char stringpool_str30[sizeof("implements")];
+ char stringpool_str32[sizeof("default")];
+ char stringpool_str33[sizeof("for")];
+ char stringpool_str35[sizeof("instanceof")];
+ char stringpool_str36[sizeof("delete")];
+ char stringpool_str37[sizeof("declare")];
+ char stringpool_str38[sizeof("try")];
+ char stringpool_str39[sizeof("namespace")];
+ char stringpool_str40[sizeof("infer")];
+ char stringpool_str42[sizeof("in")];
+ char stringpool_str43[sizeof("function")];
+ char stringpool_str44[sizeof("enum")];
+ char stringpool_str45[sizeof("await")];
+ char stringpool_str47[sizeof("as")];
+ char stringpool_str48[sizeof("abstract")];
+ char stringpool_str49[sizeof("from")];
+ char stringpool_str50[sizeof("throw")];
+ char stringpool_str51[sizeof("return")];
+ char stringpool_str52[sizeof("finally")];
+ char stringpool_str53[sizeof("debugger")];
+ char stringpool_str55[sizeof("while")];
+ char stringpool_str57[sizeof("private")];
+ char stringpool_str60[sizeof("async")];
+ char stringpool_str61[sizeof("switch")];
+ char stringpool_str62[sizeof("of")];
+ char stringpool_str63[sizeof("readonly")];
+ char stringpool_str65[sizeof("yield")];
+ char stringpool_str66[sizeof("public")];
+ char stringpool_str70[sizeof("catch")];
+ char stringpool_str73[sizeof("new")];
+ char stringpool_str77[sizeof("do")];
+ char stringpool_str79[sizeof("protected")];
+ char stringpool_str99[sizeof("with")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "let",
+ "break",
+ "target",
+ "set",
+ "type",
+ "export",
+ "get",
+ "else",
+ "keyof",
+ "typeof",
+ "extends",
+ "satisfies",
+ "const",
+ "import",
+ "is",
+ "var",
+ "case",
+ "class",
+ "static",
+ "if",
+ "continue",
+ "interface",
+ "implements",
+ "default",
+ "for",
+ "instanceof",
+ "delete",
+ "declare",
+ "try",
+ "namespace",
+ "infer",
+ "in",
+ "function",
+ "enum",
+ "await",
+ "as",
+ "abstract",
+ "from",
+ "throw",
+ "return",
+ "finally",
+ "debugger",
+ "while",
+ "private",
+ "async",
+ "switch",
+ "of",
+ "readonly",
+ "yield",
+ "public",
+ "catch",
+ "new",
+ "do",
+ "protected",
+ "with"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str65,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_typescript.gperf b/llamafile/is_keyword_typescript.gperf
new file mode 100644
index 00000000..99f7426d
--- /dev/null
+++ b/llamafile/is_keyword_typescript.gperf
@@ -0,0 +1,64 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_typescript
+%%
+abstract
+as
+async
+await
+break
+case
+catch
+class
+const
+continue
+debugger
+declare
+default
+delete
+do
+else
+enum
+export
+extends
+finally
+for
+from
+function
+get
+if
+implements
+import
+in
+infer
+instanceof
+interface
+is
+keyof
+let
+namespace
+new
+of
+private
+protected
+public
+readonly
+return
+satisfies
+set
+static
+switch
+target
+throw
+try
+type
+typeof
+var
+while
+with
+yield
diff --git a/llamafile/is_keyword_typescript_type.c b/llamafile/is_keyword_typescript_type.c
new file mode 100644
index 00000000..bc37d317
--- /dev/null
+++ b/llamafile/is_keyword_typescript_type.c
@@ -0,0 +1,150 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_typescript_type.gperf */
+/* Computed positions: -k'2' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_typescript_type.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 10
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 7
+#define MIN_HASH_VALUE 4
+#define MAX_HASH_VALUE 21
+/* maximum key range = 18, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 4, 22,
+ 22, 0, 22, 22, 22, 15, 22, 22, 22, 22,
+ 5, 0, 22, 22, 22, 22, 10, 5, 22, 22,
+ 22, 0, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 22, 22, 22
+ };
+ return len + asso_values[(unsigned char)str[1]];
+}
+
+const char *
+is_keyword_typescript_type (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str4[sizeof("void")];
+ char stringpool_str5[sizeof("never")];
+ char stringpool_str6[sizeof("symbol")];
+ char stringpool_str7[sizeof("boolean")];
+ char stringpool_str8[sizeof("any")];
+ char stringpool_str10[sizeof("object")];
+ char stringpool_str11[sizeof("number")];
+ char stringpool_str12[sizeof("unknown")];
+ char stringpool_str16[sizeof("string")];
+ char stringpool_str21[sizeof("bigint")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "void",
+ "never",
+ "symbol",
+ "boolean",
+ "any",
+ "object",
+ "number",
+ "unknown",
+ "string",
+ "bigint"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_typescript_type.gperf b/llamafile/is_keyword_typescript_type.gperf
new file mode 100644
index 00000000..4a7a42e2
--- /dev/null
+++ b/llamafile/is_keyword_typescript_type.gperf
@@ -0,0 +1,19 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_typescript_type
+%%
+any
+bigint
+boolean
+never
+number
+object
+string
+symbol
+unknown
+void