Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add TypeScript syntax highlighting
Justine Tunney7db4b98cadae
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 4adcb3e0..8c266bb4 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -175,7 +175,6 @@ o/$(MODE)/llamafile/highlight_c_test: \o/$(MODE)/llamafile/highlight_c.o \o/$(MODE)/llamafile/is_keyword_c.o \o/$(MODE)/llamafile/is_keyword_c_type.o \- o/$(MODE)/llamafile/is_keyword_js.o \o/$(MODE)/llamafile/highlight_python_test: \o/$(MODE)/llamafile/highlight_python_test.o \diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cppindex eecfe7d1..5105840d 100644--- a/llamafile/chatbot.cpp+++ b/llamafile/chatbot.cpp@@ -304,6 +304,7 @@ int chatbot_main(int argc, char **argv) {// perform important setupHighlightMarkdown highlighter;+ ColorBleeder bleeder(&highlighter);struct llama_sampling_context *sampler = llama_sampling_init(params.sparams);signal(SIGINT, on_sigint);@@ -336,7 +337,7 @@ int chatbot_main(int argc, char **argv) {if (llama_token_is_eog(g_model, id))break;std::string s;- highlighter.feed(&s, llama_token_to_piece(g_ctx, id, params.special));+ bleeder.feed(&s, llama_token_to_piece(g_ctx, id, params.special));printf("%s", s.c_str());fflush(stdout);eval_id(id);@@ -344,7 +345,7 @@ int chatbot_main(int argc, char **argv) {g_got_sigint = 0;free(line);std::string s;- highlighter.flush(&s);+ bleeder.flush(&s);printf("%s\n", s.c_str());}diff --git a/llamafile/color_bleeder.cpp b/llamafile/color_bleeder.cppnew file mode 100644index 00000000..d7ca5a52--- /dev/null+++ b/llamafile/color_bleeder.cpp@@ -0,0 +1,160 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++// the `less` command was intentionally designed to prevent ansi sgr+// codes from bleeding across lines. for example, saying:+//+// printf '\e[1m line 1 \n line 2 \e[0m \n'+//+// both lines will appear bold in the terminal. however with:+//+// printf '\e[1m line 1 \n line 2 \e[0m \n' | less+//+// only the first line will appear bold. this is because less has an+// implicit \e[0m at the end of each line. this is bad, because our+// highlighting code assumes that, when emitting things like comments on+// multiple lines, we only need to set the color at the beginning.+//+// this class may be used to compose a Highlighter object. it'll relay+// the emitted bytes through, taking special care to restore the ansi+// sgr state at the beginning of each line. thus less will now work.+//+// only the subset of ansi escape codes that we use are supported.++enum {+ NORMAL,+ ESC,+ CSI,+};++ColorBleeder::ColorBleeder(Highlight *h) : h_(h) {+}++ColorBleeder::~ColorBleeder() {+}++void ColorBleeder::relay(std::string *r, const std::string &s) {+ for (char c : s) {+ *r += c;+ switch (t_) {++ Normal:+ case NORMAL:+ switch (c) {+ case 033:+ t_ = ESC;+ break;+ case '\n':+ if (intensity_ || foreground_ || background_) {+ bool got_some = false;+ *r += "\033[";+ if (intensity_) {+ *r += std::to_string(intensity_);+ got_some = true;+ }+ if (foreground_) {+ if (got_some)+ *r += ';';+ *r += std::to_string(foreground_);+ got_some = true;+ }+ if (background_) {+ if (got_some)+ *r += ';';+ *r += std::to_string(background_);+ }+ *r += 'm';+ }+ break;+ }+ break;++ case ESC:+ switch (c) {+ case '[':+ t_ = CSI;+ n_ = 0;+ x_ = 0;+ break;+ default:+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case CSI:+ if (isdigit(c)) {+ x_ *= 10;+ x_ += c - '0';+ } else if (c == ';') {+ if (n_ < sizeof(sgr_codes_)) {+ sgr_codes_[n_++] = x_;+ x_ = 0;+ }+ } else if (c == 'm') {+ if (n_ < sizeof(sgr_codes_)) {+ sgr_codes_[n_++] = x_;+ x_ = 0;+ }+ for (int i = 0; i < n_; ++i) {+ int g = sgr_codes_[i];+ if (g == 0) {+ intensity_ = 0;+ foreground_ = 0;+ background_ = 0;+ } else if (g == 1 || g == 2) {+ intensity_ = g;+ } else if (g == 22) {+ intensity_ = 0;+ } else if ((30 <= g && g <= 37) || //+ (90 <= g && g <= 97)) {+ foreground_ = g;+ } else if (g == 39) {+ foreground_ = 0;+ } else if ((40 <= g && g <= 47) || //+ (100 <= g && g <= 107)) {+ background_ = g;+ } else if (g == 49) {+ background_ = 0;+ }+ }+ t_ = NORMAL;+ } else {+ t_ = NORMAL;+ goto Normal;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void ColorBleeder::feed(std::string *r, std::string_view input) {+ std::string s;+ h_->feed(&s, input);+ relay(r, s);+}++void ColorBleeder::flush(std::string *r) {+ std::string s;+ h_->flush(&s);+ relay(r, s);+}diff --git a/llamafile/high.cpp b/llamafile/high.cppindex eaf77775..6e140954 100644--- a/llamafile/high.cpp+++ b/llamafile/high.cpp@@ -72,6 +72,7 @@ int main(int argc, char *argv[]) {// process inputstd::string res;+ ColorBleeder H(h);for (;;) {// read input chunk@@ -87,7 +88,7 @@ int main(int argc, char *argv[]) {// highlight chunkres.clear();- h->feed(&res, std::string_view(buf, got));+ H.feed(&res, std::string_view(buf, got));// write highlighted output chunkwrite(outfd, res.data(), res.size());@@ -95,6 +96,6 @@ int main(int argc, char *argv[]) {// flush highlighterres.clear();- h->flush(&res);+ H.flush(&res);write(outfd, res.data(), res.size());}diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 233b3b7d..450fa47e 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -40,7 +40,10 @@ Highlight *Highlight::create(const std::string_view &lang) {} else if (lang == "js" || //lang == "json" || //lang == "javascript") {- return new HighlightC(is_keyword_js);+ return new HighlightJs(is_keyword_js, nullptr);+ } else if (lang == "ts" || //+ lang == "typescript") {+ return new HighlightJs(is_keyword_typescript, is_keyword_typescript_type);} else if (lang == "java") {return new HighlightC(is_keyword_java);} else if (lang == "py" || //@@ -60,7 +63,7 @@ Highlight *Highlight::create(const std::string_view &lang) {lang == "pascal") {return new HighlightPascal;} else if (lang == "go") {- return new HighlightC(is_keyword_go, is_keyword_go_type);+ return new HighlightGo;} else if (lang == "sql") {return new HighlightSql;} else if (lang == "css") {diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex ca6f88b0..132c92d1 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -51,6 +51,8 @@ is_keyword_f is_keyword_c_builtin;is_keyword_f is_keyword_c_constant;is_keyword_f is_keyword_cxx;is_keyword_f is_keyword_js;+is_keyword_f is_keyword_js_builtin;+is_keyword_f is_keyword_js_constant;is_keyword_f is_keyword_java;is_keyword_f is_keyword_python;is_keyword_f is_keyword_rust;@@ -90,6 +92,9 @@ is_keyword_f is_keyword_tcl_builtin;is_keyword_f is_keyword_ruby;is_keyword_f is_keyword_ruby_builtin;is_keyword_f is_keyword_ruby_constant;+is_keyword_f is_keyword_typescript;+is_keyword_f is_keyword_typescript_type;+is_keyword_f is_keyword_typescript_constant;}class Highlight {@@ -108,10 +113,29 @@ class HighlightPlain : public Highlight {void flush(std::string *result) override;};+class ColorBleeder : public Highlight {+ public:+ ColorBleeder(Highlight *h);+ ~ColorBleeder() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ void relay(std::string *r, const std::string &s);+ unsigned char t_ = 0;+ unsigned char x_ = 0;+ unsigned char n_ = 0;+ unsigned char intensity_ = 0;+ unsigned char foreground_ = 0;+ unsigned char background_ = 0;+ unsigned char sgr_codes_[8];+ Highlight *h_;+};+class HighlightC : public Highlight {public:- HighlightC(is_keyword_f is_keyword = is_keyword_c, is_keyword_f is_type = nullptr,- is_keyword_f is_builtin = nullptr, is_keyword_f is_constant = nullptr);+ HighlightC(is_keyword_f *is_keyword = is_keyword_c, is_keyword_f *is_type = nullptr,+ is_keyword_f *is_builtin = nullptr, is_keyword_f *is_constant = nullptr);~HighlightC() override;void feed(std::string *result, std::string_view input) override;void flush(std::string *result) override;@@ -127,6 +151,33 @@ class HighlightC : public Highlight {is_keyword_f *is_constant_;};+class HighlightGo : public Highlight {+ public:+ HighlightGo();+ ~HighlightGo() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightJs : public Highlight {+ public:+ HighlightJs(is_keyword_f *is_keyword, is_keyword_f *is_type);+ ~HighlightJs() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ int expect_;+ std::string word_;+ is_keyword_f *is_type_;+ is_keyword_f *is_keyword_;+};+class HighlightPython : public Highlight {public:HighlightPython();@@ -302,6 +353,7 @@ class HighlightHaskell : public Highlight {private:int t_ = 0;+ int level_ = 0;std::string symbol_;};@@ -315,6 +367,11 @@ class HighlightPerl : public Highlight {private:int t_ = 0;int i_ = 0;+ int c_ = 0;+ int last_ = 0;+ int expect_ = 0;+ unsigned char opener_ = 0;+ unsigned char closer_ = 0;bool pending_heredoc_ = false;bool indented_heredoc_ = false;std::string word_;diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cppindex 1be0706c..2175485f 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight_c.cpp@@ -30,8 +30,6 @@ enum {SLASH_SLASH,SLASH_STAR,SLASH_STAR_STAR,- TICK,- TICK_BACKSLASH,R,R_DQUOTE,RAW,@@ -39,8 +37,8 @@ enum {HighlightC::HighlightC(is_keyword_f *is_keyword, //is_keyword_f *is_type, //- is_keyword_f is_builtin, //- is_keyword_f is_constant)+ is_keyword_f *is_builtin, //+ is_keyword_f *is_constant): is_keyword_(is_keyword),is_type_(is_type),is_builtin_(is_builtin),@@ -73,10 +71,6 @@ void HighlightC::feed(std::string *r, std::string_view input) {t_ = DQUOTE;*r += HI_STRING;*r += c;- } else if (c == '`' && is_keyword_ == is_keyword_js) {- t_ = TICK;- *r += HI_STRING;- *r += c;} else {*r += c;}@@ -184,21 +178,6 @@ void HighlightC::feed(std::string *r, std::string_view input) {t_ = DQUOTE;break;- case TICK:- *r += c;- if (c == '`') {- *r += HI_RESET;- t_ = NORMAL;- } else if (c == '\\') {- t_ = TICK_BACKSLASH;- }- break;-- case TICK_BACKSLASH:- *r += c;- t_ = TICK;- break;-case R:if (c == '"') {t_ = R_DQUOTE;@@ -272,8 +251,6 @@ void HighlightC::flush(std::string *r) {case R:*r += 'R';break;- case TICK:- case TICK_BACKSLASH:case QUOTE:case QUOTE_BACKSLASH:case DQUOTE:diff --git a/llamafile/highlight_go.cpp b/llamafile/highlight_go.cppnew file mode 100644index 00000000..64aa5612--- /dev/null+++ b/llamafile/highlight_go.cpp@@ -0,0 +1,222 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ QUOTE,+ QUOTE_BACKSLASH,+ DQUOTE,+ DQUOTE_BACKSLASH,+ SLASH,+ SLASH_SLASH,+ SLASH_STAR,+ SLASH_STAR_STAR,+ TICK,+ TICK_BACKSLASH,+};++HighlightGo::HighlightGo() {+}++HighlightGo::~HighlightGo() {+}++void HighlightGo::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '/') {+ t_ = SLASH;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '`') {+ t_ = TICK;+ *r += HI_STRING;+ *r += c;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_go(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_go_type(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH:+ if (c == '/') {+ *r += HI_COMMENT;+ *r += "//";+ t_ = SLASH_SLASH;+ } else if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ = SLASH_STAR;+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH_SLASH:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*')+ t_ = SLASH_STAR_STAR;+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c != '*') {+ t_ = SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE_BACKSLASH;+ }+ break;++ case DQUOTE_BACKSLASH:+ *r += c;+ t_ = DQUOTE;+ break;++ case TICK:+ *r += c;+ if (c == '`') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = TICK_BACKSLASH;+ }+ break;++ case TICK_BACKSLASH:+ *r += c;+ t_ = TICK;+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightGo::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_go(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_go_type(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case SLASH:+ *r += '/';+ break;+ case QUOTE:+ case QUOTE_BACKSLASH:+ case DQUOTE:+ case DQUOTE_BACKSLASH:+ case SLASH_SLASH:+ case SLASH_STAR:+ case SLASH_STAR_STAR:+ case TICK:+ case TICK_BACKSLASH:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_haskell.cpp b/llamafile/highlight_haskell.cppindex 59b21509..1120b122 100644--- a/llamafile/highlight_haskell.cpp+++ b/llamafile/highlight_haskell.cpp@@ -22,8 +22,6 @@enum {NORMAL,WORD,- QUOTE,- QUOTE_BACKSLASH,DQUOTE,DQUOTE_BACKSLASH,TICK,@@ -31,6 +29,7 @@ enum {CURL,CURL_HYPHEN,CURL_HYPHEN_HYPHEN,+ CURL_HYPHEN_CURL,HYPHEN,HYPHEN_HYPHEN,HYPHEN_LT,@@ -60,10 +59,6 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {t_ = HYPHEN;} else if (c == '{') {t_ = CURL;- } else if (c == '\'') {- t_ = QUOTE;- *r += HI_STRING;- *r += c;} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;@@ -204,6 +199,7 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {*r += HI_COMMENT;*r += "{-";t_ = CURL_HYPHEN;+ level_ = 1;} else {*r += '{';t_ = NORMAL;@@ -213,35 +209,35 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {case CURL_HYPHEN:*r += c;- if (c == '-')+ if (c == '-') {t_ = CURL_HYPHEN_HYPHEN;+ } else if (c == '{') {+ t_ = CURL_HYPHEN_CURL;+ }break;- case CURL_HYPHEN_HYPHEN:+ case CURL_HYPHEN_CURL:*r += c;- if (c == '}') {- *r += HI_RESET;- t_ = NORMAL;- } else if (c != '-') {+ if (c == '-') {+ t_ = CURL_HYPHEN;+ ++level_;+ } else if (c != '{') {t_ = CURL_HYPHEN;}break;- case QUOTE:+ case CURL_HYPHEN_HYPHEN:*r += c;- if (c == '\'') {- *r += HI_RESET;- t_ = NORMAL;- } else if (c == '\\') {- t_ = QUOTE_BACKSLASH;+ if (c == '}') {+ if (!--level_) {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ } else if (c != '-') {+ t_ = CURL_HYPHEN;}break;- case QUOTE_BACKSLASH:- *r += c;- t_ = QUOTE;- break;-case DQUOTE:*r += c;if (c == '"') {@@ -312,13 +308,12 @@ void HighlightHaskell::flush(std::string *r) {break;case TICK:case TICK_BACKSLASH:- case QUOTE:- case QUOTE_BACKSLASH:case DQUOTE:case DQUOTE_BACKSLASH:case HYPHEN_HYPHEN:case CURL_HYPHEN:case CURL_HYPHEN_HYPHEN:+ case CURL_HYPHEN_CURL:*r += HI_RESET;break;default:diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight_js.cppnew file mode 100644index 00000000..fa4d236a--- /dev/null+++ b/llamafile/highlight_js.cpp@@ -0,0 +1,290 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ QUOTE,+ QUOTE_BACKSLASH,+ DQUOTE,+ DQUOTE_BACKSLASH,+ SLASH,+ SLASH_SLASH,+ SLASH_STAR,+ SLASH_STAR_STAR,+ TICK,+ TICK_BACKSLASH,+ REGEX,+ REGEX_BACKSLASH,+};++enum {+ EXPECT_VALUE,+ EXPECT_OPERATOR,+};++HighlightJs::HighlightJs(is_keyword_f *is_keyword, is_keyword_f *is_type)+ : expect_(EXPECT_VALUE), is_keyword_(is_keyword), is_type_(is_type) {+}++HighlightJs::~HighlightJs() {+}++void HighlightJs::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '/') {+ t_ = SLASH;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ expect_ = EXPECT_OPERATOR;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ expect_ = EXPECT_OPERATOR;+ } else if (c == '`') {+ t_ = TICK;+ *r += HI_STRING;+ *r += c;+ expect_ = EXPECT_OPERATOR;+ } else if (c == ')' || c == '}' || c == ']') {+ expect_ = EXPECT_OPERATOR;+ *r += c;+ } else if (ispunct(c)) {+ expect_ = EXPECT_VALUE;+ *r += c;+ } else if (isdigit(c)) {+ expect_ = EXPECT_OPERATOR;+ *r += c;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ expect_ = EXPECT_VALUE;+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;+ expect_ = EXPECT_VALUE;+ } else if (is_keyword_js_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ expect_ = EXPECT_OPERATOR;+ } else if (is_keyword_js_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ expect_ = EXPECT_OPERATOR;+ } else {+ *r += word_;+ expect_ = EXPECT_OPERATOR;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH:+ if (c == '/') {+ *r += HI_COMMENT;+ *r += "//";+ t_ = SLASH_SLASH;+ } else if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ = SLASH_STAR;+ } else if (expect_ == EXPECT_VALUE) {+ expect_ = EXPECT_OPERATOR;+ *r += HI_STRING;+ *r += '/';+ *r += c;+ if (c == '\\') {+ t_ = REGEX_BACKSLASH;+ } else {+ t_ = REGEX;+ }+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH_SLASH:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*')+ t_ = SLASH_STAR_STAR;+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c != '*') {+ t_ = SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE_BACKSLASH;+ }+ break;++ case DQUOTE_BACKSLASH:+ *r += c;+ t_ = DQUOTE;+ break;++ case TICK:+ *r += c;+ if (c == '`') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = TICK_BACKSLASH;+ }+ break;++ case TICK_BACKSLASH:+ *r += c;+ t_ = TICK;+ break;++ case REGEX:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = REGEX_BACKSLASH;+ }+ break;++ case REGEX_BACKSLASH:+ *r += c;+ t_ = REGEX;+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightJs::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_type_ && is_type_(word_.data(), word_.size())) {+ *r += HI_TYPE;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_js_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_js_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case SLASH:+ *r += '/';+ break;+ case TICK:+ case TICK_BACKSLASH:+ case QUOTE:+ case QUOTE_BACKSLASH:+ case DQUOTE:+ case DQUOTE_BACKSLASH:+ case SLASH_SLASH:+ case SLASH_STAR:+ case SLASH_STAR_STAR:+ case REGEX:+ case REGEX_BACKSLASH:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_lua.cpp b/llamafile/highlight_lua.cppindex d393f3ed..fa95127b 100644--- a/llamafile/highlight_lua.cpp+++ b/llamafile/highlight_lua.cpp@@ -19,6 +19,8 @@#include <ctype.h>+// TODO: Support --[[ syntax+enum {NORMAL,WORD,diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cppindex fe2465c5..98559c1a 100644--- a/llamafile/highlight_perl.cpp+++ b/llamafile/highlight_perl.cpp@@ -37,8 +37,108 @@ enum {LT_LT_QNAME,HEREDOC_BOL,HEREDOC,+ REGEX,+ REGEX_BACKSLASH,+ S_REGEX,+ S_REGEX_BACKSLASH,+ S_REGEX_S,+ S_REGEX_S_BACKSLASH,+ EQUAL,};+enum {+ EXPECT_VALUE,+ EXPECT_OPERATOR,+};++static int mirror(int c) {+ switch (c) {+ case '(':+ return ')';+ case '{':+ return '}';+ case '[':+ return ']';+ case '<':+ return '>';+ default:+ return c;+ }+}++static bool is_magic_var(int c) {+ switch (c) {+ case '!':+ case '"':+ case '#':+ case '&':+ case '-':+ case '/':+ case '<':+ case '=':+ case '>':+ case '?':+ case '@':+ case '\'':+ case '\\':+ case '^':+ case '_':+ case '`':+ return true;+ default:+ return false;+ }+}++static bool is_regex_punct(int c) {+ switch (c) {+ case '!':+ case '"':+ case '#':+ case '%':+ case '&':+ case '(':+ case '*':+ case ',':+ case '-':+ case '.':+ case '/':+ case ':':+ case ';':+ case '<':+ case '=':+ case '@':+ case '[':+ case '\'':+ case '^':+ case '`':+ case '{':+ case '|':+ case '~':+ return true;+ default:+ return false;+ }+}++static bool is_regex_prefix(const std::string_view &s) {+ return s == "m" || //+ s == "s" || //+ s == "y" || //+ s == "q" || //+ s == "tr" || //+ s == "qq" || //+ s == "qw" || //+ s == "qx" || //+ s == "qr";+}++static bool is_double_regex(const std::string_view &s) {+ return s == "s" || //+ s == "y" || //+ s == "tr";+}+HighlightPerl::HighlightPerl() {}@@ -46,9 +146,10 @@ HighlightPerl::~HighlightPerl() {}void HighlightPerl::feed(std::string *r, std::string_view input) {- int c;+ int c = 0;for (size_t i = 0; i < input.size(); ++i) {- c = input[i] & 255;+ last_ = c_;+ c_ = c = input[i] & 255;switch (t_) {Normal:@@ -60,21 +161,28 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {t_ = QUOTE;*r += HI_STRING;*r += c;+ expect_ = EXPECT_OPERATOR;} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;*r += c;+ expect_ = EXPECT_OPERATOR;+ } else if (c == '=' && (!last_ || last_ == '\n')) {+ t_ = EQUAL;} else if (c == '`') {t_ = TICK;*r += HI_STRING;*r += c;+ expect_ = EXPECT_OPERATOR;} else if (c == '$') {*r += c;t_ = VAR;+ expect_ = EXPECT_OPERATOR;} else if (c == '@' || c == '%') {*r += c;*r += HI_VAR;t_ = VAR2;+ expect_ = EXPECT_OPERATOR;} else if (c == '#') {*r += HI_COMMENT;*r += c;@@ -82,6 +190,14 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {} else if (c == '<') {*r += c;t_ = LT;+ expect_ = EXPECT_VALUE;+ } else if (c == '/' && expect_ == EXPECT_VALUE && last_ != '/') {+ opener_ = '/';+ closer_ = '/';+ expect_ = EXPECT_OPERATOR;+ *r += HI_STRING;+ *r += c;+ t_ = REGEX;} else if (c == '\n') {*r += c;if (pending_heredoc_) {@@ -90,11 +206,21 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {t_ = HEREDOC_BOL;i_ = 0;}+ } else if (c == ')' || c == '}' || c == ']') {+ expect_ = EXPECT_OPERATOR;+ *r += c;+ } else if (ispunct(c)) {+ expect_ = EXPECT_VALUE;+ *r += c;+ } else if (isdigit(c)) {+ expect_ = EXPECT_OPERATOR;+ *r += c;} else {*r += c;}break;+ Word:case WORD:if (!isascii(c) || isalnum(c) || c == '_') {word_ += c;@@ -103,8 +229,27 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;+ if (word_ == "shift") {+ expect_ = EXPECT_OPERATOR;+ } else {+ expect_ = EXPECT_VALUE;+ }} else {*r += word_;+ expect_ = EXPECT_VALUE;+ if (is_regex_punct(c) && is_regex_prefix(word_)) {+ opener_ = c;+ closer_ = mirror(c);+ *r += HI_STRING;+ *r += c;+ if (is_double_regex(word_)) {+ t_ = S_REGEX;+ } else {+ t_ = REGEX;+ }+ word_.clear();+ break;+ }}word_.clear();t_ = NORMAL;@@ -113,13 +258,20 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case VAR:- t_ = VAR2;- if (c == '{') {+ if (isdigit(c) || is_magic_var(c)) {+ *r += HI_VAR;+ *r += c;+ *r += HI_RESET;+ t_ = NORMAL;+ break;+ } else if (c == '{') {+ t_ = VAR2;*r += c;*r += HI_VAR;break;} else {*r += HI_VAR;+ t_ = VAR2;}// fallthrough@@ -143,6 +295,50 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {}break;+ case REGEX:+ *r += c;+ if (c == closer_) {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = REGEX_BACKSLASH;+ }+ break;++ case REGEX_BACKSLASH:+ *r += c;+ t_ = REGEX;+ break;++ case S_REGEX:+ *r += c;+ if (c == opener_) {+ t_ = S_REGEX_S;+ } else if (c == '\\') {+ t_ = S_REGEX_BACKSLASH;+ }+ break;++ case S_REGEX_BACKSLASH:+ *r += c;+ t_ = S_REGEX;+ break;++ case S_REGEX_S:+ *r += c;+ if (c == closer_) {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = S_REGEX_S_BACKSLASH;+ }+ break;++ case S_REGEX_S_BACKSLASH:+ *r += c;+ t_ = S_REGEX_S;+ break;+case QUOTE:*r += c;if (c == '\'') {@@ -188,6 +384,21 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {t_ = TICK;break;+ case EQUAL:+ if (isalpha(c)) {+ *r += HI_COMMENT;+ *r += '=';+ *r += c;+ heredoc_ = "=cut";+ t_ = HEREDOC;+ i_ = 0;+ } else {+ *r += '=';+ t_ = NORMAL;+ goto Normal;+ }+ break;+case LT:if (c == '<') {*r += c;@@ -205,7 +416,8 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {if (c == '-') {indented_heredoc_ = true;*r += c;- } else if (c == '\'') {+ } else if (c == '"' || c == '\'') {+ closer_ = c;t_ = LT_LT_QNAME;*r += HI_STRING;*r += c;@@ -237,7 +449,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case LT_LT_QNAME:*r += c;- if (c == '\'') {+ if (c == closer_) {*r += HI_RESET;t_ = HEREDOC_BOL;pending_heredoc_ = true;@@ -267,8 +479,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case HEREDOC:*r += c;- if (c == '\n')+ if (c == '\n') {t_ = HEREDOC_BOL;+ i_ = 0;+ }break;default:@@ -289,6 +503,9 @@ void HighlightPerl::flush(std::string *r) {}word_.clear();break;+ case EQUAL:+ *r += '=';+ break;case VAR2:case TICK:case TICK_BACKSLASH:@@ -300,6 +517,12 @@ void HighlightPerl::flush(std::string *r) {case HEREDOC_BOL:case HEREDOC:case LT_LT_QNAME:+ case REGEX:+ case REGEX_BACKSLASH:+ case S_REGEX:+ case S_REGEX_BACKSLASH:+ case S_REGEX_S:+ case S_REGEX_S_BACKSLASH:*r += HI_RESET;break;default:diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cppindex 75f72e5b..82a8bed7 100644--- a/llamafile/highlight_ruby.cpp+++ b/llamafile/highlight_ruby.cpp@@ -366,6 +366,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case DOLLAR:if (isdigit(c) || //c == '!' || //+ c == '"' || //c == '#' || //c == '$' || //c == '&' || //@@ -375,6 +376,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {c == '=' || //c == '>' || //c == '@' || //+ c == '\'' || //c == '\\' || //c == '^' || //c == '_' || //@@ -383,7 +385,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {*r += '$';*r += c;*r += HI_RESET;- t_ = AT_WORD;+ t_ = NORMAL;} else if (isalpha(c)) {*r += HI_VAR;*r += '$';@@ -487,7 +489,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {t_ = LT_LT_NAME;heredoc_ += c;*r += c;- } else if (!isblank(c)) {+ } else {t_ = NORMAL;goto Normal;}diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex 0ab73bac..ce172712 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -56,6 +56,7 @@ const char *const kLanguages[] = {"sql", //"swift", //"tcl", //+ "typescript", //"zig", //};diff --git a/llamafile/is_keyword_cxx.c b/llamafile/is_keyword_cxx.cindex 122b1d6c..65a2eada 100644--- a/llamafile/is_keyword_cxx.c+++ b/llamafile/is_keyword_cxx.c@@ -33,7 +33,7 @@#include <string.h>-#define TOTAL_KEYWORDS 167+#define TOTAL_KEYWORDS 165#define MIN_WORD_LENGTH 2#define MAX_WORD_LENGTH 27#define MIN_HASH_VALUE 3@@ -55,16 +55,16 @@ hash (register const char *str, register size_t len)451, 451, 451, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 451, 451,- 451, 451, 25, 451, 451, 451, 451, 451, 451, 451,- 451, 451, 451, 451, 451, 451, 451, 451, 0, 60,- 50, 30, 35, 451, 25, 451, 5, 451, 451, 451,+ 451, 451, 30, 451, 451, 451, 451, 451, 451, 451,+ 451, 451, 451, 451, 451, 451, 451, 451, 10, 60,+ 50, 30, 35, 451, 25, 451, 0, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 0, 10,- 5, 451, 451, 451, 451, 451, 451, 451, 5, 451,+ 10, 451, 451, 451, 451, 451, 451, 451, 5, 451,5, 451, 451, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 0, 451, 15, 130, 0,20, 10, 80, 145, 140, 45, 451, 30, 70, 35,- 0, 65, 95, 125, 5, 15, 0, 110, 105, 140,- 85, 5, 5, 451, 451, 451, 451, 451, 451, 451,+ 0, 65, 95, 120, 5, 15, 0, 110, 85, 140,+ 85, 5, 10, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 451, 451,451, 451, 451, 451, 451, 451, 451, 451, 451, 451,@@ -116,11 +116,11 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str18[sizeof("noexcept")];char stringpool_str19[sizeof("co_return")];char stringpool_str21[sizeof("extern")];- char stringpool_str22[sizeof("__FUNCTION__")];+ char stringpool_str22[sizeof("char8_t")];char stringpool_str23[sizeof("__extension__")];char stringpool_str24[sizeof("char")];char stringpool_str26[sizeof("struct")];- char stringpool_str27[sizeof("char8_t")];+ char stringpool_str27[sizeof("__FUNCTION__")];char stringpool_str28[sizeof("__attribute__")];char stringpool_str29[sizeof("case")];char stringpool_str30[sizeof("__restrict")];@@ -140,9 +140,9 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str51[sizeof("strlenesque")];char stringpool_str52[sizeof("returnspointerwithnoaliases")];char stringpool_str53[sizeof("char32_t")];- char stringpool_str54[sizeof("__real__ ")];char stringpool_str57[sizeof("__asm__")];char stringpool_str58[sizeof("and")];+ char stringpool_str59[sizeof("__real__ ")];char stringpool_str61[sizeof("interruptfn")];char stringpool_str62[sizeof("dontcallback")];char stringpool_str63[sizeof("continue")];@@ -165,10 +165,10 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str90[sizeof("__asm")];char stringpool_str92[sizeof("thatispacked")];char stringpool_str93[sizeof("for")];- char stringpool_str94[sizeof("__imag__ ")];+ char stringpool_str94[sizeof("_Float128")];char stringpool_str96[sizeof("__real")];char stringpool_str98[sizeof("xor")];- char stringpool_str99[sizeof("_Float128")];+ char stringpool_str99[sizeof("__imag__ ")];char stringpool_str100[sizeof("__forceinline__")];char stringpool_str102[sizeof("thread_local")];char stringpool_str103[sizeof("operator")];@@ -180,11 +180,11 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str112[sizeof("reallocesque")];char stringpool_str113[sizeof("typename")];char stringpool_str114[sizeof("threadIdx")];+ char stringpool_str115[sizeof("__device__")];char stringpool_str116[sizeof("export")];char stringpool_str118[sizeof("_Float16")];char stringpool_str120[sizeof("forcealignargpointer")];char stringpool_str123[sizeof("template")];- char stringpool_str124[sizeof("true")];char stringpool_str125[sizeof("__inline__")];char stringpool_str127[sizeof("if")];char stringpool_str128[sizeof("_Float64")];@@ -192,10 +192,9 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str130[sizeof("scanfesque")];char stringpool_str131[sizeof("inline")];char stringpool_str133[sizeof("__inline")];- char stringpool_str135[sizeof("__device__")];+ char stringpool_str136[sizeof("not_eq")];char stringpool_str138[sizeof("atomic_commit")];char stringpool_str140[sizeof("atomic_noexcept")];- char stringpool_str141[sizeof("not_eq")];char stringpool_str143[sizeof("_Float32")];char stringpool_str145[sizeof("bitor")];char stringpool_str146[sizeof("__null")];@@ -204,7 +203,7 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str149[sizeof("__alignof")];char stringpool_str150[sizeof("float")];char stringpool_str151[sizeof("friend")];- char stringpool_str154[sizeof("__float80")];+ char stringpool_str154[sizeof("void")];char stringpool_str155[sizeof("wontreturn")];char stringpool_str156[sizeof("bitand")];char stringpool_str158[sizeof("explicit")];@@ -212,14 +211,15 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str160[sizeof("union")];char stringpool_str161[sizeof("printfesque")];char stringpool_str163[sizeof("__host__")];+ char stringpool_str164[sizeof("__float80")];char stringpool_str166[sizeof("typeid")];char stringpool_str169[sizeof("dontthrow")];char stringpool_str170[sizeof("__signed__")];- char stringpool_str171[sizeof("sizeof")];+ char stringpool_str171[sizeof("and_eq")];char stringpool_str172[sizeof("private")];- char stringpool_str174[sizeof("void")];- char stringpool_str175[sizeof("false")];- char stringpool_str176[sizeof("and_eq")];+ char stringpool_str173[sizeof("volatile")];+ char stringpool_str175[sizeof("__volatile")];+ char stringpool_str176[sizeof("sizeof")];char stringpool_str177[sizeof("nullptr")];char stringpool_str178[sizeof("register")];char stringpool_str180[sizeof("compl")];@@ -229,11 +229,10 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str187[sizeof("synchronized")];char stringpool_str188[sizeof("__signed")];char stringpool_str191[sizeof("mallocesque")];- char stringpool_str193[sizeof("volatile")];- char stringpool_str195[sizeof("__volatile")];+ char stringpool_str193[sizeof("requires")];char stringpool_str196[sizeof("signed")];char stringpool_str197[sizeof("__launch_bounds__")];- char stringpool_str198[sizeof("requires")];+ char stringpool_str198[sizeof("__volatile__ ")];char stringpool_str199[sizeof("protected")];char stringpool_str202[sizeof("typedef")];char stringpool_str205[sizeof("break")];@@ -242,7 +241,6 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str209[sizeof("__label__")];char stringpool_str210[sizeof("while")];char stringpool_str211[sizeof("__imag")];- char stringpool_str213[sizeof("__volatile__ ")];char stringpool_str214[sizeof("goto")];char stringpool_str215[sizeof("privileged")];char stringpool_str216[sizeof("double")];@@ -251,23 +249,23 @@ is_keyword_cxx (register const char *str, register size_t len)char stringpool_str219[sizeof("long")];char stringpool_str220[sizeof("__global__")];char stringpool_str221[sizeof("__bf16")];+ char stringpool_str226[sizeof("xor_eq")];char stringpool_str229[sizeof("libcesque")];- char stringpool_str231[sizeof("xor_eq")];char stringpool_str232[sizeof("gridDim")];+ char stringpool_str241[sizeof("vallocesque")];char stringpool_str251[sizeof("textwindows")];- char stringpool_str261[sizeof("vallocesque")];char stringpool_str268[sizeof("blockDim")];char stringpool_str269[sizeof("bool")];char stringpool_str273[sizeof("__builtin_offsetof")];char stringpool_str276[sizeof("public")];+ char stringpool_str277[sizeof("virtual")];char stringpool_str283[sizeof("new")];char stringpool_str285[sizeof("catch")];char stringpool_str290[sizeof("throw")];char stringpool_str292[sizeof("wchar_t")];- char stringpool_str297[sizeof("virtual")];char stringpool_str298[sizeof("unsigned")];+ char stringpool_str310[sizeof("or_eq")];char stringpool_str318[sizeof("blockIdx")];- char stringpool_str320[sizeof("or_eq")];char stringpool_str336[sizeof("__builtin_va_arg")];char stringpool_str450[sizeof("using")];};@@ -286,11 +284,11 @@ is_keyword_cxx (register const char *str, register size_t len)"noexcept","co_return","extern",- "__FUNCTION__",+ "char8_t","__extension__","char","struct",- "char8_t",+ "__FUNCTION__","__attribute__","case","__restrict",@@ -310,9 +308,9 @@ is_keyword_cxx (register const char *str, register size_t len)"strlenesque","returnspointerwithnoaliases","char32_t",- "__real__ ","__asm__","and",+ "__real__ ","interruptfn","dontcallback","continue",@@ -335,10 +333,10 @@ is_keyword_cxx (register const char *str, register size_t len)"__asm","thatispacked","for",- "__imag__ ",+ "_Float128","__real","xor",- "_Float128",+ "__imag__ ","__forceinline__","thread_local","operator",@@ -350,11 +348,11 @@ is_keyword_cxx (register const char *str, register size_t len)"reallocesque","typename","threadIdx",+ "__device__","export","_Float16","forcealignargpointer","template",- "true","__inline__","if","_Float64",@@ -362,10 +360,9 @@ is_keyword_cxx (register const char *str, register size_t len)"scanfesque","inline","__inline",- "__device__",+ "not_eq","atomic_commit","atomic_noexcept",- "not_eq","_Float32","bitor","__null",@@ -374,7 +371,7 @@ is_keyword_cxx (register const char *str, register size_t len)"__alignof","float","friend",- "__float80",+ "void","wontreturn","bitand","explicit",@@ -382,14 +379,15 @@ is_keyword_cxx (register const char *str, register size_t len)"union","printfesque","__host__",+ "__float80","typeid","dontthrow","__signed__",- "sizeof",- "private",- "void",- "false","and_eq",+ "private",+ "volatile",+ "__volatile",+ "sizeof","nullptr","register","compl",@@ -399,11 +397,10 @@ is_keyword_cxx (register const char *str, register size_t len)"synchronized","__signed","mallocesque",- "volatile",- "__volatile",+ "requires","signed","__launch_bounds__",- "requires",+ "__volatile__ ","protected","typedef","break",@@ -412,7 +409,6 @@ is_keyword_cxx (register const char *str, register size_t len)"__label__","while","__imag",- "__volatile__ ","goto","privileged","double",@@ -421,23 +417,23 @@ is_keyword_cxx (register const char *str, register size_t len)"long","__global__","__bf16",- "libcesque","xor_eq",+ "libcesque","gridDim",- "textwindows","vallocesque",+ "textwindows","blockDim","bool","__builtin_offsetof","public",+ "virtual","new","catch","throw","wchar_t",- "virtual","unsigned",- "blockIdx","or_eq",+ "blockIdx","__builtin_va_arg","using"};@@ -495,11 +491,11 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,- -1, -1,+ -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,- -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,@@ -549,7 +545,7 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str112,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str113,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str114,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str118,@@ -557,7 +553,7 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str120,-1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str123,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str124,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str127,@@ -567,14 +563,13 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str131,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str133,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str135,-1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str138,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str140,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str141,- -1,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str143,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str145,@@ -595,15 +590,16 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str161,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str163,- -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str164,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str166,-1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str169,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str170,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str171,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str172,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str173,-1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str174,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str175,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str176,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str177,@@ -621,8 +617,7 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str191,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str193,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str195,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str196,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str197,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str198,@@ -637,8 +632,7 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str209,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str210,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str211,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str213,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str214,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str215,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str216,@@ -647,24 +641,26 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str219,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str220,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str221,- -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str226,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str229,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str231,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str232,- -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str241,-1, -1, -1, -1, -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str251,-1, -1, -1, -1, -1, -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str261,- -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str268,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str269,-1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str273,-1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str276,- -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str277,+ -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str283,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str285,@@ -672,17 +668,15 @@ is_keyword_cxx (register const char *str, register size_t len)(int)(size_t)&((struct stringpool_t *)0)->stringpool_str290,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str292,- -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str297,+ -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str298,-1, -1, -1, -1, -1, -1, -1, -1, -1,- -1, -1, -1, -1, -1, -1, -1, -1, -1,- -1,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str310,+ -1, -1, -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str318,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str320,-1, -1, -1, -1, -1, -1, -1, -1, -1,- -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str336,-1, -1, -1, -1, -1, -1, -1, -1, -1,-1, -1, -1, -1, -1, -1, -1, -1, -1,diff --git a/llamafile/is_keyword_cxx.gperf b/llamafile/is_keyword_cxx.gperfindex 00969515..099ba0cb 100644--- a/llamafile/is_keyword_cxx.gperf+++ b/llamafile/is_keyword_cxx.gperf@@ -96,7 +96,6 @@ enumexplicitexportextern-falsefloatforforcealign@@ -157,7 +156,6 @@ thisthreadIdxthread_localthrow-truetrytypedeftypeiddiff --git a/llamafile/is_keyword_go_type.c b/llamafile/is_keyword_go_type.cindex a42b8ac2..a991d39c 100644--- a/llamafile/is_keyword_go_type.c+++ b/llamafile/is_keyword_go_type.c@@ -33,7 +33,7 @@#include <string.h>-#define TOTAL_KEYWORDS 17+#define TOTAL_KEYWORDS 18#define MIN_WORD_LENGTH 3#define MAX_WORD_LENGTH 10#define MIN_HASH_VALUE 4@@ -62,8 +62,8 @@ hash (register const char *str, register size_t len)27, 27, 27, 27, 27, 27, 27, 27, 27, 27,27, 27, 27, 27, 27, 27, 27, 27, 27, 27,27, 27, 27, 27, 27, 27, 27, 27, 5, 10,- 27, 10, 0, 27, 27, 0, 27, 27, 27, 27,- 27, 27, 27, 27, 0, 27, 5, 0, 27, 27,+ 27, 10, 0, 0, 27, 0, 27, 27, 27, 27,+ 27, 27, 27, 27, 0, 0, 5, 0, 27, 27,27, 27, 27, 27, 27, 27, 27, 27, 27, 27,27, 27, 27, 27, 27, 27, 27, 27, 27, 27,27, 27, 27, 27, 27, 27, 27, 27, 27, 27,@@ -89,6 +89,7 @@ is_keyword_go_type (register const char *str, register size_t len){char stringpool_str4[sizeof("int8")];char stringpool_str5[sizeof("uint8")];+ char stringpool_str6[sizeof("string")];char stringpool_str7[sizeof("uintptr")];char stringpool_str8[sizeof("int")];char stringpool_str9[sizeof("uint")];@@ -109,6 +110,7 @@ is_keyword_go_type (register const char *str, register size_t len){"int8","uint8",+ "string","uintptr","int","uint",@@ -131,7 +133,7 @@ is_keyword_go_type (register const char *str, register size_t len)-1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,diff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/is_keyword_go_type.gperfindex ec5eada3..2667eab1 100644--- a/llamafile/is_keyword_go_type.gperf+++ b/llamafile/is_keyword_go_type.gperf@@ -18,6 +18,7 @@ int32int64int8rune+stringuintuint16uint32diff --git a/llamafile/is_keyword_js.c b/llamafile/is_keyword_js.cindex 3a4fd745..8e186680 100644--- a/llamafile/is_keyword_js.c+++ b/llamafile/is_keyword_js.c@@ -33,12 +33,12 @@#include <string.h>-#define TOTAL_KEYWORDS 39+#define TOTAL_KEYWORDS 36#define MIN_WORD_LENGTH 2#define MAX_WORD_LENGTH 10-#define MIN_HASH_VALUE 3-#define MAX_HASH_VALUE 55-/* maximum key range = 53, duplicates = 0 */+#define MIN_HASH_VALUE 2+#define MAX_HASH_VALUE 61+/* maximum key range = 60, duplicates = 0 */#ifdef __GNUC____inline@@ -52,32 +52,32 @@ hash (register const char *str, register size_t len){static const unsigned char asso_values[] ={- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 5, 45, 0,- 5, 0, 15, 56, 40, 0, 56, 56, 35, 15,- 25, 0, 56, 56, 0, 10, 0, 0, 45, 10,- 30, 25, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56, 56, 56, 56, 56,- 56, 56, 56, 56, 56, 56+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 0, 15, 0,+ 0, 0, 10, 62, 10, 0, 62, 62, 10, 5,+ 0, 20, 62, 62, 30, 0, 20, 40, 15, 15,+ 25, 35, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62, 62, 62, 62, 62,+ 62, 62, 62, 62, 62, 62};return len + asso_values[(unsigned char)str[1]] + asso_values[(unsigned char)str[0]];}@@ -87,138 +87,133 @@ is_keyword_js (register const char *str, register size_t len){struct stringpool_t{- char stringpool_str3[sizeof("try")];- char stringpool_str4[sizeof("true")];- char stringpool_str5[sizeof("const")];- char stringpool_str6[sizeof("return")];- char stringpool_str7[sizeof("do")];- char stringpool_str8[sizeof("continue")];- char stringpool_str9[sizeof("case")];- char stringpool_str10[sizeof("catch")];- char stringpool_str11[sizeof("delete")];- char stringpool_str12[sizeof("default")];- char stringpool_str13[sizeof("debugger")];- char stringpool_str14[sizeof("with")];- char stringpool_str15[sizeof("super")];- char stringpool_str16[sizeof("static")];- char stringpool_str17[sizeof("if")];- char stringpool_str18[sizeof("for")];+ char stringpool_str2[sizeof("in")];+ char stringpool_str3[sizeof("new")];+ char stringpool_str4[sizeof("case")];+ char stringpool_str5[sizeof("catch")];+ char stringpool_str6[sizeof("delete")];+ char stringpool_str7[sizeof("default")];+ char stringpool_str8[sizeof("debugger")];+ char stringpool_str10[sizeof("instanceof")];+ char stringpool_str11[sizeof("import")];+ char stringpool_str12[sizeof("if")];+ char stringpool_str13[sizeof("let")];+ char stringpool_str14[sizeof("else")];+ char stringpool_str15[sizeof("class")];+ char stringpool_str17[sizeof("finally")];+ char stringpool_str18[sizeof("var")];+ char stringpool_str19[sizeof("with")];char stringpool_str20[sizeof("await")];- char stringpool_str21[sizeof("import")];- char stringpool_str22[sizeof("finally")];- char stringpool_str23[sizeof("function")];- char stringpool_str25[sizeof("false")];- char stringpool_str26[sizeof("switch")];- char stringpool_str27[sizeof("in")];- char stringpool_str28[sizeof("new")];- char stringpool_str29[sizeof("null")];- char stringpool_str30[sizeof("yield")];- char stringpool_str31[sizeof("typeof")];- char stringpool_str35[sizeof("instanceof")];- char stringpool_str36[sizeof("export")];- char stringpool_str37[sizeof("extends")];- char stringpool_str38[sizeof("let")];- char stringpool_str39[sizeof("else")];- char stringpool_str40[sizeof("class")];- char stringpool_str44[sizeof("this")];- char stringpool_str45[sizeof("throw")];- char stringpool_str49[sizeof("void")];+ char stringpool_str21[sizeof("switch")];+ char stringpool_str22[sizeof("do")];+ char stringpool_str25[sizeof("const")];+ char stringpool_str26[sizeof("static")];+ char stringpool_str28[sizeof("continue")];+ char stringpool_str30[sizeof("while")];+ char stringpool_str31[sizeof("export")];+ char stringpool_str32[sizeof("extends")];+ char stringpool_str33[sizeof("for")];+ char stringpool_str34[sizeof("this")];+ char stringpool_str35[sizeof("throw")];+ char stringpool_str36[sizeof("return")];+ char stringpool_str39[sizeof("void")];+ char stringpool_str40[sizeof("yield")];+ char stringpool_str45[sizeof("super")];char stringpool_str50[sizeof("break")];- char stringpool_str53[sizeof("var")];- char stringpool_str55[sizeof("while")];+ char stringpool_str53[sizeof("try")];+ char stringpool_str58[sizeof("function")];+ char stringpool_str61[sizeof("typeof")];};static const struct stringpool_t stringpool_contents ={- "try",- "true",- "const",- "return",- "do",- "continue",+ "in",+ "new","case","catch","delete","default","debugger",- "with",- "super",- "static",- "if",- "for",- "await",+ "instanceof","import",+ "if",+ "let",+ "else",+ "class","finally",- "function",- "false",+ "var",+ "with",+ "await","switch",- "in",- "new",- "null",- "yield",- "typeof",- "instanceof",+ "do",+ "const",+ "static",+ "continue",+ "while","export","extends",- "let",- "else",- "class",+ "for","this","throw",+ "return","void",+ "yield",+ "super","break",- "var",- "while"+ "try",+ "function",+ "typeof"};#define stringpool ((const char *) &stringpool_contents)static const int wordlist[] ={- -1, -1, -1,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str2,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,- -1,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,- -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,- -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,- -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,+ -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,-1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61};if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)diff --git a/llamafile/is_keyword_js.gperf b/llamafile/is_keyword_js.gperfindex 00beff2a..c59e6580 100644--- a/llamafile/is_keyword_js.gperf+++ b/llamafile/is_keyword_js.gperf@@ -21,7 +21,6 @@ doelseexportextends-falsefinallyforfunction@@ -31,14 +30,12 @@ ininstanceofletnew-nullreturnstaticsuperswitchthisthrow-truetrytypeofvardiff --git a/llamafile/is_keyword_js_builtin.c b/llamafile/is_keyword_js_builtin.cnew file mode 100644index 00000000..1ca65467--- /dev/null+++ b/llamafile/is_keyword_js_builtin.c@@ -0,0 +1,341 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_js_builtin.gperf */+/* Computed positions: -k'1,5-6' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_js_builtin.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 65+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 22+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 102+/* maximum key range = 100, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 25,+ 15, 15, 103, 103, 10, 103, 5, 103, 103, 103,+ 103, 103, 103, 103, 103, 10, 25, 25, 50, 15,+ 25, 0, 103, 0, 70, 103, 103, 60, 5, 10,+ 5, 103, 25, 0, 40, 25, 15, 20, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 15, 103, 10,+ 0, 5, 103, 30, 103, 0, 103, 103, 0, 103,+ 0, 0, 0, 103, 5, 0, 15, 35, 103, 103,+ 0, 5, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103, 103, 103, 103, 103,+ 103, 103, 103, 103, 103, 103+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[5]];+ /*FALLTHROUGH*/+ case 5:+ hval += asso_values[(unsigned char)str[4]];+ /*FALLTHROUGH*/+ case 4:+ case 3:+ case 2:+ case 1:+ hval += asso_values[(unsigned char)str[0]];+ break;+ }+ return hval;+}++const char *+is_keyword_js_builtin (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("Set")];+ char stringpool_str4[sizeof("Intl")];+ char stringpool_str6[sizeof("Symbol")];+ char stringpool_str8[sizeof("isFinite")];+ char stringpool_str9[sizeof("eval")];+ char stringpool_str10[sizeof("isNaN")];+ char stringpool_str12[sizeof("Promise")];+ char stringpool_str13[sizeof("parseInt")];+ char stringpool_str14[sizeof("decodeURI")];+ char stringpool_str15[sizeof("Proxy")];+ char stringpool_str16[sizeof("escape")];+ char stringpool_str18[sizeof("InternalError")];+ char stringpool_str19[sizeof("encodeURI")];+ char stringpool_str20[sizeof("Array")];+ char stringpool_str21[sizeof("Number")];+ char stringpool_str22[sizeof("SharedArrayBuffer")];+ char stringpool_str23[sizeof("decodeURIComponent")];+ char stringpool_str24[sizeof("Int8Array")];+ char stringpool_str25[sizeof("Error")];+ char stringpool_str26[sizeof("SyntaxError")];+ char stringpool_str27[sizeof("Atomics")];+ char stringpool_str28[sizeof("encodeURIComponent")];+ char stringpool_str29[sizeof("Generator")];+ char stringpool_str30[sizeof("Int16Array")];+ char stringpool_str31[sizeof("RegExp")];+ char stringpool_str32[sizeof("WeakSet")];+ char stringpool_str33[sizeof("AsyncIterator")];+ char stringpool_str34[sizeof("AsyncGenerator")];+ char stringpool_str35[sizeof("Int32Array")];+ char stringpool_str36[sizeof("String")];+ char stringpool_str37[sizeof("GeneratorFunction")];+ char stringpool_str38[sizeof("Iterator")];+ char stringpool_str39[sizeof("BigUint64Array")];+ char stringpool_str40[sizeof("parseFloat")];+ char stringpool_str41[sizeof("Object")];+ char stringpool_str42[sizeof("AsyncGeneratorFunction")];+ char stringpool_str43[sizeof("URIError")];+ char stringpool_str44[sizeof("EvalError")];+ char stringpool_str45[sizeof("FinalizationRegistry")];+ char stringpool_str46[sizeof("BigInt")];+ char stringpool_str47[sizeof("Reflect")];+ char stringpool_str48[sizeof("Function")];+ char stringpool_str49[sizeof("ReferenceError")];+ char stringpool_str50[sizeof("Uint8Array")];+ char stringpool_str51[sizeof("ArrayBuffer")];+ char stringpool_str52[sizeof("Boolean")];+ char stringpool_str53[sizeof("BigInt64Array")];+ char stringpool_str54[sizeof("Date")];+ char stringpool_str55[sizeof("RangeError")];+ char stringpool_str57[sizeof("WeakRef")];+ char stringpool_str58[sizeof("AsyncFunction")];+ char stringpool_str59[sizeof("AggregateError")];+ char stringpool_str62[sizeof("Float64Array")];+ char stringpool_str63[sizeof("Map")];+ char stringpool_str64[sizeof("Math")];+ char stringpool_str66[sizeof("Uint32Array")];+ char stringpool_str67[sizeof("Float32Array")];+ char stringpool_str68[sizeof("unescape")];+ char stringpool_str69[sizeof("TypeError")];+ char stringpool_str71[sizeof("Uint16Array")];+ char stringpool_str72[sizeof("Uint8ClampedArray")];+ char stringpool_str73[sizeof("DataView")];+ char stringpool_str74[sizeof("JSON")];+ char stringpool_str77[sizeof("Float16Array")];+ char stringpool_str102[sizeof("WeakMap")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "Set",+ "Intl",+ "Symbol",+ "isFinite",+ "eval",+ "isNaN",+ "Promise",+ "parseInt",+ "decodeURI",+ "Proxy",+ "escape",+ "InternalError",+ "encodeURI",+ "Array",+ "Number",+ "SharedArrayBuffer",+ "decodeURIComponent",+ "Int8Array",+ "Error",+ "SyntaxError",+ "Atomics",+ "encodeURIComponent",+ "Generator",+ "Int16Array",+ "RegExp",+ "WeakSet",+ "AsyncIterator",+ "AsyncGenerator",+ "Int32Array",+ "String",+ "GeneratorFunction",+ "Iterator",+ "BigUint64Array",+ "parseFloat",+ "Object",+ "AsyncGeneratorFunction",+ "URIError",+ "EvalError",+ "FinalizationRegistry",+ "BigInt",+ "Reflect",+ "Function",+ "ReferenceError",+ "Uint8Array",+ "ArrayBuffer",+ "Boolean",+ "BigInt64Array",+ "Date",+ "RangeError",+ "WeakRef",+ "AsyncFunction",+ "AggregateError",+ "Float64Array",+ "Map",+ "Math",+ "Uint32Array",+ "Float32Array",+ "unescape",+ "TypeError",+ "Uint16Array",+ "Uint8ClampedArray",+ "DataView",+ "JSON",+ "Float16Array",+ "WeakMap"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str72,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_js_builtin.gperf b/llamafile/is_keyword_js_builtin.gperfnew file mode 100644index 00000000..3d273221--- /dev/null+++ b/llamafile/is_keyword_js_builtin.gperf@@ -0,0 +1,74 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_js_builtin+%%+AggregateError+Array+ArrayBuffer+AsyncFunction+AsyncGenerator+AsyncGeneratorFunction+AsyncIterator+Atomics+BigInt+BigInt64Array+BigUint64Array+Boolean+DataView+Date+Error+EvalError+FinalizationRegistry+Float16Array+Float32Array+Float64Array+Function+Generator+GeneratorFunction+Int16Array+Int32Array+Int8Array+InternalError+Intl+Iterator+JSON+Map+Math+Number+Object+Promise+Proxy+RangeError+ReferenceError+Reflect+RegExp+Set+SharedArrayBuffer+String+Symbol+SyntaxError+TypeError+URIError+Uint16Array+Uint32Array+Uint8Array+Uint8ClampedArray+WeakMap+WeakRef+WeakSet+decodeURI+decodeURIComponent+encodeURI+encodeURIComponent+escape+eval+isFinite+isNaN+parseFloat+parseInt+unescapediff --git a/llamafile/is_keyword_js_constant.c b/llamafile/is_keyword_js_constant.cnew file mode 100644index 00000000..5ce7694d--- /dev/null+++ b/llamafile/is_keyword_js_constant.c@@ -0,0 +1,139 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_js_constant.gperf */+/* Computed positions: -k'1' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_js_constant.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 7+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 10+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 10+/* maximum key range = 8, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 0, 11, 11, 11, 11, 0, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 0, 0, 11, 11, 11, 11, 11, 11,+ 2, 11, 11, 11, 11, 11, 0, 0, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,+ 11, 11, 11, 11, 11, 11+ };+ return len + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_js_constant (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("NaN")];+ char stringpool_str4[sizeof("true")];+ char stringpool_str5[sizeof("false")];+ char stringpool_str6[sizeof("null")];+ char stringpool_str8[sizeof("Infinity")];+ char stringpool_str9[sizeof("undefined")];+ char stringpool_str10[sizeof("globalThis")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "NaN",+ "true",+ "false",+ "null",+ "Infinity",+ "undefined",+ "globalThis"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_js_constant.gperf b/llamafile/is_keyword_js_constant.gperfnew file mode 100644index 00000000..26e8a0d3--- /dev/null+++ b/llamafile/is_keyword_js_constant.gperf@@ -0,0 +1,16 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_js_constant+%%+Infinity+NaN+false+globalThis+null+true+undefineddiff --git a/llamafile/is_keyword_typescript.c b/llamafile/is_keyword_typescript.cnew file mode 100644index 00000000..9eb5270c--- /dev/null+++ b/llamafile/is_keyword_typescript.c@@ -0,0 +1,302 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_typescript.gperf */+/* Computed positions: -k'1,$' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_typescript.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 55+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 10+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 99+/* maximum key range = 97, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100, 40, 0, 15,+ 25, 5, 10, 10, 50, 15,100, 0, 0, 35,+ 25, 50, 45,100, 20, 5, 0,100, 0, 45,+ 100, 35,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100,100,100,100,100,+ 100,100,100,100,100,100+ };+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_typescript (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("let")];+ char stringpool_str5[sizeof("break")];+ char stringpool_str6[sizeof("target")];+ char stringpool_str8[sizeof("set")];+ char stringpool_str9[sizeof("type")];+ char stringpool_str11[sizeof("export")];+ char stringpool_str13[sizeof("get")];+ char stringpool_str14[sizeof("else")];+ char stringpool_str15[sizeof("keyof")];+ char stringpool_str16[sizeof("typeof")];+ char stringpool_str17[sizeof("extends")];+ char stringpool_str19[sizeof("satisfies")];+ char stringpool_str20[sizeof("const")];+ char stringpool_str21[sizeof("import")];+ char stringpool_str22[sizeof("is")];+ char stringpool_str23[sizeof("var")];+ char stringpool_str24[sizeof("case")];+ char stringpool_str25[sizeof("class")];+ char stringpool_str26[sizeof("static")];+ char stringpool_str27[sizeof("if")];+ char stringpool_str28[sizeof("continue")];+ char stringpool_str29[sizeof("interface")];+ char stringpool_str30[sizeof("implements")];+ char stringpool_str32[sizeof("default")];+ char stringpool_str33[sizeof("for")];+ char stringpool_str35[sizeof("instanceof")];+ char stringpool_str36[sizeof("delete")];+ char stringpool_str37[sizeof("declare")];+ char stringpool_str38[sizeof("try")];+ char stringpool_str39[sizeof("namespace")];+ char stringpool_str40[sizeof("infer")];+ char stringpool_str42[sizeof("in")];+ char stringpool_str43[sizeof("function")];+ char stringpool_str44[sizeof("enum")];+ char stringpool_str45[sizeof("await")];+ char stringpool_str47[sizeof("as")];+ char stringpool_str48[sizeof("abstract")];+ char stringpool_str49[sizeof("from")];+ char stringpool_str50[sizeof("throw")];+ char stringpool_str51[sizeof("return")];+ char stringpool_str52[sizeof("finally")];+ char stringpool_str53[sizeof("debugger")];+ char stringpool_str55[sizeof("while")];+ char stringpool_str57[sizeof("private")];+ char stringpool_str60[sizeof("async")];+ char stringpool_str61[sizeof("switch")];+ char stringpool_str62[sizeof("of")];+ char stringpool_str63[sizeof("readonly")];+ char stringpool_str65[sizeof("yield")];+ char stringpool_str66[sizeof("public")];+ char stringpool_str70[sizeof("catch")];+ char stringpool_str73[sizeof("new")];+ char stringpool_str77[sizeof("do")];+ char stringpool_str79[sizeof("protected")];+ char stringpool_str99[sizeof("with")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "let",+ "break",+ "target",+ "set",+ "type",+ "export",+ "get",+ "else",+ "keyof",+ "typeof",+ "extends",+ "satisfies",+ "const",+ "import",+ "is",+ "var",+ "case",+ "class",+ "static",+ "if",+ "continue",+ "interface",+ "implements",+ "default",+ "for",+ "instanceof",+ "delete",+ "declare",+ "try",+ "namespace",+ "infer",+ "in",+ "function",+ "enum",+ "await",+ "as",+ "abstract",+ "from",+ "throw",+ "return",+ "finally",+ "debugger",+ "while",+ "private",+ "async",+ "switch",+ "of",+ "readonly",+ "yield",+ "public",+ "catch",+ "new",+ "do",+ "protected",+ "with"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str65,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_typescript.gperf b/llamafile/is_keyword_typescript.gperfnew file mode 100644index 00000000..99f7426d--- /dev/null+++ b/llamafile/is_keyword_typescript.gperf@@ -0,0 +1,64 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_typescript+%%+abstract+as+async+await+break+case+catch+class+const+continue+debugger+declare+default+delete+do+else+enum+export+extends+finally+for+from+function+get+if+implements+import+in+infer+instanceof+interface+is+keyof+let+namespace+new+of+private+protected+public+readonly+return+satisfies+set+static+switch+target+throw+try+type+typeof+var+while+with+yielddiff --git a/llamafile/is_keyword_typescript_type.c b/llamafile/is_keyword_typescript_type.cnew file mode 100644index 00000000..bc37d317--- /dev/null+++ b/llamafile/is_keyword_typescript_type.c@@ -0,0 +1,150 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_typescript_type.gperf */+/* Computed positions: -k'2' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_typescript_type.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 10+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 7+#define MIN_HASH_VALUE 4+#define MAX_HASH_VALUE 21+/* maximum key range = 18, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 4, 22,+ 22, 0, 22, 22, 22, 15, 22, 22, 22, 22,+ 5, 0, 22, 22, 22, 22, 10, 5, 22, 22,+ 22, 0, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,+ 22, 22, 22, 22, 22, 22+ };+ return len + asso_values[(unsigned char)str[1]];+}++const char *+is_keyword_typescript_type (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str4[sizeof("void")];+ char stringpool_str5[sizeof("never")];+ char stringpool_str6[sizeof("symbol")];+ char stringpool_str7[sizeof("boolean")];+ char stringpool_str8[sizeof("any")];+ char stringpool_str10[sizeof("object")];+ char stringpool_str11[sizeof("number")];+ char stringpool_str12[sizeof("unknown")];+ char stringpool_str16[sizeof("string")];+ char stringpool_str21[sizeof("bigint")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "void",+ "never",+ "symbol",+ "boolean",+ "any",+ "object",+ "number",+ "unknown",+ "string",+ "bigint"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_typescript_type.gperf b/llamafile/is_keyword_typescript_type.gperfnew file mode 100644index 00000000..4a7a42e2--- /dev/null+++ b/llamafile/is_keyword_typescript_type.gperf@@ -0,0 +1,19 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_typescript_type+%%+any+bigint+boolean+never+number+object+string+symbol+unknown+void