Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add m4 syntax highlighting
Justine Tunneyf28e95490b02
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 24aeb0de..83588f1e 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -18,110 +18,145 @@#include "highlight.h"Highlight *Highlight::create(const std::string_view &lang) {- if (lang == "markdown" || lang == "md") {++ if (lang == "md" || //+ lang == "markdown")return new HighlightMarkdown;- } else if (lang == "c" || //- lang == "h" || //- lang == "m") {++ if (lang == "c" || //+ lang == "h" || //+ lang == "m")return new HighlightC(is_keyword_c, //is_keyword_c_type, //is_keyword_c_builtin, //is_keyword_c_constant);- } else if (lang == "c++" || //- lang == "cxx" || //- lang == "cpp" || //- lang == "hpp" || //- lang == "cc" || //- lang == "cu") {++ if (lang == "c++" || //+ lang == "cxx" || //+ lang == "cpp" || //+ lang == "hpp" || //+ lang == "cc" || //+ lang == "cu")return new HighlightC(is_keyword_cxx, //is_keyword_c_type, //is_keyword_c_builtin, //is_keyword_c_constant);- } else if (lang == "js" || //- lang == "json" || //- lang == "javascript") {++ if (lang == "js" || //+ lang == "json" || //+ lang == "javascript")return new HighlightJs(is_keyword_js, nullptr);- } else if (lang == "ts" || //- lang == "typescript") {++ if (lang == "ts" || //+ lang == "typescript")return new HighlightJs(is_keyword_typescript, is_keyword_typescript_type);- } else if (lang == "java") {- return new HighlightC(is_keyword_java);- } else if (lang == "py" || //- lang == "python") {++ if (lang == "java")+ return new HighlightJava(is_keyword_java);++ if (lang == "py" || //+ lang == "python")return new HighlightPython;- } else if (lang == "rs" || //- lang == "rust") {++ if (lang == "rs" || //+ lang == "rust")return new HighlightRust;- } else if (lang == "f" || //- lang == "fortran") {++ if (lang == "f" || //+ lang == "fortran")return new HighlightFortran;- } else if (lang == "cob" || //- lang == "cbl" || //- lang == "cobol") {++ if (lang == "cob" || //+ lang == "cbl" || //+ lang == "cobol")return new HighlightCobol;- } else if (lang == "pas" || //- lang == "pascal") {++ if (lang == "pas" || //+ lang == "pascal")return new HighlightPascal;- } else if (lang == "go") {++ if (lang == "go")return new HighlightGo;- } else if (lang == "sql") {++ if (lang == "sql")return new HighlightSql;- } else if (lang == "css") {++ if (lang == "css")return new HighlightCss;- } else if (lang == "html" || //- lang == "xml") {++ if (lang == "html" || //+ lang == "xml")return new HighlightHtml;- } else if (lang == "php") {++ if (lang == "php")return new HighlightHtml; // sic- } else if (lang == "php!") {++ if (lang == "php!")return new HighlightPhp;- } else if (lang == "csharp" || //- lang == "cs" || //- lang == "c#") {++ if (lang == "csharp" || //+ lang == "cs" || //+ lang == "c#")return new HighlightCsharp;- } else if (lang == "kotlin") {- return new HighlightC(is_keyword_kotlin);- } else if (lang == "lua") {++ if (lang == "kotlin")+ return new HighlightJava(is_keyword_kotlin);++ if (lang == "lua")return new HighlightLua;- } else if (lang == "lisp" || //- lang == "el" || //- lang == "elisp" || //- lang == "cl" || //- lang == "clisp" || //- lang == "scheme" || //- lang == "clojure") {++ if (lang == "lisp" || //+ lang == "el" || //+ lang == "elisp" || //+ lang == "cl" || //+ lang == "clisp" || //+ lang == "scheme" || //+ lang == "clojure")return new HighlightLisp;- } else if (lang == "ada") {++ if (lang == "ada")return new HighlightAda;- } else if (lang == "haskell" || //- lang == "hs") {++ if (lang == "haskell" || //+ lang == "hs")return new HighlightHaskell;- } else if (lang == "perl" || //- lang == "pl") {++ if (lang == "perl" || //+ lang == "pl")return new HighlightPerl;- } else if (lang == "shell" || //- lang == "bash" || //- lang == "sh" || //- lang == "ksh") {++ if (lang == "shell" || //+ lang == "bash" || //+ lang == "sh" || //+ lang == "ksh")return new HighlightShell;- } else if (lang == "swift") {++ if (lang == "swift")return new HighlightC(is_keyword_swift, is_keyword_swift_type);- } else if (lang == "d") {++ if (lang == "d")return new HighlightC(is_keyword_d);- } else if (lang == "zig") {++ if (lang == "zig")return new HighlightZig;- } else if (lang == "tcl") {++ if (lang == "tcl")return new HighlightTcl;- } else if (lang == "ruby" || //- lang == "rb") {++ if (lang == "m4" || //+ lang == "ac")+ return new HighlightM4;++ if (lang == "ruby" || //+ lang == "rb")return new HighlightRuby;- } else if (lang == "fs" || //- lang == "4th" || //- lang == "frt" || //- lang == "fth" || //- lang == "forth") {++ if (lang == "fs" || //+ lang == "4th" || //+ lang == "frt" || //+ lang == "fth" || //+ lang == "forth")return new HighlightForth;- } else {- return new HighlightPlain;- }++ return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 8c409abf..ca685651 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -58,6 +58,7 @@ is_keyword_f is_keyword_js;is_keyword_f is_keyword_js_builtin;is_keyword_f is_keyword_js_constant;is_keyword_f is_keyword_java;+is_keyword_f is_keyword_java_constant;is_keyword_f is_keyword_python;is_keyword_f is_keyword_rust;is_keyword_f is_keyword_rust_type;@@ -104,6 +105,7 @@ is_keyword_f is_keyword_typescript_type;is_keyword_f is_keyword_typescript_constant;is_keyword_f is_keyword_forth;is_keyword_f is_keyword_forth_def;+is_keyword_f is_keyword_m4;}class Highlight {@@ -162,6 +164,19 @@ class HighlightC : public Highlight {is_keyword_f *is_constant_;};+class HighlightJava : public Highlight {+ public:+ explicit HighlightJava(is_keyword_f *is_keyword);+ ~HighlightJava() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+ is_keyword_f *is_keyword_;+};+class HighlightGo : public Highlight {public:HighlightGo();@@ -479,3 +494,15 @@ class HighlightForth : public Highlight {bool is_label_ = false;std::string word_;};++class HighlightM4 : public Highlight {+ public:+ HighlightM4();+ ~HighlightM4() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cppindex 6c2bcbf3..6047f297 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight_c.cpp@@ -33,6 +33,8 @@ enum {R,R_DQUOTE,RAW,+ QUESTION,+ TRIGRAPH,};HighlightC::HighlightC(is_keyword_f *is_keyword, //@@ -63,6 +65,8 @@ void HighlightC::feed(std::string *r, std::string_view input) {word_ += c;} else if (c == '/') {t_ = SLASH;+ } else if (c == '?') {+ t_ = QUESTION;} else if (c == '\'') {t_ = QUOTE;*r += HI_STRING;@@ -112,6 +116,38 @@ void HighlightC::feed(std::string *r, std::string_view input) {}break;+ case QUESTION:+ if (c == '?') {+ t_ = TRIGRAPH;+ } else {+ *r += '?';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case TRIGRAPH:+ if (c == '=' || // '#'+ c == '(' || // '['+ c == '/' || // '\\'+ c == ')' || // ']'+ c == '\'' || // '^'+ c == '<' || // '{'+ c == '!' || // '|'+ c == '>' || // '}'+ c == '-') { // '~'+ *r += HI_ESCAPE;+ *r += "??";+ *r += c;+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += "??";+ t_ = NORMAL;+ goto Normal;+ }+ break;+case SLASH:if (c == '/') {*r += HI_COMMENT;@@ -254,6 +290,12 @@ void HighlightC::flush(std::string *r) {case SLASH:*r += '/';break;+ case QUESTION:+ *r += '?';+ break;+ case TRIGRAPH:+ *r += "??";+ break;case R:*r += 'R';break;diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight_c_test.cppindex 5ca13e43..e4cdc2c4 100644--- a/llamafile/highlight_c_test.cpp+++ b/llamafile/highlight_c_test.cpp@@ -25,53 +25,49 @@ using namespace std::string_literals;HighlightC g_highlight;void test(std::string input, std::string expect) {- std::string got;- g_highlight.feed(&got, input);- if (got != expect) {- fprintf(stderr,- "error: highlighting failed\n"- "\t input: %`'s\n"- "\texpected: %`'s\n"- "\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());- exit(1);- }+ std::string got;+ g_highlight.feed(&got, input);+ if (got != expect) {+ fprintf(stderr,+ "error: highlighting failed\n"+ "\t input: %`'s\n"+ "\texpected: %`'s\n"+ "\t got: %`'s\n",+ input.c_str(), expect.c_str(), got.c_str());+ exit(1);+ }}void testf(std::string input, std::string expect) {- std::string got;- g_highlight.feed(&got, input);- g_highlight.flush(&got);- if (got != expect) {- fprintf(stderr,- "error: highlighting failed\n"- "\t input: %`'s\n"- "\texpected: %`'s\n"- "\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());- exit(1);- }+ std::string got;+ g_highlight.feed(&got, input);+ g_highlight.flush(&got);+ if (got != expect) {+ fprintf(stderr,+ "error: highlighting failed\n"+ "\t input: %`'s\n"+ "\texpected: %`'s\n"+ "\t got: %`'s\n",+ input.c_str(), expect.c_str(), got.c_str());+ exit(1);+ }}int main(int argc, char *argv[]) {- test("", "");- test("vola", "");- test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");- test("yo // hi\nthere ",- "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");- testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);- test("yo /* hi */ there ",- "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");- testf("yo /* hi */ there",- "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");- testf("yo /* h*i */ there",- "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");- testf("vola", "vola");- testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);- testf("/hi/", "/hi/");- testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");- testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");- testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");- testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");- testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");+ test("", "");+ test("vola", "");+ test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");+ test("yo // hi\nthere ", "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");+ testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);+ test("yo /* hi */ there ", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");+ testf("yo /* hi */ there", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");+ testf("yo /* h*i */ there", "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");+ testf("vola", "vola");+ testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);+ testf("/hi/", "/hi/");+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");}diff --git a/llamafile/highlight_fortran.cpp b/llamafile/highlight_fortran.cppindex e7835e40..12322c20 100644--- a/llamafile/highlight_fortran.cpp+++ b/llamafile/highlight_fortran.cpp@@ -87,7 +87,7 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {Word:case WORD:- if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {+ if (!isascii(c) || isalnum(c) || c == '_') {word_ += c;} else if (c == '.' && word_[0] == '.') {word_ += c;diff --git a/llamafile/highlight_java.cpp b/llamafile/highlight_java.cppnew file mode 100644index 00000000..545ae481--- /dev/null+++ b/llamafile/highlight_java.cpp@@ -0,0 +1,283 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ QUOTE,+ QUOTE_BACKSLASH,+ ANNOTATION,+ ANNOTATION2,+ SLASH,+ SLASH_SLASH,+ SLASH_STAR,+ SLASH_STAR_STAR,+ DQUOTE, // "+ DQUOTESTR, // "...+ DQUOTESTR_BACKSLASH, // "...+ DQUOTE2, // ""+ DQUOTE3, // """...+ DQUOTE31, // """..."+ DQUOTE32, // """...""+};++HighlightJava::HighlightJava(is_keyword_f is_keyword) : is_keyword_(is_keyword) {+}++HighlightJava::~HighlightJava() {+}++void HighlightJava::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '/') {+ t_ = SLASH;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += '\'';+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += '"';+ } else if (c == '@') {+ t_ = ANNOTATION;+ } else {+ *r += c;+ }+ break;++ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_java_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH:+ if (c == '/') {+ *r += HI_COMMENT;+ *r += "//";+ t_ = SLASH_SLASH;+ } else if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ = SLASH_STAR;+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH_SLASH:+ if (c == '\n') {+ *r += HI_RESET;+ *r += '\n';+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*')+ t_ = SLASH_STAR_STAR;+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c != '*') {+ t_ = SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ // handle "string"+ case DQUOTE:+ *r += c;+ if (c == '"') {+ t_ = DQUOTE2;+ } else if (c == '\\') {+ t_ = DQUOTESTR_BACKSLASH;+ } else {+ t_ = DQUOTESTR;+ }+ break;+ case DQUOTESTR:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTESTR_BACKSLASH;+ }+ break;+ case DQUOTESTR_BACKSLASH:+ *r += c;+ t_ = DQUOTESTR;+ break;++ // handle """string""" from java 15++ case DQUOTE2:+ if (c == '"') {+ *r += '"';+ t_ = DQUOTE3;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;+ case DQUOTE3:+ *r += c;+ if (c == '"')+ t_ = DQUOTE31;+ break;+ case DQUOTE31:+ *r += c;+ if (c == '"') {+ t_ = DQUOTE32;+ } else {+ t_ = DQUOTE3;+ }+ break;+ case DQUOTE32:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ t_ = DQUOTE3;+ }+ break;++ case ANNOTATION:+ if (!isascii(c) || isalpha(c) || c == '_') {+ *r += HI_ATTRIB;+ *r += '@';+ *r += c;+ t_ = ANNOTATION2;+ } else {+ *r += '@';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case ANNOTATION2:+ if (!isascii(c) || isalnum(c) || c == '_') {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightJava::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_java_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case SLASH:+ *r += '/';+ break;+ case ANNOTATION:+ *r += '@';+ break;+ case QUOTE:+ case QUOTE_BACKSLASH:+ case SLASH_SLASH:+ case SLASH_STAR:+ case SLASH_STAR_STAR:+ case DQUOTE:+ case DQUOTESTR:+ case DQUOTESTR_BACKSLASH:+ case DQUOTE2:+ case DQUOTE3:+ case DQUOTE31:+ case DQUOTE32:+ case ANNOTATION2:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_m4.cpp b/llamafile/highlight_m4.cppnew file mode 100644index 00000000..06f323b1--- /dev/null+++ b/llamafile/highlight_m4.cpp@@ -0,0 +1,134 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ COMMENT,+ DOLLAR,+};++HighlightM4::HighlightM4() {+}++HighlightM4::~HighlightM4() {+}++void HighlightM4::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '#') {+ t_ = COMMENT;+ *r += HI_COMMENT;+ *r += '#';+ } else if (c == '$') {+ t_ = DOLLAR;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_m4(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (word_ == "dnl" || word_ == "m4_dnl") {+ *r += HI_COMMENT;+ *r += word_;+ word_.clear();+ t_ = COMMENT;+ goto Comment;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case DOLLAR:+ if (isdigit(c) || c == '*' || c == '#' || c == '@') {+ *r += '$';+ *r += HI_VAR;+ *r += c;+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += '$';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ Comment:+ case COMMENT:+ if (c == '\n') {+ *r += HI_RESET;+ *r += '\n';+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightM4::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_m4(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case DOLLAR:+ *r += '$';+ break;+ case COMMENT:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cppindex 3b0b9533..208aa24e 100644--- a/llamafile/highlight_markdown.cpp+++ b/llamafile/highlight_markdown.cpp@@ -16,6 +16,8 @@// limitations under the License.#include "highlight.h"+#include "string.h"+#include <cosmo.h>enum {NORMAL,@@ -44,9 +46,8 @@ HighlightMarkdown::~HighlightMarkdown() {}void HighlightMarkdown::feed(std::string *r, std::string_view input) {- char c;- for (size_t i = 0; i < input.size(); ++i) {- c = input[i];+ for (size_t i = 0; i < input.size();) {+ wchar_t c = read_wchar(input, &i);switch (t_) {Normal:@@ -60,9 +61,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {} else if (c == '\\') {// handle \*\*not bold\*\* etc.t_ = BACKSLASH;- *r += c;+ *r += '\\';} else {- *r += c;+ append_wchar(r, c);}if (c == '\n') {bol_ = true;@@ -75,7 +76,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {break;case BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = NORMAL;break;@@ -84,18 +85,17 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {// handle **strong** textt_ = STRONG;*r += HI_BOLD;- *r += '*';- *r += c;+ *r += "**";} else if (bol_ && isblank(c)) {*r += '*';- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {// handle *emphasized* text// inverted because \e[3m has a poorly supported western bias*r += '*';*r += HI_ITALIC;- *r += c;+ append_wchar(r, c);t_ = EMPHASIS;if (c == '\\')t_ = EMPHASIS_BACKSLASH;@@ -107,23 +107,23 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {if (c == '*') {t_ = NORMAL;*r += HI_RESET;- *r += c;+ *r += '*';} else if (c == '\\') {t_ = EMPHASIS_BACKSLASH;- *r += c;+ *r += '\\';} else {- *r += c;+ append_wchar(r, c);}break;case EMPHASIS_BACKSLASH:// so we can say *unbroken \* italic* and have it work- *r += c;+ append_wchar(r, c);t_ = EMPHASIS;break;case STRONG:- *r += c;+ append_wchar(r, c);if (c == '*') {t_ = STRONG_STAR;} else if (c == '\\') {@@ -133,12 +133,12 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case STRONG_BACKSLASH:// so we can say **unbroken \*\* bold** and have it work- *r += c;+ append_wchar(r, c);t_ = STRONG;break;case STRONG_STAR:- *r += c;+ append_wchar(r, c);if (c == '*' || // handle **bold** ending(c == '\n' && !tail_)) { // handle *** line breakt_ = NORMAL;@@ -156,7 +156,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {} else {*r += HI_INCODE;*r += '`';- *r += c;+ append_wchar(r, c);t_ = INCODE;}break;@@ -164,7 +164,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case INCODE:// this is for `inline code` like that// no backslash escapes are supported here- *r += c;+ append_wchar(r, c);if (c == '`') {*r += HI_RESET;t_ = NORMAL;@@ -174,14 +174,14 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case INCODE2:// this is for ``inline ` code`` like that// it lets you put backtick inside the code- *r += c;+ append_wchar(r, c);if (c == '`') {t_ = INCODE2_TICK;}break;case INCODE2_TICK:- *r += c;+ append_wchar(r, c);if (c == '`') {*r += HI_RESET;t_ = NORMAL;@@ -197,14 +197,14 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {} else {*r += HI_INCODE;*r += "``";- *r += c;+ append_wchar(r, c);t_ = INCODE2;}break;case LANG:if (!isascii(c) || !isspace(c)) {- *r += c;+ append_wchar(r, c);lang_ += tolower(c);} else {highlighter_ = Highlight::create(lang_);@@ -219,7 +219,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {if (c == '`') {t_ = CODE_TICK;} else {- char cs[2] = {c};+ char cs[8];+ uint64_t w = tpenc(c);+ WRITE64LE(cs, w);highlighter_->feed(r, cs);}break;@@ -228,7 +230,10 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {if (c == '`') {t_ = CODE_TICK_TICK;} else {- char cs[3] = {'`', c};+ char cs[9];+ uint64_t w = tpenc(c);+ cs[0] = '`';+ WRITE64LE(cs + 1, w);highlighter_->feed(r, cs);t_ = CODE;}@@ -242,7 +247,11 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {highlighter_ = nullptr;*r += "```";} else {- char cs[4] = {'`', '`', c};+ char cs[10];+ uint64_t w = tpenc(c);+ cs[0] = '`';+ cs[1] = '`';+ WRITE64LE(cs + 2, w);highlighter_->feed(r, cs);t_ = CODE;}diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cppindex 1c365d89..d0894f43 100644--- a/llamafile/highlight_perl.cpp+++ b/llamafile/highlight_perl.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-+#include "string.h"#include <ctype.h>enum {@@ -148,52 +148,52 @@ HighlightPerl::~HighlightPerl() {void HighlightPerl::feed(std::string *r, std::string_view input) {int c = 0;- for (size_t i = 0; i < input.size(); ++i) {+ for (size_t i = 0; i < input.size();) {last_ = c_;- c_ = c = input[i] & 255;+ c_ = c = read_wchar(input, &i);switch (t_) {Normal:case NORMAL:if (!isascii(c) || isalpha(c) || c == '_') {t_ = WORD;- word_ += c;+ append_wchar(&word_, c);} else if (c == '\'') {t_ = QUOTE;*r += HI_STRING;- *r += c;+ append_wchar(r, c);expect_ = EXPECT_OPERATOR;} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;- *r += c;+ append_wchar(r, c);expect_ = EXPECT_OPERATOR;} else if (c == '=' && (!last_ || last_ == '\n')) {t_ = EQUAL;} else if (c == '\\') {t_ = BACKSLASH;*r += HI_ESCAPE;- *r += c;+ append_wchar(r, c);} else if (c == '`') {t_ = TICK;*r += HI_STRING;- *r += c;+ append_wchar(r, c);expect_ = EXPECT_OPERATOR;} else if (c == '$') {- *r += c;+ append_wchar(r, c);t_ = VAR;expect_ = EXPECT_OPERATOR;} else if (c == '@' || c == '%') {- *r += c;+ append_wchar(r, c);*r += HI_VAR;t_ = VAR2;expect_ = EXPECT_OPERATOR;} else if (c == '#') {*r += HI_COMMENT;- *r += c;+ append_wchar(r, c);t_ = COMMENT;} else if (c == '<') {- *r += c;+ append_wchar(r, c);t_ = LT;expect_ = EXPECT_VALUE;} else if (c == '/' && expect_ == EXPECT_VALUE && last_ != '/') {@@ -201,10 +201,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {closer_ = '/';expect_ = EXPECT_OPERATOR;*r += HI_STRING;- *r += c;+ append_wchar(r, c);t_ = REGEX;} else if (c == '\n') {- *r += c;+ append_wchar(r, c);if (pending_heredoc_) {*r += HI_STRING;pending_heredoc_ = false;@@ -213,22 +213,22 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {}} else if (c == ')' || c == '}' || c == ']') {expect_ = EXPECT_OPERATOR;- *r += c;+ append_wchar(r, c);} else if (ispunct(c)) {expect_ = EXPECT_VALUE;- *r += c;+ append_wchar(r, c);} else if (isdigit(c)) {expect_ = EXPECT_OPERATOR;- *r += c;+ append_wchar(r, c);} else {- *r += c;+ append_wchar(r, c);}break;Word:case WORD:if (!isascii(c) || isalnum(c) || c == '_') {- word_ += c;+ append_wchar(&word_, c);} else {if (is_keyword_perl(word_.data(), word_.size())) {*r += HI_KEYWORD;@@ -246,7 +246,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {opener_ = c;closer_ = mirror(c);*r += HI_STRING;- *r += c;+ append_wchar(r, c);if (is_double_regex(word_)) {t_ = S_REGEX;} else {@@ -263,7 +263,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case BACKSLASH:- *r += c;+ append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;break;@@ -271,13 +271,13 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case VAR:if (isdigit(c) || is_magic_var(c)) {*r += HI_VAR;- *r += c;+ append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;break;} else if (c == '{') {t_ = VAR2;- *r += c;+ append_wchar(r, c);*r += HI_VAR;break;} else {@@ -288,7 +288,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case VAR2:if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {- *r += c;+ append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -299,15 +299,15 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case COMMENT:if (c == '\n') {*r += HI_RESET;- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {- *r += c;+ append_wchar(r, c);}break;case REGEX:- *r += c;+ append_wchar(r, c);if (c == closer_) {*r += HI_RESET;t_ = NORMAL;@@ -317,12 +317,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case REGEX_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = REGEX;break;case S_REGEX:- *r += c;+ append_wchar(r, c);if (c == opener_) {t_ = S_REGEX_S;} else if (c == '\\') {@@ -331,12 +331,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case S_REGEX_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = S_REGEX;break;case S_REGEX_S:- *r += c;+ append_wchar(r, c);if (c == closer_) {*r += HI_RESET;t_ = NORMAL;@@ -346,12 +346,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case S_REGEX_S_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = S_REGEX_S;break;case QUOTE:- *r += c;+ append_wchar(r, c);if (c == '\'') {*r += HI_RESET;t_ = NORMAL;@@ -361,12 +361,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case QUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = QUOTE;break;case DQUOTE:- *r += c;+ append_wchar(r, c);if (c == '"') {*r += HI_RESET;t_ = NORMAL;@@ -376,12 +376,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case DQUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = DQUOTE;break;case TICK:- *r += c;+ append_wchar(r, c);if (c == '`') {*r += HI_RESET;t_ = NORMAL;@@ -391,7 +391,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case TICK_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = TICK;break;@@ -399,7 +399,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {if (isalpha(c)) {*r += HI_COMMENT;*r += '=';- *r += c;+ append_wchar(r, c);heredoc_ = "=cut";t_ = HEREDOC;i_ = 0;@@ -412,7 +412,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case LT:if (c == '<') {- *r += c;+ append_wchar(r, c);t_ = LT_LT;heredoc_.clear();pending_heredoc_ = false;@@ -426,16 +426,16 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case LT_LT:if (c == '-') {indented_heredoc_ = true;- *r += c;+ append_wchar(r, c);} else if (c == '"' || c == '\'') {closer_ = c;t_ = LT_LT_QNAME;*r += HI_STRING;- *r += c;+ append_wchar(r, c);} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;- heredoc_ += c;- *r += c;+ append_wchar(&heredoc_, c);+ append_wchar(r, c);} else if (!isblank(c)) {t_ = NORMAL;goto Normal;@@ -445,10 +445,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {case LT_LT_NAME:if (isalnum(c) || c == '_') {t_ = LT_LT_NAME;- heredoc_ += c;- *r += c;+ append_wchar(&heredoc_, c);+ append_wchar(r, c);} else if (c == '\n') {- *r += c;+ append_wchar(r, c);*r += HI_STRING;t_ = HEREDOC_BOL;} else {@@ -459,19 +459,19 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case LT_LT_QNAME:- *r += c;+ append_wchar(r, c);if (c == closer_) {*r += HI_RESET;t_ = HEREDOC_BOL;pending_heredoc_ = true;t_ = NORMAL;} else {- heredoc_ += c;+ append_wchar(&heredoc_, c);}break;case HEREDOC_BOL:- *r += c;+ append_wchar(r, c);if (c == '\n') {if (i_ == heredoc_.size()) {t_ = NORMAL;@@ -489,7 +489,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case HEREDOC:- *r += c;+ append_wchar(r, c);if (c == '\n') {t_ = HEREDOC_BOL;i_ = 0;diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cppindex 3b83139d..61582909 100644--- a/llamafile/highlight_python.cpp+++ b/llamafile/highlight_python.cpp@@ -190,7 +190,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {// handle """string"""case DQUOTE2:if (c == '"') {- *r += c;+ *r += '"';t_ = DQUOTE3;} else {*r += HI_RESET;diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cppindex 3c60d10d..e9f9191c 100644--- a/llamafile/highlight_shell.cpp+++ b/llamafile/highlight_shell.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-+#include "string.h"#include <ctype.h>enum {@@ -46,44 +46,43 @@ HighlightShell::~HighlightShell() {}void HighlightShell::feed(std::string *r, std::string_view input) {- int c;- for (size_t i = 0; i < input.size(); ++i) {- c = input[i] & 255;+ for (size_t i = 0; i < input.size();) {+ wchar_t c = read_wchar(input, &i);switch (t_) {Normal:case NORMAL:if (!isascii(c) || isalpha(c) || c == '_') {t_ = WORD;- word_ += c;+ append_wchar(&word_, c);} else if (c == '\'') {t_ = QUOTE;*r += HI_STRING;- *r += c;+ *r += '\'';} else if (c == '\\') {t_ = BACKSLASH;*r += HI_ESCAPE;- *r += c;+ *r += '\\';} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;- *r += c;+ *r += '"';} else if (c == '`') {t_ = TICK;*r += HI_STRING;- *r += c;+ *r += '`';} else if (c == '$') {- *r += c;t_ = VAR;+ *r += '$';} else if (c == '<') {- *r += c;t_ = LT;+ *r += '<';} else if (c == '#') {*r += HI_COMMENT;- *r += c;+ *r += '#';t_ = COMMENT;} else if (c == '\n') {- *r += c;+ *r += '\n';if (pending_heredoc_) {*r += HI_STRING;pending_heredoc_ = false;@@ -91,19 +90,19 @@ void HighlightShell::feed(std::string *r, std::string_view input) {i_ = 0;}} else {- *r += c;+ append_wchar(r, c);}break;case BACKSLASH:- *r += c;+ append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;break;case WORD:if (!isascii(c) || isalnum(c) || c == '_') {- word_ += c;+ append_wchar(&word_, c);} else {if (is_keyword_shell(word_.data(), word_.size())) {*r += HI_KEYWORD;@@ -133,12 +132,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {c == '\\' || //c == '^') {*r += HI_VAR;- *r += c;+ append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;break;} else if (c == '{') {- *r += c;+ append_wchar(r, c);*r += HI_VAR;t_ = VAR2;break;@@ -150,7 +149,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {case VAR2:if (!isascii(c) || isalnum(c) || c == '_') {- *r += c;+ append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -161,15 +160,15 @@ void HighlightShell::feed(std::string *r, std::string_view input) {case COMMENT:if (c == '\n') {*r += HI_RESET;- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {- *r += c;+ append_wchar(r, c);}break;case QUOTE:- *r += c;+ append_wchar(r, c);if (c == '\'') {*r += HI_RESET;t_ = NORMAL;@@ -177,7 +176,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case DQUOTE:- *r += c;+ append_wchar(r, c);if (c == '"') {*r += HI_RESET;t_ = NORMAL;@@ -187,12 +186,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case DQUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = DQUOTE;break;case TICK:- *r += c;+ append_wchar(r, c);if (c == '`') {*r += HI_RESET;t_ = NORMAL;@@ -202,13 +201,13 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case TICK_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = TICK;break;case LT:if (c == '<') {- *r += c;+ append_wchar(r, c);t_ = LT_LT;heredoc_.clear();pending_heredoc_ = false;@@ -222,17 +221,17 @@ void HighlightShell::feed(std::string *r, std::string_view input) {case LT_LT:if (c == '-') {indented_heredoc_ = true;- *r += c;+ append_wchar(r, c);} else if (c == '\\') {- *r += c;+ append_wchar(r, c);} else if (c == '\'') {t_ = LT_LT_QNAME;*r += HI_STRING;- *r += c;+ append_wchar(r, c);} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;- heredoc_ += c;- *r += c;+ append_wchar(&heredoc_, c);+ append_wchar(r, c);} else if (!isblank(c)) {t_ = NORMAL;goto Normal;@@ -242,10 +241,10 @@ void HighlightShell::feed(std::string *r, std::string_view input) {case LT_LT_NAME:if (isalnum(c) || c == '_') {t_ = LT_LT_NAME;- heredoc_ += c;- *r += c;+ append_wchar(&heredoc_, c);+ append_wchar(r, c);} else if (c == '\n') {- *r += c;+ append_wchar(r, c);*r += HI_STRING;t_ = HEREDOC_BOL;} else {@@ -256,19 +255,19 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case LT_LT_QNAME:- *r += c;+ append_wchar(r, c);if (c == '\'') {*r += HI_RESET;t_ = HEREDOC_BOL;pending_heredoc_ = true;t_ = NORMAL;} else {- heredoc_ += c;+ append_wchar(&heredoc_, c);}break;case HEREDOC_BOL:- *r += c;+ append_wchar(r, c);if (c == '\n') {if (i_ == heredoc_.size()) {t_ = NORMAL;@@ -286,7 +285,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case HEREDOC:- *r += c;+ append_wchar(r, c);if (c == '\n')t_ = HEREDOC_BOL;break;diff --git a/llamafile/highlight_tcl.cpp b/llamafile/highlight_tcl.cppindex 849fec39..495eb256 100644--- a/llamafile/highlight_tcl.cpp+++ b/llamafile/highlight_tcl.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-+#include "string.h"#include <ctype.h>enum {@@ -26,7 +26,9 @@ enum {DQUOTE_BACKSLASH,VAR,VAR2,+ VAR_CURLY,COMMENT,+ BACKSLASH,};HighlightTcl::HighlightTcl() {@@ -36,35 +38,38 @@ HighlightTcl::~HighlightTcl() {}void HighlightTcl::feed(std::string *r, std::string_view input) {- int c;- for (size_t i = 0; i < input.size(); ++i) {- c = input[i] & 255;+ for (size_t i = 0; i < input.size();) {+ wchar_t c = read_wchar(input, &i);switch (t_) {Normal:case NORMAL:if (!isascii(c) || isalpha(c) || c == '_') {t_ = WORD;- word_ += c;+ append_wchar(&word_, c);} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;- *r += c;+ append_wchar(r, c);} else if (c == '$') {- *r += c;+ append_wchar(r, c);t_ = VAR;} else if (c == '#') {*r += HI_COMMENT;- *r += c;+ append_wchar(r, c);t_ = COMMENT;+ } else if (c == '\\') {+ t_ = BACKSLASH;+ *r += HI_ESCAPE;+ append_wchar(r, c);} else {- *r += c;+ append_wchar(r, c);}break;case WORD:- if (!isascii(c) || isalnum(c) || c == '_') {- word_ += c;+ if (!(isspace(c) || c == ';')) {+ append_wchar(&word_, c);} else {if (is_keyword_tcl(word_.data(), word_.size())) {*r += HI_KEYWORD;@@ -87,11 +92,17 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {}break;+ case BACKSLASH:+ append_wchar(r, c);+ *r += HI_RESET;+ t_ = NORMAL;+ break;+case VAR:if (c == '{') {- *r += c;+ append_wchar(r, c);*r += HI_VAR;- t_ = VAR2;+ t_ = VAR_CURLY;break;} else {*r += HI_VAR;@@ -101,7 +112,7 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {case VAR2:if (!isascii(c) || isalnum(c) || c == '_') {- *r += c;+ append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -109,18 +120,28 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {}break;+ case VAR_CURLY:+ if (c == '}') {+ *r += HI_RESET;+ *r += '}';+ t_ = NORMAL;+ } else {+ append_wchar(r, c);+ }+ break;+case COMMENT:if (c == '\n') {*r += HI_RESET;- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {- *r += c;+ append_wchar(r, c);}break;case DQUOTE:- *r += c;+ append_wchar(r, c);if (c == '"') {*r += HI_RESET;t_ = NORMAL;@@ -130,7 +151,7 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {break;case DQUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = DQUOTE;break;@@ -161,9 +182,11 @@ void HighlightTcl::flush(std::string *r) {word_.clear();break;case VAR2:+ case VAR_CURLY:case DQUOTE:case DQUOTE_BACKSLASH:case COMMENT:+ case BACKSLASH:*r += HI_RESET;break;default:diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex 4192b536..4d147607 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -46,6 +46,7 @@ const char *const kLanguages[] = {"kotlin", //"lisp", //"lua", //+ "m4", //"markdown", //"pascal", //"perl", //diff --git a/llamafile/highlight_zig.cpp b/llamafile/highlight_zig.cppindex cfe598fc..fed96513 100644--- a/llamafile/highlight_zig.cpp+++ b/llamafile/highlight_zig.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-+#include "string.h"#include <ctype.h>enum {@@ -39,16 +39,15 @@ HighlightZig::~HighlightZig() {}void HighlightZig::feed(std::string *r, std::string_view input) {- int c;- for (size_t i = 0; i < input.size(); ++i) {- c = input[i] & 255;+ for (size_t i = 0; i < input.size();) {+ wchar_t c = read_wchar(input, &i);switch (t_) {Normal:case NORMAL:if (!isascii(c) || isalpha(c) || c == '_' || c == '@') {t_ = WORD;- word_ += c;+ append_wchar(&word_, c);} else if (c == '/') {t_ = SLASH;} else if (c == '\\') {@@ -56,19 +55,19 @@ void HighlightZig::feed(std::string *r, std::string_view input) {} else if (c == '\'') {t_ = QUOTE;*r += HI_STRING;- *r += c;+ *r += '\'';} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;- *r += c;+ *r += '"';} else {- *r += c;+ append_wchar(r, c);}break;case WORD:if (!isascii(c) || isalnum(c) || c == '_') {- word_ += c;+ append_wchar(&word_, c);} else {if (is_keyword_zig(word_.data(), word_.size())) {*r += HI_KEYWORD;@@ -110,10 +109,10 @@ void HighlightZig::feed(std::string *r, std::string_view input) {case BACKSLASH_BACKSLASH:if (c == '\n') {*r += HI_RESET;- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {- *r += c;+ append_wchar(r, c);}break;@@ -132,15 +131,15 @@ void HighlightZig::feed(std::string *r, std::string_view input) {case SLASH_SLASH:if (c == '\n') {*r += HI_RESET;- *r += c;+ append_wchar(r, c);t_ = NORMAL;} else {- *r += c;+ append_wchar(r, c);}break;case QUOTE:- *r += c;+ append_wchar(r, c);if (c == '\'') {*r += HI_RESET;t_ = NORMAL;@@ -150,12 +149,12 @@ void HighlightZig::feed(std::string *r, std::string_view input) {break;case QUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = QUOTE;break;case DQUOTE:- *r += c;+ append_wchar(r, c);if (c == '"') {*r += HI_RESET;t_ = NORMAL;@@ -165,7 +164,7 @@ void HighlightZig::feed(std::string *r, std::string_view input) {break;case DQUOTE_BACKSLASH:- *r += c;+ append_wchar(r, c);t_ = DQUOTE;break;diff --git a/llamafile/is_keyword_java_constant.c b/llamafile/is_keyword_java_constant.cnew file mode 100644index 00000000..fa2cfccf--- /dev/null+++ b/llamafile/is_keyword_java_constant.c@@ -0,0 +1,126 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf --output-file=llamafile/is_keyword_java_constant.c llamafile/is_keyword_java_constant.gperf */+/* Computed positions: -k'1' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_java_constant.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 3+#define MIN_WORD_LENGTH 4+#define MAX_WORD_LENGTH 5+#define MIN_HASH_VALUE 4+#define MAX_HASH_VALUE 6+/* maximum key range = 3, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 1, 7, 7, 7, 7, 7, 7, 7,+ 1, 7, 7, 7, 7, 7, 0, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,+ 7, 7, 7, 7, 7, 7+ };+ return len + asso_values[(unsigned char)str[0]];+}++const char *+is_keyword_java_constant (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str4[sizeof("true")];+ char stringpool_str5[sizeof("null")];+ char stringpool_str6[sizeof("false")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "true",+ "null",+ "false"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_java_constant.gperf b/llamafile/is_keyword_java_constant.gperfnew file mode 100644index 00000000..8c7f898a--- /dev/null+++ b/llamafile/is_keyword_java_constant.gperf@@ -0,0 +1,12 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_java_constant+%%+true+false+nulldiff --git a/llamafile/is_keyword_kotlin.c b/llamafile/is_keyword_kotlin.cindex 39030455..3cff1963 100644--- a/llamafile/is_keyword_kotlin.c+++ b/llamafile/is_keyword_kotlin.c@@ -33,12 +33,12 @@#include <string.h>-#define TOTAL_KEYWORDS 75+#define TOTAL_KEYWORDS 72#define MIN_WORD_LENGTH 2#define MAX_WORD_LENGTH 11#define MIN_HASH_VALUE 3-#define MAX_HASH_VALUE 154-/* maximum key range = 152, duplicates = 0 */+#define MAX_HASH_VALUE 125+/* maximum key range = 123, duplicates = 0 */#ifdef __GNUC____inline@@ -52,32 +52,32 @@ hash (register const char *str, register size_t len){static const unsigned char asso_values[] ={- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 60, 15, 10,- 45, 5, 30, 0, 0, 15, 15, 35, 0, 55,- 10, 60, 40, 155, 25, 40, 0, 0, 85, 5,- 0, 35, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,- 155, 155, 155, 155, 155, 155+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 50, 20, 5,+ 35, 5, 65, 40, 30, 15, 20, 5, 50, 55,+ 5, 35, 20, 126, 0, 25, 0, 0, 0, 5,+ 5, 45, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,+ 126, 126, 126, 126, 126, 126};register unsigned int hval = len;@@ -99,268 +99,259 @@ is_keyword_kotlin (register const char *str, register size_t len){struct stringpool_t{- char stringpool_str3[sizeof("get")];- char stringpool_str8[sizeof("lateinit")];- char stringpool_str9[sizeof("true")];- char stringpool_str13[sizeof("external")];- char stringpool_str14[sizeof("null")];- char stringpool_str15[sizeof("catch")];+ char stringpool_str3[sizeof("var")];+ char stringpool_str10[sizeof("throw")];+ char stringpool_str11[sizeof("return")];+ char stringpool_str13[sizeof("receiver")];+ char stringpool_str15[sizeof("const")];char stringpool_str17[sizeof("it")];+ char stringpool_str19[sizeof("when")];char stringpool_str20[sizeof("where")];- char stringpool_str23[sizeof("internal")];- char stringpool_str24[sizeof("when")];- char stringpool_str25[sizeof("const")];- char stringpool_str26[sizeof("inline")];- char stringpool_str27[sizeof("in")];+ char stringpool_str21[sizeof("constructor")];+ char stringpool_str22[sizeof("in")];+ char stringpool_str23[sizeof("continue")];+ char stringpool_str25[sizeof("inner")];+ char stringpool_str27[sizeof("tailrec")];+ char stringpool_str28[sizeof("set")];char stringpool_str29[sizeof("interface")];char stringpool_str30[sizeof("while")];- char stringpool_str32[sizeof("tailrec")];- char stringpool_str33[sizeof("continue")];+ char stringpool_str31[sizeof("expect")];+ char stringpool_str33[sizeof("noinline")];char stringpool_str34[sizeof("init")];- char stringpool_str35[sizeof("throw")];- char stringpool_str38[sizeof("noinline")];- char stringpool_str39[sizeof("file")];- char stringpool_str40[sizeof("false")];- char stringpool_str41[sizeof("return")];- char stringpool_str43[sizeof("set")];- char stringpool_str45[sizeof("final")];- char stringpool_str47[sizeof("if")];- char stringpool_str50[sizeof("infix")];- char stringpool_str51[sizeof("expect")];- char stringpool_str52[sizeof("by")];- char stringpool_str53[sizeof("fun")];- char stringpool_str54[sizeof("else")];- char stringpool_str55[sizeof("inner")];- char stringpool_str56[sizeof("constructor")];- char stringpool_str57[sizeof("is")];- char stringpool_str58[sizeof("delegate")];- char stringpool_str59[sizeof("this")];- char stringpool_str60[sizeof("break")];- char stringpool_str61[sizeof("import")];- char stringpool_str62[sizeof("package")];- char stringpool_str63[sizeof("out")];+ char stringpool_str35[sizeof("break")];+ char stringpool_str37[sizeof("package")];+ char stringpool_str38[sizeof("out")];+ char stringpool_str39[sizeof("else")];+ char stringpool_str40[sizeof("catch")];+ char stringpool_str41[sizeof("import")];+ char stringpool_str42[sizeof("is")];+ char stringpool_str43[sizeof("get")];+ char stringpool_str44[sizeof("this")];+ char stringpool_str46[sizeof("vararg")];+ char stringpool_str47[sizeof("private")];+ char stringpool_str48[sizeof("operator")];+ char stringpool_str49[sizeof("open")];+ char stringpool_str50[sizeof("super")];+ char stringpool_str51[sizeof("public")];+ char stringpool_str52[sizeof("dynamic")];+ char stringpool_str53[sizeof("override")];+ char stringpool_str54[sizeof("typealias")];+ char stringpool_str56[sizeof("crossinline")];+ char stringpool_str57[sizeof("reified")];+ char stringpool_str58[sizeof("lateinit")];+ char stringpool_str61[sizeof("object")];+ char stringpool_str63[sizeof("external")];char stringpool_str64[sizeof("enum")];- char stringpool_str66[sizeof("actual")];- char stringpool_str67[sizeof("private")];- char stringpool_str68[sizeof("receiver")];- char stringpool_str71[sizeof("public")];- char stringpool_str72[sizeof("dynamic")];- char stringpool_str73[sizeof("try")];- char stringpool_str76[sizeof("typeof")];- char stringpool_str78[sizeof("override")];- char stringpool_str79[sizeof("open")];- char stringpool_str81[sizeof("object")];- char stringpool_str82[sizeof("finally")];- char stringpool_str83[sizeof("for")];- char stringpool_str84[sizeof("companion")];- char stringpool_str85[sizeof("field")];- char stringpool_str86[sizeof("crossinline")];- char stringpool_str88[sizeof("val")];- char stringpool_str89[sizeof("typealias")];- char stringpool_str90[sizeof("annotation")];- char stringpool_str92[sizeof("reified")];- char stringpool_str98[sizeof("operator")];- char stringpool_str102[sizeof("as")];- char stringpool_str103[sizeof("setparam")];- char stringpool_str107[sizeof("do")];- char stringpool_str108[sizeof("abstract")];- char stringpool_str109[sizeof("data")];- char stringpool_str110[sizeof("super")];- char stringpool_str115[sizeof("class")];- char stringpool_str116[sizeof("vararg")];- char stringpool_str125[sizeof("param")];- char stringpool_str132[sizeof("suspend")];- char stringpool_str138[sizeof("var")];- char stringpool_str143[sizeof("property")];- char stringpool_str151[sizeof("sealed")];- char stringpool_str154[sizeof("protected")];+ char stringpool_str67[sizeof("by")];+ char stringpool_str68[sizeof("for")];+ char stringpool_str70[sizeof("annotation")];+ char stringpool_str72[sizeof("do")];+ char stringpool_str73[sizeof("internal")];+ char stringpool_str74[sizeof("companion")];+ char stringpool_str76[sizeof("inline")];+ char stringpool_str77[sizeof("as")];+ char stringpool_str78[sizeof("fun")];+ char stringpool_str80[sizeof("param")];+ char stringpool_str82[sizeof("if")];+ char stringpool_str83[sizeof("abstract")];+ char stringpool_str85[sizeof("class")];+ char stringpool_str88[sizeof("setparam")];+ char stringpool_str89[sizeof("data")];+ char stringpool_str90[sizeof("infix")];+ char stringpool_str91[sizeof("typeof")];+ char stringpool_str92[sizeof("suspend")];+ char stringpool_str93[sizeof("try")];+ char stringpool_str98[sizeof("delegate")];+ char stringpool_str99[sizeof("protected")];+ char stringpool_str103[sizeof("val")];+ char stringpool_str106[sizeof("actual")];+ char stringpool_str108[sizeof("property")];+ char stringpool_str110[sizeof("field")];+ char stringpool_str116[sizeof("sealed")];+ char stringpool_str122[sizeof("finally")];+ char stringpool_str124[sizeof("file")];+ char stringpool_str125[sizeof("final")];};static const struct stringpool_t stringpool_contents ={- "get",- "lateinit",- "true",- "external",- "null",- "catch",+ "var",+ "throw",+ "return",+ "receiver",+ "const","it",- "where",- "internal","when",- "const",- "inline",+ "where",+ "constructor","in",- "interface",- "while",- "tailrec","continue",- "init",- "throw",- "noinline",- "file",- "false",- "return",+ "inner",+ "tailrec","set",- "final",- "if",- "infix",+ "interface",+ "while","expect",- "by",- "fun",- "else",- "inner",- "constructor",- "is",- "delegate",- "this",+ "noinline",+ "init","break",- "import","package","out",- "enum",- "actual",+ "else",+ "catch",+ "import",+ "is",+ "get",+ "this",+ "vararg","private",- "receiver",+ "operator",+ "open",+ "super","public","dynamic",- "try",- "typeof","override",- "open",+ "typealias",+ "crossinline",+ "reified",+ "lateinit","object",- "finally",+ "external",+ "enum",+ "by","for",- "companion",- "field",- "crossinline",- "val",- "typealias","annotation",- "reified",- "operator",- "as",- "setparam","do",+ "internal",+ "companion",+ "inline",+ "as",+ "fun",+ "param",+ "if","abstract",- "data",- "super","class",- "vararg",- "param",+ "setparam",+ "data",+ "infix",+ "typeof","suspend",- "var",+ "try",+ "delegate",+ "protected",+ "val",+ "actual","property",+ "field","sealed",- "protected"+ "finally",+ "file",+ "final"};#define stringpool ((const char *) &stringpool_contents)static const int wordlist[] ={-1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,- -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,- -1, -1, -1,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,-1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,- -1, -1,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,- -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,-1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,-1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,- -1, -1,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,-1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,- -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str56,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,- -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,- -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str72,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,- -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,-1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str78,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,-1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str81,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str80,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str83,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str84,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str86,-1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,+ -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str88,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str89,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str90,- -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str91,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str92,- -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str93,+ -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str98,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,-1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str103,- -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str107,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str106,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str108,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str109,+ -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str110,- -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,+ -1, -1, -1, -1, -1,(int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,- -1, -1, -1, -1, -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,- -1, -1, -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str132,-1, -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str138,- -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str143,- -1, -1, -1, -1, -1, -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str151,- -1, -1,- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str154+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str124,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125};if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)diff --git a/llamafile/is_keyword_kotlin.gperf b/llamafile/is_keyword_kotlin.gperfindex bf4e3e64..643d5d14 100644--- a/llamafile/is_keyword_kotlin.gperf+++ b/llamafile/is_keyword_kotlin.gperf@@ -28,7 +28,6 @@ elseenumexpectexternal-falsefieldfilefinal@@ -49,7 +48,6 @@ isitlateinitnoinline-nullobjectopenoperator@@ -72,7 +70,6 @@ suspendtailrecthisthrow-truetrytypealiastypeofdiff --git a/llamafile/is_keyword_m4.c b/llamafile/is_keyword_m4.cnew file mode 100644index 00000000..e26357da--- /dev/null+++ b/llamafile/is_keyword_m4.c@@ -0,0 +1,739 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf --output-file=llamafile/is_keyword_m4.c llamafile/is_keyword_m4.gperf */+/* Computed positions: -k'1,4-7,9' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_m4.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 166+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 25+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 439+/* maximum key range = 437, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned short asso_values[] =+ {+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 30, 440, 5, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 40, 145, 70, 120, 40,+ 10, 0, 10, 70, 65, 5, 35, 100, 35, 0,+ 0, 0, 25, 155, 40, 55, 5, 15, 150, 10,+ 145, 130, 10, 0, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,+ 440, 440, 440, 440, 440, 440, 440+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[8]+1];+ /*FALLTHROUGH*/+ case 8:+ case 7:+ hval += asso_values[(unsigned char)str[6]];+ /*FALLTHROUGH*/+ case 6:+ hval += asso_values[(unsigned char)str[5]];+ /*FALLTHROUGH*/+ case 5:+ hval += asso_values[(unsigned char)str[4]];+ /*FALLTHROUGH*/+ case 4:+ hval += asso_values[(unsigned char)str[3]+1];+ /*FALLTHROUGH*/+ case 3:+ case 2:+ case 1:+ hval += asso_values[(unsigned char)str[0]];+ break;+ }+ return hval;+}++const char *+is_keyword_m4 (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("os2")];+ char stringpool_str4[sizeof("eval")];+ char stringpool_str6[sizeof("m4_len")];+ char stringpool_str9[sizeof("exch")];+ char stringpool_str12[sizeof("m4_line")];+ char stringpool_str14[sizeof("defn")];+ char stringpool_str17[sizeof("m4_defn")];+ char stringpool_str22[sizeof("traceon")];+ char stringpool_str23[sizeof("maketemp")];+ char stringpool_str24[sizeof("file")];+ char stringpool_str27[sizeof("m4_cond")];+ char stringpool_str30[sizeof("ifdef")];+ char stringpool_str31[sizeof("divnum")];+ char stringpool_str33[sizeof("traceoff")];+ char stringpool_str34[sizeof("m4_define")];+ char stringpool_str37[sizeof("include")];+ char stringpool_str38[sizeof("len")];+ char stringpool_str39[sizeof("join")];+ char stringpool_str40[sizeof("m4_define_blind")];+ char stringpool_str41[sizeof("popdef")];+ char stringpool_str42[sizeof("forloop")];+ char stringpool_str44[sizeof("cond")];+ char stringpool_str46[sizeof("dquote")];+ char stringpool_str47[sizeof("mkstemp")];+ char stringpool_str48[sizeof("undefine")];+ char stringpool_str49[sizeof("line")];+ char stringpool_str51[sizeof("define")];+ char stringpool_str53[sizeof("m4_index")];+ char stringpool_str54[sizeof("copy")];+ char stringpool_str57[sizeof("pushdef")];+ char stringpool_str58[sizeof("m4_indir")];+ char stringpool_str60[sizeof("m4_dumpdef")];+ char stringpool_str62[sizeof("esyscmd")];+ char stringpool_str63[sizeof("m4_ifdef")];+ char stringpool_str64[sizeof("incr")];+ char stringpool_str66[sizeof("ifelse")];+ char stringpool_str67[sizeof("example")];+ char stringpool_str68[sizeof("m4_quote")];+ char stringpool_str69[sizeof("decr")];+ char stringpool_str71[sizeof("divert")];+ char stringpool_str73[sizeof("gnu")];+ char stringpool_str74[sizeof("argn")];+ char stringpool_str81[sizeof("syscmd")];+ char stringpool_str82[sizeof("windows")];+ char stringpool_str85[sizeof("indir")];+ char stringpool_str86[sizeof("m4_gnu")];+ char stringpool_str87[sizeof("m4_decr")];+ char stringpool_str90[sizeof("dquote_elt")];+ char stringpool_str91[sizeof("format")];+ char stringpool_str93[sizeof("m4_shift")];+ char stringpool_str97[sizeof("define_blind")];+ char stringpool_str99[sizeof("m4_ifelse")];+ char stringpool_str101[sizeof("__windows__")];+ char stringpool_str108[sizeof("translit")];+ char stringpool_str111[sizeof("substr")];+ char stringpool_str112[sizeof("m4_join")];+ char stringpool_str113[sizeof("m4_curry")];+ char stringpool_str115[sizeof("m4_joinall")];+ char stringpool_str116[sizeof("m4_os2")];+ char stringpool_str117[sizeof("m4_file")];+ char stringpool_str120[sizeof("m4_include")];+ char stringpool_str122[sizeof("m4_incr")];+ char stringpool_str123[sizeof("__line__")];+ char stringpool_str125[sizeof("nargs")];+ char stringpool_str129[sizeof("m4_cleardivert")];+ char stringpool_str130[sizeof("m4_foreach")];+ char stringpool_str131[sizeof("m4_foreachq")];+ char stringpool_str133[sizeof("sinclude")];+ char stringpool_str134[sizeof("m4_format")];+ char stringpool_str135[sizeof("shift")];+ char stringpool_str136[sizeof("m4_translit")];+ char stringpool_str137[sizeof("builtin")];+ char stringpool_str140[sizeof("m4_builtin")];+ char stringpool_str141[sizeof("m4_downcase")];+ char stringpool_str142[sizeof("__gnu__")];+ char stringpool_str144[sizeof("m4_rename")];+ char stringpool_str146[sizeof("m4exit")];+ char stringpool_str147[sizeof("m4_debugmode")];+ char stringpool_str149[sizeof("unix")];+ char stringpool_str152[sizeof("reverse")];+ char stringpool_str156[sizeof("m4wrap")];+ char stringpool_str158[sizeof("__file__")];+ char stringpool_str159[sizeof("changecom")];+ char stringpool_str161[sizeof("m4_errprint")];+ char stringpool_str162[sizeof("__os2__")];+ char stringpool_str164[sizeof("m4_divnum")];+ char stringpool_str165[sizeof("index")];+ char stringpool_str166[sizeof("rename")];+ char stringpool_str167[sizeof("m4_changecom")];+ char stringpool_str168[sizeof("m4_changeword")];+ char stringpool_str169[sizeof("m4_changequote")];+ char stringpool_str170[sizeof("m4_mkstemp")];+ char stringpool_str172[sizeof("m4_copy")];+ char stringpool_str174[sizeof("m4_m4exit")];+ char stringpool_str175[sizeof("quote")];+ char stringpool_str176[sizeof("sysval")];+ char stringpool_str179[sizeof("m4_divert")];+ char stringpool_str180[sizeof("m4_forloop")];+ char stringpool_str182[sizeof("joinall")];+ char stringpool_str183[sizeof("downcase")];+ char stringpool_str185[sizeof("changeword")];+ char stringpool_str186[sizeof("m4_undivert")];+ char stringpool_str188[sizeof("m4_nargs")];+ char stringpool_str189[sizeof("m4_dquote")];+ char stringpool_str191[sizeof("m4_maketemp")];+ char stringpool_str192[sizeof("dumpdef")];+ char stringpool_str193[sizeof("m4_dquote_elt")];+ char stringpool_str195[sizeof("capitalize")];+ char stringpool_str196[sizeof("upcase")];+ char stringpool_str200[sizeof("m4_traceon")];+ char stringpool_str201[sizeof("m4_traceoff")];+ char stringpool_str202[sizeof("foreach")];+ char stringpool_str203[sizeof("foreachq")];+ char stringpool_str206[sizeof("m4_undefine")];+ char stringpool_str208[sizeof("errprint")];+ char stringpool_str211[sizeof("m4_sinclude")];+ char stringpool_str214[sizeof("m4___windows__")];+ char stringpool_str216[sizeof("__program__")];+ char stringpool_str217[sizeof("m4_debugfile")];+ char stringpool_str219[sizeof("m4_m4wrap")];+ char stringpool_str220[sizeof("m4_reverse")];+ char stringpool_str221[sizeof("m4___file__")];+ char stringpool_str226[sizeof("regexp")];+ char stringpool_str228[sizeof("patsubst")];+ char stringpool_str230[sizeof("curry")];+ char stringpool_str235[sizeof("m4_example")];+ char stringpool_str236[sizeof("cleardivert")];+ char stringpool_str237[sizeof("m4_argn")];+ char stringpool_str238[sizeof("__unix__")];+ char stringpool_str239[sizeof("m4_syscmd")];+ char stringpool_str243[sizeof("m4_capitalize")];+ char stringpool_str246[sizeof("m4___line__")];+ char stringpool_str248[sizeof("undivert")];+ char stringpool_str249[sizeof("debugmode")];+ char stringpool_str259[sizeof("m4_substr")];+ char stringpool_str260[sizeof("m4_esyscmd")];+ char stringpool_str264[sizeof("debugfile")];+ char stringpool_str267[sizeof("m4_exch")];+ char stringpool_str269[sizeof("m4_popdef")];+ char stringpool_str271[sizeof("fatal_error")];+ char stringpool_str272[sizeof("m4_eval")];+ char stringpool_str278[sizeof("m4_array")];+ char stringpool_str281[sizeof("m4_stack_foreach")];+ char stringpool_str283[sizeof("stack_foreach")];+ char stringpool_str285[sizeof("m4_stack_foreach_sep")];+ char stringpool_str286[sizeof("m4_stack_foreach_lifo")];+ char stringpool_str287[sizeof("stack_foreach_sep")];+ char stringpool_str288[sizeof("stack_foreach_lifo")];+ char stringpool_str289[sizeof("m4_regexp")];+ char stringpool_str290[sizeof("m4_stack_foreach_sep_lifo")];+ char stringpool_str292[sizeof("stack_foreach_sep_lifo")];+ char stringpool_str301[sizeof("changequote")];+ char stringpool_str304[sizeof("m4_upcase")];+ char stringpool_str307[sizeof("m4_unix")];+ char stringpool_str310[sizeof("m4_pushdef")];+ char stringpool_str315[sizeof("m4_windows")];+ char stringpool_str325[sizeof("array")];+ char stringpool_str329[sizeof("m4___program__")];+ char stringpool_str336[sizeof("m4_patsubst")];+ char stringpool_str341[sizeof("m4___unix__")];+ char stringpool_str349[sizeof("m4_sysval")];+ char stringpool_str374[sizeof("m4_fatal_error")];+ char stringpool_str395[sizeof("m4___os2__")];+ char stringpool_str410[sizeof("m4___gnu__")];+ char stringpool_str427[sizeof("m4_array_set")];+ char stringpool_str439[sizeof("array_set")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "os2",+ "eval",+ "m4_len",+ "exch",+ "m4_line",+ "defn",+ "m4_defn",+ "traceon",+ "maketemp",+ "file",+ "m4_cond",+ "ifdef",+ "divnum",+ "traceoff",+ "m4_define",+ "include",+ "len",+ "join",+ "m4_define_blind",+ "popdef",+ "forloop",+ "cond",+ "dquote",+ "mkstemp",+ "undefine",+ "line",+ "define",+ "m4_index",+ "copy",+ "pushdef",+ "m4_indir",+ "m4_dumpdef",+ "esyscmd",+ "m4_ifdef",+ "incr",+ "ifelse",+ "example",+ "m4_quote",+ "decr",+ "divert",+ "gnu",+ "argn",+ "syscmd",+ "windows",+ "indir",+ "m4_gnu",+ "m4_decr",+ "dquote_elt",+ "format",+ "m4_shift",+ "define_blind",+ "m4_ifelse",+ "__windows__",+ "translit",+ "substr",+ "m4_join",+ "m4_curry",+ "m4_joinall",+ "m4_os2",+ "m4_file",+ "m4_include",+ "m4_incr",+ "__line__",+ "nargs",+ "m4_cleardivert",+ "m4_foreach",+ "m4_foreachq",+ "sinclude",+ "m4_format",+ "shift",+ "m4_translit",+ "builtin",+ "m4_builtin",+ "m4_downcase",+ "__gnu__",+ "m4_rename",+ "m4exit",+ "m4_debugmode",+ "unix",+ "reverse",+ "m4wrap",+ "__file__",+ "changecom",+ "m4_errprint",+ "__os2__",+ "m4_divnum",+ "index",+ "rename",+ "m4_changecom",+ "m4_changeword",+ "m4_changequote",+ "m4_mkstemp",+ "m4_copy",+ "m4_m4exit",+ "quote",+ "sysval",+ "m4_divert",+ "m4_forloop",+ "joinall",+ "downcase",+ "changeword",+ "m4_undivert",+ "m4_nargs",+ "m4_dquote",+ "m4_maketemp",+ "dumpdef",+ "m4_dquote_elt",+ "capitalize",+ "upcase",+ "m4_traceon",+ "m4_traceoff",+ "foreach",+ "foreachq",+ "m4_undefine",+ "errprint",+ "m4_sinclude",+ "m4___windows__",+ "__program__",+ "m4_debugfile",+ "m4_m4wrap",+ "m4_reverse",+ "m4___file__",+ "regexp",+ "patsubst",+ "curry",+ "m4_example",+ "cleardivert",+ "m4_argn",+ "__unix__",+ "m4_syscmd",+ "m4_capitalize",+ "m4___line__",+ "undivert",+ "debugmode",+ "m4_substr",+ "m4_esyscmd",+ "debugfile",+ "m4_exch",+ "m4_popdef",+ "fatal_error",+ "m4_eval",+ "m4_array",+ "m4_stack_foreach",+ "stack_foreach",+ "m4_stack_foreach_sep",+ "m4_stack_foreach_lifo",+ "stack_foreach_sep",+ "stack_foreach_lifo",+ "m4_regexp",+ "m4_stack_foreach_sep_lifo",+ "stack_foreach_sep_lifo",+ "changequote",+ "m4_upcase",+ "m4_unix",+ "m4_pushdef",+ "m4_windows",+ "array",+ "m4___program__",+ "m4_patsubst",+ "m4___unix__",+ "m4_sysval",+ "m4_fatal_error",+ "m4___os2__",+ "m4___gnu__",+ "m4_array_set",+ "array_set"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str81,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str86,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str87,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str90,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str91,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str93,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str97,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str101,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str108,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str111,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str112,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str113,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str117,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str120,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str123,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str129,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str130,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str131,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str133,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str134,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str135,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str137,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str140,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str141,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str142,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str144,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str146,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str147,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str149,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str152,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str156,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str158,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str159,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str161,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str162,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str164,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str165,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str166,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str167,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str168,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str169,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str170,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str172,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str174,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str175,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str176,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str179,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str180,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str182,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str183,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str185,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str186,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str188,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str189,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str191,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str192,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str193,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str195,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str196,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str200,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str201,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str202,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str203,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str206,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str208,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str211,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str214,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str216,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str217,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str219,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str220,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str221,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str226,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str228,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str230,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str235,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str236,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str237,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str238,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str239,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str243,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str246,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str248,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str249,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str259,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str260,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str264,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str267,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str269,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str271,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str272,+ -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str278,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str281,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str283,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str285,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str286,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str287,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str288,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str289,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str290,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str292,+ -1, -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str301,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str304,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str307,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str310,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str315,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str325,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str329,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str336,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str341,+ -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str349,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str374,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str395,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str410,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str427,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str439+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_m4.gperf b/llamafile/is_keyword_m4.gperfnew file mode 100644index 00000000..41c11097--- /dev/null+++ b/llamafile/is_keyword_m4.gperf@@ -0,0 +1,175 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_m4+%%+__file__+__gnu__+__line__+__os2__+__program__+__unix__+__windows__+argn+array+array_set+builtin+capitalize+changecom+changequote+changeword+cleardivert+cond+copy+curry+debugfile+debugmode+decr+define+define_blind+defn+divert+divnum+downcase+dquote+dquote_elt+dumpdef+errprint+esyscmd+eval+example+exch+fatal_error+file+foreach+foreachq+forloop+format+gnu+ifdef+ifelse+include+incr+index+indir+join+joinall+len+line+m4___file__+m4___gnu__+m4___line__+m4___os2__+m4___program__+m4___unix__+m4___windows__+m4_argn+m4_array+m4_array_set+m4_builtin+m4_capitalize+m4_changecom+m4_changequote+m4_changeword+m4_cleardivert+m4_cond+m4_copy+m4_curry+m4_debugfile+m4_debugmode+m4_decr+m4_define+m4_define_blind+m4_defn+m4_divert+m4_divnum+m4_downcase+m4_dquote+m4_dquote_elt+m4_dumpdef+m4_errprint+m4_esyscmd+m4_eval+m4_example+m4_exch+m4_fatal_error+m4_file+m4_foreach+m4_foreachq+m4_forloop+m4_format+m4_gnu+m4_ifdef+m4_ifelse+m4_include+m4_incr+m4_index+m4_indir+m4_join+m4_joinall+m4_len+m4_line+m4_m4exit+m4_m4wrap+m4_maketemp+m4_mkstemp+m4_nargs+m4_os2+m4_patsubst+m4_popdef+m4_pushdef+m4_quote+m4_regexp+m4_rename+m4_reverse+m4_shift+m4_sinclude+m4_stack_foreach+m4_stack_foreach_lifo+m4_stack_foreach_sep+m4_stack_foreach_sep_lifo+m4_substr+m4_syscmd+m4_sysval+m4_traceoff+m4_traceon+m4_translit+m4_undefine+m4_undivert+m4_unix+m4_upcase+m4_windows+m4exit+m4wrap+maketemp+mkstemp+nargs+os2+patsubst+popdef+pushdef+quote+regexp+rename+reverse+shift+sinclude+stack_foreach+stack_foreach_lifo+stack_foreach_sep+stack_foreach_sep_lifo+substr+syscmd+sysval+traceoff+traceon+translit+undefine+undivert+unix+upcase+windowsdiff --git a/llamafile/string.cpp b/llamafile/string.cppnew file mode 100644index 00000000..1eff49c0--- /dev/null+++ b/llamafile/string.cpp@@ -0,0 +1,53 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "string.h"++#include <cosmo.h>++wchar_t read_wchar(const std::string_view &s, size_t *i) {+ wint_t c = s[(*i)++] & 255;+ if (c >= 0300) {+ wint_t a = ThomPikeByte(c);+ size_t m = ThomPikeLen(c) - 1;+ if (*i + m <= s.size()) {+ for (int j = 0;;) {+ wint_t b = s[*i + j] & 255;+ if (!ThomPikeCont(b))+ break;+ a = ThomPikeMerge(a, b);+ if (++j == m) {+ c = a;+ *i += j;+ break;+ }+ }+ }+ }+ return c;+}++void append_wchar(std::string *r, wchar_t c) {+ if (isascii(c)) {+ *r += c;+ } else {+ char s[8];+ uint64_t w = tpenc(c);+ WRITE64LE(s, w);+ *r += s;+ }+}diff --git a/llamafile/string.h b/llamafile/string.hnew file mode 100644index 00000000..996c0a82--- /dev/null+++ b/llamafile/string.h@@ -0,0 +1,22 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <string>++void append_wchar(std::string *, wchar_t);+wchar_t read_wchar(const std::string_view &, size_t *);