Skip to content

Distribute and run LLMs with a single file.

Add m4 syntax highlighting

Justine Tunneyf28e95490b02
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 24aeb0de..83588f1e 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -18,110 +18,145 @@
#include "highlight.h"
Highlight *Highlight::create(const std::string_view &lang) {
- if (lang == "markdown" || lang == "md") {
+
+ if (lang == "md" || //
+ lang == "markdown")
return new HighlightMarkdown;
- } else if (lang == "c" || //
- lang == "h" || //
- lang == "m") {
+
+ if (lang == "c" || //
+ lang == "h" || //
+ lang == "m")
return new HighlightC(is_keyword_c, //
is_keyword_c_type, //
is_keyword_c_builtin, //
is_keyword_c_constant);
- } else if (lang == "c++" || //
- lang == "cxx" || //
- lang == "cpp" || //
- lang == "hpp" || //
- lang == "cc" || //
- lang == "cu") {
+
+ if (lang == "c++" || //
+ lang == "cxx" || //
+ lang == "cpp" || //
+ lang == "hpp" || //
+ lang == "cc" || //
+ lang == "cu")
return new HighlightC(is_keyword_cxx, //
is_keyword_c_type, //
is_keyword_c_builtin, //
is_keyword_c_constant);
- } else if (lang == "js" || //
- lang == "json" || //
- lang == "javascript") {
+
+ if (lang == "js" || //
+ lang == "json" || //
+ lang == "javascript")
return new HighlightJs(is_keyword_js, nullptr);
- } else if (lang == "ts" || //
- lang == "typescript") {
+
+ if (lang == "ts" || //
+ lang == "typescript")
return new HighlightJs(is_keyword_typescript, is_keyword_typescript_type);
- } else if (lang == "java") {
- return new HighlightC(is_keyword_java);
- } else if (lang == "py" || //
- lang == "python") {
+
+ if (lang == "java")
+ return new HighlightJava(is_keyword_java);
+
+ if (lang == "py" || //
+ lang == "python")
return new HighlightPython;
- } else if (lang == "rs" || //
- lang == "rust") {
+
+ if (lang == "rs" || //
+ lang == "rust")
return new HighlightRust;
- } else if (lang == "f" || //
- lang == "fortran") {
+
+ if (lang == "f" || //
+ lang == "fortran")
return new HighlightFortran;
- } else if (lang == "cob" || //
- lang == "cbl" || //
- lang == "cobol") {
+
+ if (lang == "cob" || //
+ lang == "cbl" || //
+ lang == "cobol")
return new HighlightCobol;
- } else if (lang == "pas" || //
- lang == "pascal") {
+
+ if (lang == "pas" || //
+ lang == "pascal")
return new HighlightPascal;
- } else if (lang == "go") {
+
+ if (lang == "go")
return new HighlightGo;
- } else if (lang == "sql") {
+
+ if (lang == "sql")
return new HighlightSql;
- } else if (lang == "css") {
+
+ if (lang == "css")
return new HighlightCss;
- } else if (lang == "html" || //
- lang == "xml") {
+
+ if (lang == "html" || //
+ lang == "xml")
return new HighlightHtml;
- } else if (lang == "php") {
+
+ if (lang == "php")
return new HighlightHtml; // sic
- } else if (lang == "php!") {
+
+ if (lang == "php!")
return new HighlightPhp;
- } else if (lang == "csharp" || //
- lang == "cs" || //
- lang == "c#") {
+
+ if (lang == "csharp" || //
+ lang == "cs" || //
+ lang == "c#")
return new HighlightCsharp;
- } else if (lang == "kotlin") {
- return new HighlightC(is_keyword_kotlin);
- } else if (lang == "lua") {
+
+ if (lang == "kotlin")
+ return new HighlightJava(is_keyword_kotlin);
+
+ if (lang == "lua")
return new HighlightLua;
- } else if (lang == "lisp" || //
- lang == "el" || //
- lang == "elisp" || //
- lang == "cl" || //
- lang == "clisp" || //
- lang == "scheme" || //
- lang == "clojure") {
+
+ if (lang == "lisp" || //
+ lang == "el" || //
+ lang == "elisp" || //
+ lang == "cl" || //
+ lang == "clisp" || //
+ lang == "scheme" || //
+ lang == "clojure")
return new HighlightLisp;
- } else if (lang == "ada") {
+
+ if (lang == "ada")
return new HighlightAda;
- } else if (lang == "haskell" || //
- lang == "hs") {
+
+ if (lang == "haskell" || //
+ lang == "hs")
return new HighlightHaskell;
- } else if (lang == "perl" || //
- lang == "pl") {
+
+ if (lang == "perl" || //
+ lang == "pl")
return new HighlightPerl;
- } else if (lang == "shell" || //
- lang == "bash" || //
- lang == "sh" || //
- lang == "ksh") {
+
+ if (lang == "shell" || //
+ lang == "bash" || //
+ lang == "sh" || //
+ lang == "ksh")
return new HighlightShell;
- } else if (lang == "swift") {
+
+ if (lang == "swift")
return new HighlightC(is_keyword_swift, is_keyword_swift_type);
- } else if (lang == "d") {
+
+ if (lang == "d")
return new HighlightC(is_keyword_d);
- } else if (lang == "zig") {
+
+ if (lang == "zig")
return new HighlightZig;
- } else if (lang == "tcl") {
+
+ if (lang == "tcl")
return new HighlightTcl;
- } else if (lang == "ruby" || //
- lang == "rb") {
+
+ if (lang == "m4" || //
+ lang == "ac")
+ return new HighlightM4;
+
+ if (lang == "ruby" || //
+ lang == "rb")
return new HighlightRuby;
- } else if (lang == "fs" || //
- lang == "4th" || //
- lang == "frt" || //
- lang == "fth" || //
- lang == "forth") {
+
+ if (lang == "fs" || //
+ lang == "4th" || //
+ lang == "frt" || //
+ lang == "fth" || //
+ lang == "forth")
return new HighlightForth;
- } else {
- return new HighlightPlain;
- }
+
+ return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 8c409abf..ca685651 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -58,6 +58,7 @@ is_keyword_f is_keyword_js;
is_keyword_f is_keyword_js_builtin;
is_keyword_f is_keyword_js_constant;
is_keyword_f is_keyword_java;
+is_keyword_f is_keyword_java_constant;
is_keyword_f is_keyword_python;
is_keyword_f is_keyword_rust;
is_keyword_f is_keyword_rust_type;
@@ -104,6 +105,7 @@ is_keyword_f is_keyword_typescript_type;
is_keyword_f is_keyword_typescript_constant;
is_keyword_f is_keyword_forth;
is_keyword_f is_keyword_forth_def;
+is_keyword_f is_keyword_m4;
}
class Highlight {
@@ -162,6 +164,19 @@ class HighlightC : public Highlight {
is_keyword_f *is_constant_;
};
+class HighlightJava : public Highlight {
+ public:
+ explicit HighlightJava(is_keyword_f *is_keyword);
+ ~HighlightJava() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+ is_keyword_f *is_keyword_;
+};
+
class HighlightGo : public Highlight {
public:
HighlightGo();
@@ -479,3 +494,15 @@ class HighlightForth : public Highlight {
bool is_label_ = false;
std::string word_;
};
+
+class HighlightM4 : public Highlight {
+ public:
+ HighlightM4();
+ ~HighlightM4() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cpp
index 6c2bcbf3..6047f297 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight_c.cpp
@@ -33,6 +33,8 @@ enum {
R,
R_DQUOTE,
RAW,
+ QUESTION,
+ TRIGRAPH,
};
HighlightC::HighlightC(is_keyword_f *is_keyword, //
@@ -63,6 +65,8 @@ void HighlightC::feed(std::string *r, std::string_view input) {
word_ += c;
} else if (c == '/') {
t_ = SLASH;
+ } else if (c == '?') {
+ t_ = QUESTION;
} else if (c == '\'') {
t_ = QUOTE;
*r += HI_STRING;
@@ -112,6 +116,38 @@ void HighlightC::feed(std::string *r, std::string_view input) {
}
break;
+ case QUESTION:
+ if (c == '?') {
+ t_ = TRIGRAPH;
+ } else {
+ *r += '?';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case TRIGRAPH:
+ if (c == '=' || // '#'
+ c == '(' || // '['
+ c == '/' || // '\\'
+ c == ')' || // ']'
+ c == '\'' || // '^'
+ c == '<' || // '{'
+ c == '!' || // '|'
+ c == '>' || // '}'
+ c == '-') { // '~'
+ *r += HI_ESCAPE;
+ *r += "??";
+ *r += c;
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += "??";
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
case SLASH:
if (c == '/') {
*r += HI_COMMENT;
@@ -254,6 +290,12 @@ void HighlightC::flush(std::string *r) {
case SLASH:
*r += '/';
break;
+ case QUESTION:
+ *r += '?';
+ break;
+ case TRIGRAPH:
+ *r += "??";
+ break;
case R:
*r += 'R';
break;
diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight_c_test.cpp
index 5ca13e43..e4cdc2c4 100644
--- a/llamafile/highlight_c_test.cpp
+++ b/llamafile/highlight_c_test.cpp
@@ -25,53 +25,49 @@ using namespace std::string_literals;
HighlightC g_highlight;
void test(std::string input, std::string expect) {
- std::string got;
- g_highlight.feed(&got, input);
- if (got != expect) {
- fprintf(stderr,
- "error: highlighting failed\n"
- "\t input: %`'s\n"
- "\texpected: %`'s\n"
- "\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
- exit(1);
- }
+ std::string got;
+ g_highlight.feed(&got, input);
+ if (got != expect) {
+ fprintf(stderr,
+ "error: highlighting failed\n"
+ "\t input: %`'s\n"
+ "\texpected: %`'s\n"
+ "\t got: %`'s\n",
+ input.c_str(), expect.c_str(), got.c_str());
+ exit(1);
+ }
}
void testf(std::string input, std::string expect) {
- std::string got;
- g_highlight.feed(&got, input);
- g_highlight.flush(&got);
- if (got != expect) {
- fprintf(stderr,
- "error: highlighting failed\n"
- "\t input: %`'s\n"
- "\texpected: %`'s\n"
- "\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
- exit(1);
- }
+ std::string got;
+ g_highlight.feed(&got, input);
+ g_highlight.flush(&got);
+ if (got != expect) {
+ fprintf(stderr,
+ "error: highlighting failed\n"
+ "\t input: %`'s\n"
+ "\texpected: %`'s\n"
+ "\t got: %`'s\n",
+ input.c_str(), expect.c_str(), got.c_str());
+ exit(1);
+ }
}
int main(int argc, char *argv[]) {
- test("", "");
- test("vola", "");
- test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");
- test("yo // hi\nthere ",
- "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");
- testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);
- test("yo /* hi */ there ",
- "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");
- testf("yo /* hi */ there",
- "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");
- testf("yo /* h*i */ there",
- "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");
- testf("vola", "vola");
- testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);
- testf("/hi/", "/hi/");
- testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");
- testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");
- testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");
- testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");
- testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");
+ test("", "");
+ test("vola", "");
+ test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");
+ test("yo // hi\nthere ", "yo "s + HI_COMMENT + "// hi" + HI_RESET + "\nthere ");
+ testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);
+ test("yo /* hi */ there ", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");
+ testf("yo /* hi */ there", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");
+ testf("yo /* h*i */ there", "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");
+ testf("vola", "vola");
+ testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);
+ testf("/hi/", "/hi/");
+ testf("x '' y", "x "s + HI_STRING + "''" + HI_RESET + " y");
+ testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");
+ testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");
+ testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");
+ testf("x \"\\\"\" y", "x "s + HI_STRING + "\"\\\"\"" + HI_RESET + " y");
}
diff --git a/llamafile/highlight_fortran.cpp b/llamafile/highlight_fortran.cpp
index e7835e40..12322c20 100644
--- a/llamafile/highlight_fortran.cpp
+++ b/llamafile/highlight_fortran.cpp
@@ -87,7 +87,7 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {
Word:
case WORD:
- if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {
+ if (!isascii(c) || isalnum(c) || c == '_') {
word_ += c;
} else if (c == '.' && word_[0] == '.') {
word_ += c;
diff --git a/llamafile/highlight_java.cpp b/llamafile/highlight_java.cpp
new file mode 100644
index 00000000..545ae481
--- /dev/null
+++ b/llamafile/highlight_java.cpp
@@ -0,0 +1,283 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ ANNOTATION,
+ ANNOTATION2,
+ SLASH,
+ SLASH_SLASH,
+ SLASH_STAR,
+ SLASH_STAR_STAR,
+ DQUOTE, // "
+ DQUOTESTR, // "...
+ DQUOTESTR_BACKSLASH, // "...
+ DQUOTE2, // ""
+ DQUOTE3, // """...
+ DQUOTE31, // """..."
+ DQUOTE32, // """...""
+};
+
+HighlightJava::HighlightJava(is_keyword_f is_keyword) : is_keyword_(is_keyword) {
+}
+
+HighlightJava::~HighlightJava() {
+}
+
+void HighlightJava::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '/') {
+ t_ = SLASH;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += '\'';
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += '"';
+ } else if (c == '@') {
+ t_ = ANNOTATION;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_java_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH:
+ if (c == '/') {
+ *r += HI_COMMENT;
+ *r += "//";
+ t_ = SLASH_SLASH;
+ } else if (c == '*') {
+ *r += HI_COMMENT;
+ *r += "/*";
+ t_ = SLASH_STAR;
+ } else {
+ *r += '/';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case SLASH_SLASH:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += '\n';
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case SLASH_STAR:
+ *r += c;
+ if (c == '*')
+ t_ = SLASH_STAR_STAR;
+ break;
+
+ case SLASH_STAR_STAR:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c != '*') {
+ t_ = SLASH_STAR;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ // handle "string"
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ t_ = DQUOTE2;
+ } else if (c == '\\') {
+ t_ = DQUOTESTR_BACKSLASH;
+ } else {
+ t_ = DQUOTESTR;
+ }
+ break;
+ case DQUOTESTR:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTESTR_BACKSLASH;
+ }
+ break;
+ case DQUOTESTR_BACKSLASH:
+ *r += c;
+ t_ = DQUOTESTR;
+ break;
+
+ // handle """string""" from java 15+
+ case DQUOTE2:
+ if (c == '"') {
+ *r += '"';
+ t_ = DQUOTE3;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+ case DQUOTE3:
+ *r += c;
+ if (c == '"')
+ t_ = DQUOTE31;
+ break;
+ case DQUOTE31:
+ *r += c;
+ if (c == '"') {
+ t_ = DQUOTE32;
+ } else {
+ t_ = DQUOTE3;
+ }
+ break;
+ case DQUOTE32:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ t_ = DQUOTE3;
+ }
+ break;
+
+ case ANNOTATION:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ *r += HI_ATTRIB;
+ *r += '@';
+ *r += c;
+ t_ = ANNOTATION2;
+ } else {
+ *r += '@';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case ANNOTATION2:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightJava::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_java_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case SLASH:
+ *r += '/';
+ break;
+ case ANNOTATION:
+ *r += '@';
+ break;
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case SLASH_SLASH:
+ case SLASH_STAR:
+ case SLASH_STAR_STAR:
+ case DQUOTE:
+ case DQUOTESTR:
+ case DQUOTESTR_BACKSLASH:
+ case DQUOTE2:
+ case DQUOTE3:
+ case DQUOTE31:
+ case DQUOTE32:
+ case ANNOTATION2:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_m4.cpp b/llamafile/highlight_m4.cpp
new file mode 100644
index 00000000..06f323b1
--- /dev/null
+++ b/llamafile/highlight_m4.cpp
@@ -0,0 +1,134 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ COMMENT,
+ DOLLAR,
+};
+
+HighlightM4::HighlightM4() {
+}
+
+HighlightM4::~HighlightM4() {
+}
+
+void HighlightM4::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '#') {
+ t_ = COMMENT;
+ *r += HI_COMMENT;
+ *r += '#';
+ } else if (c == '$') {
+ t_ = DOLLAR;
+ } else {
+ *r += c;
+ }
+ break;
+
+ Word:
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_m4(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (word_ == "dnl" || word_ == "m4_dnl") {
+ *r += HI_COMMENT;
+ *r += word_;
+ word_.clear();
+ t_ = COMMENT;
+ goto Comment;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case DOLLAR:
+ if (isdigit(c) || c == '*' || c == '#' || c == '@') {
+ *r += '$';
+ *r += HI_VAR;
+ *r += c;
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += '$';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ Comment:
+ case COMMENT:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += '\n';
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightM4::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_m4(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case DOLLAR:
+ *r += '$';
+ break;
+ case COMMENT:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cpp
index 3b0b9533..208aa24e 100644
--- a/llamafile/highlight_markdown.cpp
+++ b/llamafile/highlight_markdown.cpp
@@ -16,6 +16,8 @@
// limitations under the License.
#include "highlight.h"
+#include "string.h"
+#include <cosmo.h>
enum {
NORMAL,
@@ -44,9 +46,8 @@ HighlightMarkdown::~HighlightMarkdown() {
}
void HighlightMarkdown::feed(std::string *r, std::string_view input) {
- char c;
- for (size_t i = 0; i < input.size(); ++i) {
- c = input[i];
+ for (size_t i = 0; i < input.size();) {
+ wchar_t c = read_wchar(input, &i);
switch (t_) {
Normal:
@@ -60,9 +61,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
} else if (c == '\\') {
// handle \*\*not bold\*\* etc.
t_ = BACKSLASH;
- *r += c;
+ *r += '\\';
} else {
- *r += c;
+ append_wchar(r, c);
}
if (c == '\n') {
bol_ = true;
@@ -75,7 +76,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
break;
case BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
break;
@@ -84,18 +85,17 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
// handle **strong** text
t_ = STRONG;
*r += HI_BOLD;
- *r += '*';
- *r += c;
+ *r += "**";
} else if (bol_ && isblank(c)) {
*r += '*';
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
// handle *emphasized* text
// inverted because \e[3m has a poorly supported western bias
*r += '*';
*r += HI_ITALIC;
- *r += c;
+ append_wchar(r, c);
t_ = EMPHASIS;
if (c == '\\')
t_ = EMPHASIS_BACKSLASH;
@@ -107,23 +107,23 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
if (c == '*') {
t_ = NORMAL;
*r += HI_RESET;
- *r += c;
+ *r += '*';
} else if (c == '\\') {
t_ = EMPHASIS_BACKSLASH;
- *r += c;
+ *r += '\\';
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case EMPHASIS_BACKSLASH:
// so we can say *unbroken \* italic* and have it work
- *r += c;
+ append_wchar(r, c);
t_ = EMPHASIS;
break;
case STRONG:
- *r += c;
+ append_wchar(r, c);
if (c == '*') {
t_ = STRONG_STAR;
} else if (c == '\\') {
@@ -133,12 +133,12 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
case STRONG_BACKSLASH:
// so we can say **unbroken \*\* bold** and have it work
- *r += c;
+ append_wchar(r, c);
t_ = STRONG;
break;
case STRONG_STAR:
- *r += c;
+ append_wchar(r, c);
if (c == '*' || // handle **bold** ending
(c == '\n' && !tail_)) { // handle *** line break
t_ = NORMAL;
@@ -156,7 +156,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
} else {
*r += HI_INCODE;
*r += '`';
- *r += c;
+ append_wchar(r, c);
t_ = INCODE;
}
break;
@@ -164,7 +164,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
case INCODE:
// this is for `inline code` like that
// no backslash escapes are supported here
- *r += c;
+ append_wchar(r, c);
if (c == '`') {
*r += HI_RESET;
t_ = NORMAL;
@@ -174,14 +174,14 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
case INCODE2:
// this is for ``inline ` code`` like that
// it lets you put backtick inside the code
- *r += c;
+ append_wchar(r, c);
if (c == '`') {
t_ = INCODE2_TICK;
}
break;
case INCODE2_TICK:
- *r += c;
+ append_wchar(r, c);
if (c == '`') {
*r += HI_RESET;
t_ = NORMAL;
@@ -197,14 +197,14 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
} else {
*r += HI_INCODE;
*r += "``";
- *r += c;
+ append_wchar(r, c);
t_ = INCODE2;
}
break;
case LANG:
if (!isascii(c) || !isspace(c)) {
- *r += c;
+ append_wchar(r, c);
lang_ += tolower(c);
} else {
highlighter_ = Highlight::create(lang_);
@@ -219,7 +219,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
if (c == '`') {
t_ = CODE_TICK;
} else {
- char cs[2] = {c};
+ char cs[8];
+ uint64_t w = tpenc(c);
+ WRITE64LE(cs, w);
highlighter_->feed(r, cs);
}
break;
@@ -228,7 +230,10 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
if (c == '`') {
t_ = CODE_TICK_TICK;
} else {
- char cs[3] = {'`', c};
+ char cs[9];
+ uint64_t w = tpenc(c);
+ cs[0] = '`';
+ WRITE64LE(cs + 1, w);
highlighter_->feed(r, cs);
t_ = CODE;
}
@@ -242,7 +247,11 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
highlighter_ = nullptr;
*r += "```";
} else {
- char cs[4] = {'`', '`', c};
+ char cs[10];
+ uint64_t w = tpenc(c);
+ cs[0] = '`';
+ cs[1] = '`';
+ WRITE64LE(cs + 2, w);
highlighter_->feed(r, cs);
t_ = CODE;
}
diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cpp
index 1c365d89..d0894f43 100644
--- a/llamafile/highlight_perl.cpp
+++ b/llamafile/highlight_perl.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-
+#include "string.h"
#include <ctype.h>
enum {
@@ -148,52 +148,52 @@ HighlightPerl::~HighlightPerl() {
void HighlightPerl::feed(std::string *r, std::string_view input) {
int c = 0;
- for (size_t i = 0; i < input.size(); ++i) {
+ for (size_t i = 0; i < input.size();) {
last_ = c_;
- c_ = c = input[i] & 255;
+ c_ = c = read_wchar(input, &i);
switch (t_) {
Normal:
case NORMAL:
if (!isascii(c) || isalpha(c) || c == '_') {
t_ = WORD;
- word_ += c;
+ append_wchar(&word_, c);
} else if (c == '\'') {
t_ = QUOTE;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
expect_ = EXPECT_OPERATOR;
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
expect_ = EXPECT_OPERATOR;
} else if (c == '=' && (!last_ || last_ == '\n')) {
t_ = EQUAL;
} else if (c == '\\') {
t_ = BACKSLASH;
*r += HI_ESCAPE;
- *r += c;
+ append_wchar(r, c);
} else if (c == '`') {
t_ = TICK;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
expect_ = EXPECT_OPERATOR;
} else if (c == '$') {
- *r += c;
+ append_wchar(r, c);
t_ = VAR;
expect_ = EXPECT_OPERATOR;
} else if (c == '@' || c == '%') {
- *r += c;
+ append_wchar(r, c);
*r += HI_VAR;
t_ = VAR2;
expect_ = EXPECT_OPERATOR;
} else if (c == '#') {
*r += HI_COMMENT;
- *r += c;
+ append_wchar(r, c);
t_ = COMMENT;
} else if (c == '<') {
- *r += c;
+ append_wchar(r, c);
t_ = LT;
expect_ = EXPECT_VALUE;
} else if (c == '/' && expect_ == EXPECT_VALUE && last_ != '/') {
@@ -201,10 +201,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
closer_ = '/';
expect_ = EXPECT_OPERATOR;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
t_ = REGEX;
} else if (c == '\n') {
- *r += c;
+ append_wchar(r, c);
if (pending_heredoc_) {
*r += HI_STRING;
pending_heredoc_ = false;
@@ -213,22 +213,22 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
}
} else if (c == ')' || c == '}' || c == ']') {
expect_ = EXPECT_OPERATOR;
- *r += c;
+ append_wchar(r, c);
} else if (ispunct(c)) {
expect_ = EXPECT_VALUE;
- *r += c;
+ append_wchar(r, c);
} else if (isdigit(c)) {
expect_ = EXPECT_OPERATOR;
- *r += c;
+ append_wchar(r, c);
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
Word:
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
- word_ += c;
+ append_wchar(&word_, c);
} else {
if (is_keyword_perl(word_.data(), word_.size())) {
*r += HI_KEYWORD;
@@ -246,7 +246,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
opener_ = c;
closer_ = mirror(c);
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
if (is_double_regex(word_)) {
t_ = S_REGEX;
} else {
@@ -263,7 +263,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case BACKSLASH:
- *r += c;
+ append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
break;
@@ -271,13 +271,13 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case VAR:
if (isdigit(c) || is_magic_var(c)) {
*r += HI_VAR;
- *r += c;
+ append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
break;
} else if (c == '{') {
t_ = VAR2;
- *r += c;
+ append_wchar(r, c);
*r += HI_VAR;
break;
} else {
@@ -288,7 +288,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case VAR2:
if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {
- *r += c;
+ append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -299,15 +299,15 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case COMMENT:
if (c == '\n') {
*r += HI_RESET;
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case REGEX:
- *r += c;
+ append_wchar(r, c);
if (c == closer_) {
*r += HI_RESET;
t_ = NORMAL;
@@ -317,12 +317,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case REGEX_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = REGEX;
break;
case S_REGEX:
- *r += c;
+ append_wchar(r, c);
if (c == opener_) {
t_ = S_REGEX_S;
} else if (c == '\\') {
@@ -331,12 +331,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case S_REGEX_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = S_REGEX;
break;
case S_REGEX_S:
- *r += c;
+ append_wchar(r, c);
if (c == closer_) {
*r += HI_RESET;
t_ = NORMAL;
@@ -346,12 +346,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case S_REGEX_S_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = S_REGEX_S;
break;
case QUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = NORMAL;
@@ -361,12 +361,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case QUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = QUOTE;
break;
case DQUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '"') {
*r += HI_RESET;
t_ = NORMAL;
@@ -376,12 +376,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case DQUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = DQUOTE;
break;
case TICK:
- *r += c;
+ append_wchar(r, c);
if (c == '`') {
*r += HI_RESET;
t_ = NORMAL;
@@ -391,7 +391,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case TICK_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = TICK;
break;
@@ -399,7 +399,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
if (isalpha(c)) {
*r += HI_COMMENT;
*r += '=';
- *r += c;
+ append_wchar(r, c);
heredoc_ = "=cut";
t_ = HEREDOC;
i_ = 0;
@@ -412,7 +412,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case LT:
if (c == '<') {
- *r += c;
+ append_wchar(r, c);
t_ = LT_LT;
heredoc_.clear();
pending_heredoc_ = false;
@@ -426,16 +426,16 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case LT_LT:
if (c == '-') {
indented_heredoc_ = true;
- *r += c;
+ append_wchar(r, c);
} else if (c == '"' || c == '\'') {
closer_ = c;
t_ = LT_LT_QNAME;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
- heredoc_ += c;
- *r += c;
+ append_wchar(&heredoc_, c);
+ append_wchar(r, c);
} else if (!isblank(c)) {
t_ = NORMAL;
goto Normal;
@@ -445,10 +445,10 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
case LT_LT_NAME:
if (isalnum(c) || c == '_') {
t_ = LT_LT_NAME;
- heredoc_ += c;
- *r += c;
+ append_wchar(&heredoc_, c);
+ append_wchar(r, c);
} else if (c == '\n') {
- *r += c;
+ append_wchar(r, c);
*r += HI_STRING;
t_ = HEREDOC_BOL;
} else {
@@ -459,19 +459,19 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case LT_LT_QNAME:
- *r += c;
+ append_wchar(r, c);
if (c == closer_) {
*r += HI_RESET;
t_ = HEREDOC_BOL;
pending_heredoc_ = true;
t_ = NORMAL;
} else {
- heredoc_ += c;
+ append_wchar(&heredoc_, c);
}
break;
case HEREDOC_BOL:
- *r += c;
+ append_wchar(r, c);
if (c == '\n') {
if (i_ == heredoc_.size()) {
t_ = NORMAL;
@@ -489,7 +489,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case HEREDOC:
- *r += c;
+ append_wchar(r, c);
if (c == '\n') {
t_ = HEREDOC_BOL;
i_ = 0;
diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cpp
index 3b83139d..61582909 100644
--- a/llamafile/highlight_python.cpp
+++ b/llamafile/highlight_python.cpp
@@ -190,7 +190,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
// handle """string"""
case DQUOTE2:
if (c == '"') {
- *r += c;
+ *r += '"';
t_ = DQUOTE3;
} else {
*r += HI_RESET;
diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cpp
index 3c60d10d..e9f9191c 100644
--- a/llamafile/highlight_shell.cpp
+++ b/llamafile/highlight_shell.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-
+#include "string.h"
#include <ctype.h>
enum {
@@ -46,44 +46,43 @@ HighlightShell::~HighlightShell() {
}
void HighlightShell::feed(std::string *r, std::string_view input) {
- int c;
- for (size_t i = 0; i < input.size(); ++i) {
- c = input[i] & 255;
+ for (size_t i = 0; i < input.size();) {
+ wchar_t c = read_wchar(input, &i);
switch (t_) {
Normal:
case NORMAL:
if (!isascii(c) || isalpha(c) || c == '_') {
t_ = WORD;
- word_ += c;
+ append_wchar(&word_, c);
} else if (c == '\'') {
t_ = QUOTE;
*r += HI_STRING;
- *r += c;
+ *r += '\'';
} else if (c == '\\') {
t_ = BACKSLASH;
*r += HI_ESCAPE;
- *r += c;
+ *r += '\\';
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
- *r += c;
+ *r += '"';
} else if (c == '`') {
t_ = TICK;
*r += HI_STRING;
- *r += c;
+ *r += '`';
} else if (c == '$') {
- *r += c;
t_ = VAR;
+ *r += '$';
} else if (c == '<') {
- *r += c;
t_ = LT;
+ *r += '<';
} else if (c == '#') {
*r += HI_COMMENT;
- *r += c;
+ *r += '#';
t_ = COMMENT;
} else if (c == '\n') {
- *r += c;
+ *r += '\n';
if (pending_heredoc_) {
*r += HI_STRING;
pending_heredoc_ = false;
@@ -91,19 +90,19 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
i_ = 0;
}
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case BACKSLASH:
- *r += c;
+ append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
break;
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
- word_ += c;
+ append_wchar(&word_, c);
} else {
if (is_keyword_shell(word_.data(), word_.size())) {
*r += HI_KEYWORD;
@@ -133,12 +132,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
c == '\\' || //
c == '^') {
*r += HI_VAR;
- *r += c;
+ append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
break;
} else if (c == '{') {
- *r += c;
+ append_wchar(r, c);
*r += HI_VAR;
t_ = VAR2;
break;
@@ -150,7 +149,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
case VAR2:
if (!isascii(c) || isalnum(c) || c == '_') {
- *r += c;
+ append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -161,15 +160,15 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
case COMMENT:
if (c == '\n') {
*r += HI_RESET;
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case QUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = NORMAL;
@@ -177,7 +176,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case DQUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '"') {
*r += HI_RESET;
t_ = NORMAL;
@@ -187,12 +186,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case DQUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = DQUOTE;
break;
case TICK:
- *r += c;
+ append_wchar(r, c);
if (c == '`') {
*r += HI_RESET;
t_ = NORMAL;
@@ -202,13 +201,13 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case TICK_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = TICK;
break;
case LT:
if (c == '<') {
- *r += c;
+ append_wchar(r, c);
t_ = LT_LT;
heredoc_.clear();
pending_heredoc_ = false;
@@ -222,17 +221,17 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
case LT_LT:
if (c == '-') {
indented_heredoc_ = true;
- *r += c;
+ append_wchar(r, c);
} else if (c == '\\') {
- *r += c;
+ append_wchar(r, c);
} else if (c == '\'') {
t_ = LT_LT_QNAME;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
- heredoc_ += c;
- *r += c;
+ append_wchar(&heredoc_, c);
+ append_wchar(r, c);
} else if (!isblank(c)) {
t_ = NORMAL;
goto Normal;
@@ -242,10 +241,10 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
case LT_LT_NAME:
if (isalnum(c) || c == '_') {
t_ = LT_LT_NAME;
- heredoc_ += c;
- *r += c;
+ append_wchar(&heredoc_, c);
+ append_wchar(r, c);
} else if (c == '\n') {
- *r += c;
+ append_wchar(r, c);
*r += HI_STRING;
t_ = HEREDOC_BOL;
} else {
@@ -256,19 +255,19 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case LT_LT_QNAME:
- *r += c;
+ append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = HEREDOC_BOL;
pending_heredoc_ = true;
t_ = NORMAL;
} else {
- heredoc_ += c;
+ append_wchar(&heredoc_, c);
}
break;
case HEREDOC_BOL:
- *r += c;
+ append_wchar(r, c);
if (c == '\n') {
if (i_ == heredoc_.size()) {
t_ = NORMAL;
@@ -286,7 +285,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case HEREDOC:
- *r += c;
+ append_wchar(r, c);
if (c == '\n')
t_ = HEREDOC_BOL;
break;
diff --git a/llamafile/highlight_tcl.cpp b/llamafile/highlight_tcl.cpp
index 849fec39..495eb256 100644
--- a/llamafile/highlight_tcl.cpp
+++ b/llamafile/highlight_tcl.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-
+#include "string.h"
#include <ctype.h>
enum {
@@ -26,7 +26,9 @@ enum {
DQUOTE_BACKSLASH,
VAR,
VAR2,
+ VAR_CURLY,
COMMENT,
+ BACKSLASH,
};
HighlightTcl::HighlightTcl() {
@@ -36,35 +38,38 @@ HighlightTcl::~HighlightTcl() {
}
void HighlightTcl::feed(std::string *r, std::string_view input) {
- int c;
- for (size_t i = 0; i < input.size(); ++i) {
- c = input[i] & 255;
+ for (size_t i = 0; i < input.size();) {
+ wchar_t c = read_wchar(input, &i);
switch (t_) {
Normal:
case NORMAL:
if (!isascii(c) || isalpha(c) || c == '_') {
t_ = WORD;
- word_ += c;
+ append_wchar(&word_, c);
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
- *r += c;
+ append_wchar(r, c);
} else if (c == '$') {
- *r += c;
+ append_wchar(r, c);
t_ = VAR;
} else if (c == '#') {
*r += HI_COMMENT;
- *r += c;
+ append_wchar(r, c);
t_ = COMMENT;
+ } else if (c == '\\') {
+ t_ = BACKSLASH;
+ *r += HI_ESCAPE;
+ append_wchar(r, c);
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case WORD:
- if (!isascii(c) || isalnum(c) || c == '_') {
- word_ += c;
+ if (!(isspace(c) || c == ';')) {
+ append_wchar(&word_, c);
} else {
if (is_keyword_tcl(word_.data(), word_.size())) {
*r += HI_KEYWORD;
@@ -87,11 +92,17 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
}
break;
+ case BACKSLASH:
+ append_wchar(r, c);
+ *r += HI_RESET;
+ t_ = NORMAL;
+ break;
+
case VAR:
if (c == '{') {
- *r += c;
+ append_wchar(r, c);
*r += HI_VAR;
- t_ = VAR2;
+ t_ = VAR_CURLY;
break;
} else {
*r += HI_VAR;
@@ -101,7 +112,7 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
case VAR2:
if (!isascii(c) || isalnum(c) || c == '_') {
- *r += c;
+ append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -109,18 +120,28 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
}
break;
+ case VAR_CURLY:
+ if (c == '}') {
+ *r += HI_RESET;
+ *r += '}';
+ t_ = NORMAL;
+ } else {
+ append_wchar(r, c);
+ }
+ break;
+
case COMMENT:
if (c == '\n') {
*r += HI_RESET;
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case DQUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '"') {
*r += HI_RESET;
t_ = NORMAL;
@@ -130,7 +151,7 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
break;
case DQUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = DQUOTE;
break;
@@ -161,9 +182,11 @@ void HighlightTcl::flush(std::string *r) {
word_.clear();
break;
case VAR2:
+ case VAR_CURLY:
case DQUOTE:
case DQUOTE_BACKSLASH:
case COMMENT:
+ case BACKSLASH:
*r += HI_RESET;
break;
default:
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index 4192b536..4d147607 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -46,6 +46,7 @@ const char *const kLanguages[] = {
"kotlin", //
"lisp", //
"lua", //
+ "m4", //
"markdown", //
"pascal", //
"perl", //
diff --git a/llamafile/highlight_zig.cpp b/llamafile/highlight_zig.cpp
index cfe598fc..fed96513 100644
--- a/llamafile/highlight_zig.cpp
+++ b/llamafile/highlight_zig.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-
+#include "string.h"
#include <ctype.h>
enum {
@@ -39,16 +39,15 @@ HighlightZig::~HighlightZig() {
}
void HighlightZig::feed(std::string *r, std::string_view input) {
- int c;
- for (size_t i = 0; i < input.size(); ++i) {
- c = input[i] & 255;
+ for (size_t i = 0; i < input.size();) {
+ wchar_t c = read_wchar(input, &i);
switch (t_) {
Normal:
case NORMAL:
if (!isascii(c) || isalpha(c) || c == '_' || c == '@') {
t_ = WORD;
- word_ += c;
+ append_wchar(&word_, c);
} else if (c == '/') {
t_ = SLASH;
} else if (c == '\\') {
@@ -56,19 +55,19 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
} else if (c == '\'') {
t_ = QUOTE;
*r += HI_STRING;
- *r += c;
+ *r += '\'';
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
- *r += c;
+ *r += '"';
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case WORD:
if (!isascii(c) || isalnum(c) || c == '_') {
- word_ += c;
+ append_wchar(&word_, c);
} else {
if (is_keyword_zig(word_.data(), word_.size())) {
*r += HI_KEYWORD;
@@ -110,10 +109,10 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
case BACKSLASH_BACKSLASH:
if (c == '\n') {
*r += HI_RESET;
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
@@ -132,15 +131,15 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
case SLASH_SLASH:
if (c == '\n') {
*r += HI_RESET;
- *r += c;
+ append_wchar(r, c);
t_ = NORMAL;
} else {
- *r += c;
+ append_wchar(r, c);
}
break;
case QUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = NORMAL;
@@ -150,12 +149,12 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
break;
case QUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = QUOTE;
break;
case DQUOTE:
- *r += c;
+ append_wchar(r, c);
if (c == '"') {
*r += HI_RESET;
t_ = NORMAL;
@@ -165,7 +164,7 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
break;
case DQUOTE_BACKSLASH:
- *r += c;
+ append_wchar(r, c);
t_ = DQUOTE;
break;
diff --git a/llamafile/is_keyword_java_constant.c b/llamafile/is_keyword_java_constant.c
new file mode 100644
index 00000000..fa2cfccf
--- /dev/null
+++ b/llamafile/is_keyword_java_constant.c
@@ -0,0 +1,126 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf --output-file=llamafile/is_keyword_java_constant.c llamafile/is_keyword_java_constant.gperf */
+/* Computed positions: -k'1' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_java_constant.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 3
+#define MIN_WORD_LENGTH 4
+#define MAX_WORD_LENGTH 5
+#define MIN_HASH_VALUE 4
+#define MAX_HASH_VALUE 6
+/* maximum key range = 3, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 1, 7, 7, 7, 7, 7, 7, 7,
+ 1, 7, 7, 7, 7, 7, 0, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7
+ };
+ return len + asso_values[(unsigned char)str[0]];
+}
+
+const char *
+is_keyword_java_constant (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str4[sizeof("true")];
+ char stringpool_str5[sizeof("null")];
+ char stringpool_str6[sizeof("false")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "true",
+ "null",
+ "false"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_java_constant.gperf b/llamafile/is_keyword_java_constant.gperf
new file mode 100644
index 00000000..8c7f898a
--- /dev/null
+++ b/llamafile/is_keyword_java_constant.gperf
@@ -0,0 +1,12 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_java_constant
+%%
+true
+false
+null
diff --git a/llamafile/is_keyword_kotlin.c b/llamafile/is_keyword_kotlin.c
index 39030455..3cff1963 100644
--- a/llamafile/is_keyword_kotlin.c
+++ b/llamafile/is_keyword_kotlin.c
@@ -33,12 +33,12 @@
#include <string.h>
-#define TOTAL_KEYWORDS 75
+#define TOTAL_KEYWORDS 72
#define MIN_WORD_LENGTH 2
#define MAX_WORD_LENGTH 11
#define MIN_HASH_VALUE 3
-#define MAX_HASH_VALUE 154
-/* maximum key range = 152, duplicates = 0 */
+#define MAX_HASH_VALUE 125
+/* maximum key range = 123, duplicates = 0 */
#ifdef __GNUC__
__inline
@@ -52,32 +52,32 @@ hash (register const char *str, register size_t len)
{
static const unsigned char asso_values[] =
{
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 60, 15, 10,
- 45, 5, 30, 0, 0, 15, 15, 35, 0, 55,
- 10, 60, 40, 155, 25, 40, 0, 0, 85, 5,
- 0, 35, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155, 155, 155, 155, 155,
- 155, 155, 155, 155, 155, 155
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 50, 20, 5,
+ 35, 5, 65, 40, 30, 15, 20, 5, 50, 55,
+ 5, 35, 20, 126, 0, 25, 0, 0, 0, 5,
+ 5, 45, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126,
+ 126, 126, 126, 126, 126, 126
};
register unsigned int hval = len;
@@ -99,268 +99,259 @@ is_keyword_kotlin (register const char *str, register size_t len)
{
struct stringpool_t
{
- char stringpool_str3[sizeof("get")];
- char stringpool_str8[sizeof("lateinit")];
- char stringpool_str9[sizeof("true")];
- char stringpool_str13[sizeof("external")];
- char stringpool_str14[sizeof("null")];
- char stringpool_str15[sizeof("catch")];
+ char stringpool_str3[sizeof("var")];
+ char stringpool_str10[sizeof("throw")];
+ char stringpool_str11[sizeof("return")];
+ char stringpool_str13[sizeof("receiver")];
+ char stringpool_str15[sizeof("const")];
char stringpool_str17[sizeof("it")];
+ char stringpool_str19[sizeof("when")];
char stringpool_str20[sizeof("where")];
- char stringpool_str23[sizeof("internal")];
- char stringpool_str24[sizeof("when")];
- char stringpool_str25[sizeof("const")];
- char stringpool_str26[sizeof("inline")];
- char stringpool_str27[sizeof("in")];
+ char stringpool_str21[sizeof("constructor")];
+ char stringpool_str22[sizeof("in")];
+ char stringpool_str23[sizeof("continue")];
+ char stringpool_str25[sizeof("inner")];
+ char stringpool_str27[sizeof("tailrec")];
+ char stringpool_str28[sizeof("set")];
char stringpool_str29[sizeof("interface")];
char stringpool_str30[sizeof("while")];
- char stringpool_str32[sizeof("tailrec")];
- char stringpool_str33[sizeof("continue")];
+ char stringpool_str31[sizeof("expect")];
+ char stringpool_str33[sizeof("noinline")];
char stringpool_str34[sizeof("init")];
- char stringpool_str35[sizeof("throw")];
- char stringpool_str38[sizeof("noinline")];
- char stringpool_str39[sizeof("file")];
- char stringpool_str40[sizeof("false")];
- char stringpool_str41[sizeof("return")];
- char stringpool_str43[sizeof("set")];
- char stringpool_str45[sizeof("final")];
- char stringpool_str47[sizeof("if")];
- char stringpool_str50[sizeof("infix")];
- char stringpool_str51[sizeof("expect")];
- char stringpool_str52[sizeof("by")];
- char stringpool_str53[sizeof("fun")];
- char stringpool_str54[sizeof("else")];
- char stringpool_str55[sizeof("inner")];
- char stringpool_str56[sizeof("constructor")];
- char stringpool_str57[sizeof("is")];
- char stringpool_str58[sizeof("delegate")];
- char stringpool_str59[sizeof("this")];
- char stringpool_str60[sizeof("break")];
- char stringpool_str61[sizeof("import")];
- char stringpool_str62[sizeof("package")];
- char stringpool_str63[sizeof("out")];
+ char stringpool_str35[sizeof("break")];
+ char stringpool_str37[sizeof("package")];
+ char stringpool_str38[sizeof("out")];
+ char stringpool_str39[sizeof("else")];
+ char stringpool_str40[sizeof("catch")];
+ char stringpool_str41[sizeof("import")];
+ char stringpool_str42[sizeof("is")];
+ char stringpool_str43[sizeof("get")];
+ char stringpool_str44[sizeof("this")];
+ char stringpool_str46[sizeof("vararg")];
+ char stringpool_str47[sizeof("private")];
+ char stringpool_str48[sizeof("operator")];
+ char stringpool_str49[sizeof("open")];
+ char stringpool_str50[sizeof("super")];
+ char stringpool_str51[sizeof("public")];
+ char stringpool_str52[sizeof("dynamic")];
+ char stringpool_str53[sizeof("override")];
+ char stringpool_str54[sizeof("typealias")];
+ char stringpool_str56[sizeof("crossinline")];
+ char stringpool_str57[sizeof("reified")];
+ char stringpool_str58[sizeof("lateinit")];
+ char stringpool_str61[sizeof("object")];
+ char stringpool_str63[sizeof("external")];
char stringpool_str64[sizeof("enum")];
- char stringpool_str66[sizeof("actual")];
- char stringpool_str67[sizeof("private")];
- char stringpool_str68[sizeof("receiver")];
- char stringpool_str71[sizeof("public")];
- char stringpool_str72[sizeof("dynamic")];
- char stringpool_str73[sizeof("try")];
- char stringpool_str76[sizeof("typeof")];
- char stringpool_str78[sizeof("override")];
- char stringpool_str79[sizeof("open")];
- char stringpool_str81[sizeof("object")];
- char stringpool_str82[sizeof("finally")];
- char stringpool_str83[sizeof("for")];
- char stringpool_str84[sizeof("companion")];
- char stringpool_str85[sizeof("field")];
- char stringpool_str86[sizeof("crossinline")];
- char stringpool_str88[sizeof("val")];
- char stringpool_str89[sizeof("typealias")];
- char stringpool_str90[sizeof("annotation")];
- char stringpool_str92[sizeof("reified")];
- char stringpool_str98[sizeof("operator")];
- char stringpool_str102[sizeof("as")];
- char stringpool_str103[sizeof("setparam")];
- char stringpool_str107[sizeof("do")];
- char stringpool_str108[sizeof("abstract")];
- char stringpool_str109[sizeof("data")];
- char stringpool_str110[sizeof("super")];
- char stringpool_str115[sizeof("class")];
- char stringpool_str116[sizeof("vararg")];
- char stringpool_str125[sizeof("param")];
- char stringpool_str132[sizeof("suspend")];
- char stringpool_str138[sizeof("var")];
- char stringpool_str143[sizeof("property")];
- char stringpool_str151[sizeof("sealed")];
- char stringpool_str154[sizeof("protected")];
+ char stringpool_str67[sizeof("by")];
+ char stringpool_str68[sizeof("for")];
+ char stringpool_str70[sizeof("annotation")];
+ char stringpool_str72[sizeof("do")];
+ char stringpool_str73[sizeof("internal")];
+ char stringpool_str74[sizeof("companion")];
+ char stringpool_str76[sizeof("inline")];
+ char stringpool_str77[sizeof("as")];
+ char stringpool_str78[sizeof("fun")];
+ char stringpool_str80[sizeof("param")];
+ char stringpool_str82[sizeof("if")];
+ char stringpool_str83[sizeof("abstract")];
+ char stringpool_str85[sizeof("class")];
+ char stringpool_str88[sizeof("setparam")];
+ char stringpool_str89[sizeof("data")];
+ char stringpool_str90[sizeof("infix")];
+ char stringpool_str91[sizeof("typeof")];
+ char stringpool_str92[sizeof("suspend")];
+ char stringpool_str93[sizeof("try")];
+ char stringpool_str98[sizeof("delegate")];
+ char stringpool_str99[sizeof("protected")];
+ char stringpool_str103[sizeof("val")];
+ char stringpool_str106[sizeof("actual")];
+ char stringpool_str108[sizeof("property")];
+ char stringpool_str110[sizeof("field")];
+ char stringpool_str116[sizeof("sealed")];
+ char stringpool_str122[sizeof("finally")];
+ char stringpool_str124[sizeof("file")];
+ char stringpool_str125[sizeof("final")];
};
static const struct stringpool_t stringpool_contents =
{
- "get",
- "lateinit",
- "true",
- "external",
- "null",
- "catch",
+ "var",
+ "throw",
+ "return",
+ "receiver",
+ "const",
"it",
- "where",
- "internal",
"when",
- "const",
- "inline",
+ "where",
+ "constructor",
"in",
- "interface",
- "while",
- "tailrec",
"continue",
- "init",
- "throw",
- "noinline",
- "file",
- "false",
- "return",
+ "inner",
+ "tailrec",
"set",
- "final",
- "if",
- "infix",
+ "interface",
+ "while",
"expect",
- "by",
- "fun",
- "else",
- "inner",
- "constructor",
- "is",
- "delegate",
- "this",
+ "noinline",
+ "init",
"break",
- "import",
"package",
"out",
- "enum",
- "actual",
+ "else",
+ "catch",
+ "import",
+ "is",
+ "get",
+ "this",
+ "vararg",
"private",
- "receiver",
+ "operator",
+ "open",
+ "super",
"public",
"dynamic",
- "try",
- "typeof",
"override",
- "open",
+ "typealias",
+ "crossinline",
+ "reified",
+ "lateinit",
"object",
- "finally",
+ "external",
+ "enum",
+ "by",
"for",
- "companion",
- "field",
- "crossinline",
- "val",
- "typealias",
"annotation",
- "reified",
- "operator",
- "as",
- "setparam",
"do",
+ "internal",
+ "companion",
+ "inline",
+ "as",
+ "fun",
+ "param",
+ "if",
"abstract",
- "data",
- "super",
"class",
- "vararg",
- "param",
+ "setparam",
+ "data",
+ "infix",
+ "typeof",
"suspend",
- "var",
+ "try",
+ "delegate",
+ "protected",
+ "val",
+ "actual",
"property",
+ "field",
"sealed",
- "protected"
+ "finally",
+ "file",
+ "final"
};
#define stringpool ((const char *) &stringpool_contents)
static const int wordlist[] =
{
-1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
- -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
- -1, -1, -1,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
-1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
- -1, -1,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
- -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
-1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,
-1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str32,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
- -1, -1,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str43,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
-1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,
- -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str56,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,
- -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,
- -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str72,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,
- -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,
-1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str77,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str78,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,
-1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str81,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str80,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str83,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str84,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str86,
-1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,
+ -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str88,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str89,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str90,
- -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str91,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str92,
- -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str93,
+ -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str98,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,
-1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str103,
- -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str107,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str106,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str108,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str109,
+ -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str110,
- -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,
+ -1, -1, -1, -1, -1,
(int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,
- -1, -1, -1, -1, -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,
- -1, -1, -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str132,
-1, -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str138,
- -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str143,
- -1, -1, -1, -1, -1, -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str151,
- -1, -1,
- (int)(size_t)&((struct stringpool_t *)0)->stringpool_str154
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str124,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
diff --git a/llamafile/is_keyword_kotlin.gperf b/llamafile/is_keyword_kotlin.gperf
index bf4e3e64..643d5d14 100644
--- a/llamafile/is_keyword_kotlin.gperf
+++ b/llamafile/is_keyword_kotlin.gperf
@@ -28,7 +28,6 @@ else
enum
expect
external
-false
field
file
final
@@ -49,7 +48,6 @@ is
it
lateinit
noinline
-null
object
open
operator
@@ -72,7 +70,6 @@ suspend
tailrec
this
throw
-true
try
typealias
typeof
diff --git a/llamafile/is_keyword_m4.c b/llamafile/is_keyword_m4.c
new file mode 100644
index 00000000..e26357da
--- /dev/null
+++ b/llamafile/is_keyword_m4.c
@@ -0,0 +1,739 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf --output-file=llamafile/is_keyword_m4.c llamafile/is_keyword_m4.gperf */
+/* Computed positions: -k'1,4-7,9' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_m4.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 166
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 25
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 439
+/* maximum key range = 437, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned short asso_values[] =
+ {
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 30, 440, 5, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 40, 145, 70, 120, 40,
+ 10, 0, 10, 70, 65, 5, 35, 100, 35, 0,
+ 0, 0, 25, 155, 40, 55, 5, 15, 150, 10,
+ 145, 130, 10, 0, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440, 440, 440, 440,
+ 440, 440, 440, 440, 440, 440, 440
+ };
+ register unsigned int hval = len;
+
+ switch (hval)
+ {
+ default:
+ hval += asso_values[(unsigned char)str[8]+1];
+ /*FALLTHROUGH*/
+ case 8:
+ case 7:
+ hval += asso_values[(unsigned char)str[6]];
+ /*FALLTHROUGH*/
+ case 6:
+ hval += asso_values[(unsigned char)str[5]];
+ /*FALLTHROUGH*/
+ case 5:
+ hval += asso_values[(unsigned char)str[4]];
+ /*FALLTHROUGH*/
+ case 4:
+ hval += asso_values[(unsigned char)str[3]+1];
+ /*FALLTHROUGH*/
+ case 3:
+ case 2:
+ case 1:
+ hval += asso_values[(unsigned char)str[0]];
+ break;
+ }
+ return hval;
+}
+
+const char *
+is_keyword_m4 (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("os2")];
+ char stringpool_str4[sizeof("eval")];
+ char stringpool_str6[sizeof("m4_len")];
+ char stringpool_str9[sizeof("exch")];
+ char stringpool_str12[sizeof("m4_line")];
+ char stringpool_str14[sizeof("defn")];
+ char stringpool_str17[sizeof("m4_defn")];
+ char stringpool_str22[sizeof("traceon")];
+ char stringpool_str23[sizeof("maketemp")];
+ char stringpool_str24[sizeof("file")];
+ char stringpool_str27[sizeof("m4_cond")];
+ char stringpool_str30[sizeof("ifdef")];
+ char stringpool_str31[sizeof("divnum")];
+ char stringpool_str33[sizeof("traceoff")];
+ char stringpool_str34[sizeof("m4_define")];
+ char stringpool_str37[sizeof("include")];
+ char stringpool_str38[sizeof("len")];
+ char stringpool_str39[sizeof("join")];
+ char stringpool_str40[sizeof("m4_define_blind")];
+ char stringpool_str41[sizeof("popdef")];
+ char stringpool_str42[sizeof("forloop")];
+ char stringpool_str44[sizeof("cond")];
+ char stringpool_str46[sizeof("dquote")];
+ char stringpool_str47[sizeof("mkstemp")];
+ char stringpool_str48[sizeof("undefine")];
+ char stringpool_str49[sizeof("line")];
+ char stringpool_str51[sizeof("define")];
+ char stringpool_str53[sizeof("m4_index")];
+ char stringpool_str54[sizeof("copy")];
+ char stringpool_str57[sizeof("pushdef")];
+ char stringpool_str58[sizeof("m4_indir")];
+ char stringpool_str60[sizeof("m4_dumpdef")];
+ char stringpool_str62[sizeof("esyscmd")];
+ char stringpool_str63[sizeof("m4_ifdef")];
+ char stringpool_str64[sizeof("incr")];
+ char stringpool_str66[sizeof("ifelse")];
+ char stringpool_str67[sizeof("example")];
+ char stringpool_str68[sizeof("m4_quote")];
+ char stringpool_str69[sizeof("decr")];
+ char stringpool_str71[sizeof("divert")];
+ char stringpool_str73[sizeof("gnu")];
+ char stringpool_str74[sizeof("argn")];
+ char stringpool_str81[sizeof("syscmd")];
+ char stringpool_str82[sizeof("windows")];
+ char stringpool_str85[sizeof("indir")];
+ char stringpool_str86[sizeof("m4_gnu")];
+ char stringpool_str87[sizeof("m4_decr")];
+ char stringpool_str90[sizeof("dquote_elt")];
+ char stringpool_str91[sizeof("format")];
+ char stringpool_str93[sizeof("m4_shift")];
+ char stringpool_str97[sizeof("define_blind")];
+ char stringpool_str99[sizeof("m4_ifelse")];
+ char stringpool_str101[sizeof("__windows__")];
+ char stringpool_str108[sizeof("translit")];
+ char stringpool_str111[sizeof("substr")];
+ char stringpool_str112[sizeof("m4_join")];
+ char stringpool_str113[sizeof("m4_curry")];
+ char stringpool_str115[sizeof("m4_joinall")];
+ char stringpool_str116[sizeof("m4_os2")];
+ char stringpool_str117[sizeof("m4_file")];
+ char stringpool_str120[sizeof("m4_include")];
+ char stringpool_str122[sizeof("m4_incr")];
+ char stringpool_str123[sizeof("__line__")];
+ char stringpool_str125[sizeof("nargs")];
+ char stringpool_str129[sizeof("m4_cleardivert")];
+ char stringpool_str130[sizeof("m4_foreach")];
+ char stringpool_str131[sizeof("m4_foreachq")];
+ char stringpool_str133[sizeof("sinclude")];
+ char stringpool_str134[sizeof("m4_format")];
+ char stringpool_str135[sizeof("shift")];
+ char stringpool_str136[sizeof("m4_translit")];
+ char stringpool_str137[sizeof("builtin")];
+ char stringpool_str140[sizeof("m4_builtin")];
+ char stringpool_str141[sizeof("m4_downcase")];
+ char stringpool_str142[sizeof("__gnu__")];
+ char stringpool_str144[sizeof("m4_rename")];
+ char stringpool_str146[sizeof("m4exit")];
+ char stringpool_str147[sizeof("m4_debugmode")];
+ char stringpool_str149[sizeof("unix")];
+ char stringpool_str152[sizeof("reverse")];
+ char stringpool_str156[sizeof("m4wrap")];
+ char stringpool_str158[sizeof("__file__")];
+ char stringpool_str159[sizeof("changecom")];
+ char stringpool_str161[sizeof("m4_errprint")];
+ char stringpool_str162[sizeof("__os2__")];
+ char stringpool_str164[sizeof("m4_divnum")];
+ char stringpool_str165[sizeof("index")];
+ char stringpool_str166[sizeof("rename")];
+ char stringpool_str167[sizeof("m4_changecom")];
+ char stringpool_str168[sizeof("m4_changeword")];
+ char stringpool_str169[sizeof("m4_changequote")];
+ char stringpool_str170[sizeof("m4_mkstemp")];
+ char stringpool_str172[sizeof("m4_copy")];
+ char stringpool_str174[sizeof("m4_m4exit")];
+ char stringpool_str175[sizeof("quote")];
+ char stringpool_str176[sizeof("sysval")];
+ char stringpool_str179[sizeof("m4_divert")];
+ char stringpool_str180[sizeof("m4_forloop")];
+ char stringpool_str182[sizeof("joinall")];
+ char stringpool_str183[sizeof("downcase")];
+ char stringpool_str185[sizeof("changeword")];
+ char stringpool_str186[sizeof("m4_undivert")];
+ char stringpool_str188[sizeof("m4_nargs")];
+ char stringpool_str189[sizeof("m4_dquote")];
+ char stringpool_str191[sizeof("m4_maketemp")];
+ char stringpool_str192[sizeof("dumpdef")];
+ char stringpool_str193[sizeof("m4_dquote_elt")];
+ char stringpool_str195[sizeof("capitalize")];
+ char stringpool_str196[sizeof("upcase")];
+ char stringpool_str200[sizeof("m4_traceon")];
+ char stringpool_str201[sizeof("m4_traceoff")];
+ char stringpool_str202[sizeof("foreach")];
+ char stringpool_str203[sizeof("foreachq")];
+ char stringpool_str206[sizeof("m4_undefine")];
+ char stringpool_str208[sizeof("errprint")];
+ char stringpool_str211[sizeof("m4_sinclude")];
+ char stringpool_str214[sizeof("m4___windows__")];
+ char stringpool_str216[sizeof("__program__")];
+ char stringpool_str217[sizeof("m4_debugfile")];
+ char stringpool_str219[sizeof("m4_m4wrap")];
+ char stringpool_str220[sizeof("m4_reverse")];
+ char stringpool_str221[sizeof("m4___file__")];
+ char stringpool_str226[sizeof("regexp")];
+ char stringpool_str228[sizeof("patsubst")];
+ char stringpool_str230[sizeof("curry")];
+ char stringpool_str235[sizeof("m4_example")];
+ char stringpool_str236[sizeof("cleardivert")];
+ char stringpool_str237[sizeof("m4_argn")];
+ char stringpool_str238[sizeof("__unix__")];
+ char stringpool_str239[sizeof("m4_syscmd")];
+ char stringpool_str243[sizeof("m4_capitalize")];
+ char stringpool_str246[sizeof("m4___line__")];
+ char stringpool_str248[sizeof("undivert")];
+ char stringpool_str249[sizeof("debugmode")];
+ char stringpool_str259[sizeof("m4_substr")];
+ char stringpool_str260[sizeof("m4_esyscmd")];
+ char stringpool_str264[sizeof("debugfile")];
+ char stringpool_str267[sizeof("m4_exch")];
+ char stringpool_str269[sizeof("m4_popdef")];
+ char stringpool_str271[sizeof("fatal_error")];
+ char stringpool_str272[sizeof("m4_eval")];
+ char stringpool_str278[sizeof("m4_array")];
+ char stringpool_str281[sizeof("m4_stack_foreach")];
+ char stringpool_str283[sizeof("stack_foreach")];
+ char stringpool_str285[sizeof("m4_stack_foreach_sep")];
+ char stringpool_str286[sizeof("m4_stack_foreach_lifo")];
+ char stringpool_str287[sizeof("stack_foreach_sep")];
+ char stringpool_str288[sizeof("stack_foreach_lifo")];
+ char stringpool_str289[sizeof("m4_regexp")];
+ char stringpool_str290[sizeof("m4_stack_foreach_sep_lifo")];
+ char stringpool_str292[sizeof("stack_foreach_sep_lifo")];
+ char stringpool_str301[sizeof("changequote")];
+ char stringpool_str304[sizeof("m4_upcase")];
+ char stringpool_str307[sizeof("m4_unix")];
+ char stringpool_str310[sizeof("m4_pushdef")];
+ char stringpool_str315[sizeof("m4_windows")];
+ char stringpool_str325[sizeof("array")];
+ char stringpool_str329[sizeof("m4___program__")];
+ char stringpool_str336[sizeof("m4_patsubst")];
+ char stringpool_str341[sizeof("m4___unix__")];
+ char stringpool_str349[sizeof("m4_sysval")];
+ char stringpool_str374[sizeof("m4_fatal_error")];
+ char stringpool_str395[sizeof("m4___os2__")];
+ char stringpool_str410[sizeof("m4___gnu__")];
+ char stringpool_str427[sizeof("m4_array_set")];
+ char stringpool_str439[sizeof("array_set")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "os2",
+ "eval",
+ "m4_len",
+ "exch",
+ "m4_line",
+ "defn",
+ "m4_defn",
+ "traceon",
+ "maketemp",
+ "file",
+ "m4_cond",
+ "ifdef",
+ "divnum",
+ "traceoff",
+ "m4_define",
+ "include",
+ "len",
+ "join",
+ "m4_define_blind",
+ "popdef",
+ "forloop",
+ "cond",
+ "dquote",
+ "mkstemp",
+ "undefine",
+ "line",
+ "define",
+ "m4_index",
+ "copy",
+ "pushdef",
+ "m4_indir",
+ "m4_dumpdef",
+ "esyscmd",
+ "m4_ifdef",
+ "incr",
+ "ifelse",
+ "example",
+ "m4_quote",
+ "decr",
+ "divert",
+ "gnu",
+ "argn",
+ "syscmd",
+ "windows",
+ "indir",
+ "m4_gnu",
+ "m4_decr",
+ "dquote_elt",
+ "format",
+ "m4_shift",
+ "define_blind",
+ "m4_ifelse",
+ "__windows__",
+ "translit",
+ "substr",
+ "m4_join",
+ "m4_curry",
+ "m4_joinall",
+ "m4_os2",
+ "m4_file",
+ "m4_include",
+ "m4_incr",
+ "__line__",
+ "nargs",
+ "m4_cleardivert",
+ "m4_foreach",
+ "m4_foreachq",
+ "sinclude",
+ "m4_format",
+ "shift",
+ "m4_translit",
+ "builtin",
+ "m4_builtin",
+ "m4_downcase",
+ "__gnu__",
+ "m4_rename",
+ "m4exit",
+ "m4_debugmode",
+ "unix",
+ "reverse",
+ "m4wrap",
+ "__file__",
+ "changecom",
+ "m4_errprint",
+ "__os2__",
+ "m4_divnum",
+ "index",
+ "rename",
+ "m4_changecom",
+ "m4_changeword",
+ "m4_changequote",
+ "m4_mkstemp",
+ "m4_copy",
+ "m4_m4exit",
+ "quote",
+ "sysval",
+ "m4_divert",
+ "m4_forloop",
+ "joinall",
+ "downcase",
+ "changeword",
+ "m4_undivert",
+ "m4_nargs",
+ "m4_dquote",
+ "m4_maketemp",
+ "dumpdef",
+ "m4_dquote_elt",
+ "capitalize",
+ "upcase",
+ "m4_traceon",
+ "m4_traceoff",
+ "foreach",
+ "foreachq",
+ "m4_undefine",
+ "errprint",
+ "m4_sinclude",
+ "m4___windows__",
+ "__program__",
+ "m4_debugfile",
+ "m4_m4wrap",
+ "m4_reverse",
+ "m4___file__",
+ "regexp",
+ "patsubst",
+ "curry",
+ "m4_example",
+ "cleardivert",
+ "m4_argn",
+ "__unix__",
+ "m4_syscmd",
+ "m4_capitalize",
+ "m4___line__",
+ "undivert",
+ "debugmode",
+ "m4_substr",
+ "m4_esyscmd",
+ "debugfile",
+ "m4_exch",
+ "m4_popdef",
+ "fatal_error",
+ "m4_eval",
+ "m4_array",
+ "m4_stack_foreach",
+ "stack_foreach",
+ "m4_stack_foreach_sep",
+ "m4_stack_foreach_lifo",
+ "stack_foreach_sep",
+ "stack_foreach_lifo",
+ "m4_regexp",
+ "m4_stack_foreach_sep_lifo",
+ "stack_foreach_sep_lifo",
+ "changequote",
+ "m4_upcase",
+ "m4_unix",
+ "m4_pushdef",
+ "m4_windows",
+ "array",
+ "m4___program__",
+ "m4_patsubst",
+ "m4___unix__",
+ "m4_sysval",
+ "m4_fatal_error",
+ "m4___os2__",
+ "m4___gnu__",
+ "m4_array_set",
+ "array_set"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str38,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str46,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str47,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str48,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str53,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str58,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str62,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str63,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str64,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str66,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str81,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str86,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str87,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str90,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str91,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str93,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str97,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str101,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str108,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str111,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str112,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str113,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str115,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str116,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str117,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str120,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str123,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str125,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str129,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str130,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str131,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str133,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str134,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str135,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str137,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str140,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str141,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str142,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str144,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str146,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str147,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str149,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str152,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str156,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str158,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str159,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str161,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str162,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str164,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str165,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str166,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str167,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str168,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str169,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str170,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str172,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str174,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str175,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str176,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str179,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str180,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str182,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str183,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str185,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str186,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str188,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str189,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str191,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str192,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str193,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str195,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str196,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str200,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str201,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str202,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str203,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str206,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str208,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str211,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str214,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str216,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str217,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str219,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str220,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str221,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str226,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str228,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str230,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str235,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str236,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str237,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str238,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str239,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str243,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str246,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str248,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str249,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str259,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str260,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str264,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str267,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str269,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str271,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str272,
+ -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str278,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str281,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str283,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str285,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str286,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str287,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str288,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str289,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str290,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str292,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str301,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str304,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str307,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str310,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str315,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str325,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str329,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str336,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str341,
+ -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str349,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str374,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str395,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str410,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str427,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str439
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_m4.gperf b/llamafile/is_keyword_m4.gperf
new file mode 100644
index 00000000..41c11097
--- /dev/null
+++ b/llamafile/is_keyword_m4.gperf
@@ -0,0 +1,175 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_m4
+%%
+__file__
+__gnu__
+__line__
+__os2__
+__program__
+__unix__
+__windows__
+argn
+array
+array_set
+builtin
+capitalize
+changecom
+changequote
+changeword
+cleardivert
+cond
+copy
+curry
+debugfile
+debugmode
+decr
+define
+define_blind
+defn
+divert
+divnum
+downcase
+dquote
+dquote_elt
+dumpdef
+errprint
+esyscmd
+eval
+example
+exch
+fatal_error
+file
+foreach
+foreachq
+forloop
+format
+gnu
+ifdef
+ifelse
+include
+incr
+index
+indir
+join
+joinall
+len
+line
+m4___file__
+m4___gnu__
+m4___line__
+m4___os2__
+m4___program__
+m4___unix__
+m4___windows__
+m4_argn
+m4_array
+m4_array_set
+m4_builtin
+m4_capitalize
+m4_changecom
+m4_changequote
+m4_changeword
+m4_cleardivert
+m4_cond
+m4_copy
+m4_curry
+m4_debugfile
+m4_debugmode
+m4_decr
+m4_define
+m4_define_blind
+m4_defn
+m4_divert
+m4_divnum
+m4_downcase
+m4_dquote
+m4_dquote_elt
+m4_dumpdef
+m4_errprint
+m4_esyscmd
+m4_eval
+m4_example
+m4_exch
+m4_fatal_error
+m4_file
+m4_foreach
+m4_foreachq
+m4_forloop
+m4_format
+m4_gnu
+m4_ifdef
+m4_ifelse
+m4_include
+m4_incr
+m4_index
+m4_indir
+m4_join
+m4_joinall
+m4_len
+m4_line
+m4_m4exit
+m4_m4wrap
+m4_maketemp
+m4_mkstemp
+m4_nargs
+m4_os2
+m4_patsubst
+m4_popdef
+m4_pushdef
+m4_quote
+m4_regexp
+m4_rename
+m4_reverse
+m4_shift
+m4_sinclude
+m4_stack_foreach
+m4_stack_foreach_lifo
+m4_stack_foreach_sep
+m4_stack_foreach_sep_lifo
+m4_substr
+m4_syscmd
+m4_sysval
+m4_traceoff
+m4_traceon
+m4_translit
+m4_undefine
+m4_undivert
+m4_unix
+m4_upcase
+m4_windows
+m4exit
+m4wrap
+maketemp
+mkstemp
+nargs
+os2
+patsubst
+popdef
+pushdef
+quote
+regexp
+rename
+reverse
+shift
+sinclude
+stack_foreach
+stack_foreach_lifo
+stack_foreach_sep
+stack_foreach_sep_lifo
+substr
+syscmd
+sysval
+traceoff
+traceon
+translit
+undefine
+undivert
+unix
+upcase
+windows
diff --git a/llamafile/string.cpp b/llamafile/string.cpp
new file mode 100644
index 00000000..1eff49c0
--- /dev/null
+++ b/llamafile/string.cpp
@@ -0,0 +1,53 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "string.h"
+
+#include <cosmo.h>
+
+wchar_t read_wchar(const std::string_view &s, size_t *i) {
+ wint_t c = s[(*i)++] & 255;
+ if (c >= 0300) {
+ wint_t a = ThomPikeByte(c);
+ size_t m = ThomPikeLen(c) - 1;
+ if (*i + m <= s.size()) {
+ for (int j = 0;;) {
+ wint_t b = s[*i + j] & 255;
+ if (!ThomPikeCont(b))
+ break;
+ a = ThomPikeMerge(a, b);
+ if (++j == m) {
+ c = a;
+ *i += j;
+ break;
+ }
+ }
+ }
+ }
+ return c;
+}
+
+void append_wchar(std::string *r, wchar_t c) {
+ if (isascii(c)) {
+ *r += c;
+ } else {
+ char s[8];
+ uint64_t w = tpenc(c);
+ WRITE64LE(s, w);
+ *r += s;
+ }
+}
diff --git a/llamafile/string.h b/llamafile/string.h
new file mode 100644
index 00000000..996c0a82
--- /dev/null
+++ b/llamafile/string.h
@@ -0,0 +1,22 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <string>
+
+void append_wchar(std::string *, wchar_t);
+wchar_t read_wchar(const std::string_view &, size_t *);