Skip to content

Distribute and run LLMs with a single file.

Add Ruby syntax highlighting

Justine Tunney0f2c86b74441
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 8013d2c7..233b3b7d 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -106,9 +106,12 @@ Highlight *Highlight::create(const std::string_view &lang) {
} else if (lang == "d") {
return new HighlightC(is_keyword_d);
} else if (lang == "zig") {
- return new HighlightZig();
+ return new HighlightZig;
} else if (lang == "tcl") {
- return new HighlightTcl();
+ return new HighlightTcl;
+ } else if (lang == "ruby" || //
+ lang == "rb") {
+ return new HighlightRuby;
} else {
return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 1c2cc8fe..ca6f88b0 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -31,6 +31,7 @@
#define HI_CONTIN "\033[33m" // yellow
#define HI_LABEL "\033[33m" // yellow
#define HI_TYPE "\033[36m" // cyan
+#define HI_CLASS "\033[1;36m" // bold cyan
#define HI_SELECTOR "\033[33m" // yellow
#define HI_PROPERTY "\033[36m" // cyan
#define HI_TAG "\033[33m" // yellow
@@ -86,6 +87,9 @@ is_keyword_f is_keyword_zig_constant;
is_keyword_f is_keyword_tcl;
is_keyword_f is_keyword_tcl_type;
is_keyword_f is_keyword_tcl_builtin;
+is_keyword_f is_keyword_ruby;
+is_keyword_f is_keyword_ruby_builtin;
+is_keyword_f is_keyword_ruby_constant;
}
class Highlight {
@@ -312,6 +316,7 @@ class HighlightPerl : public Highlight {
int t_ = 0;
int i_ = 0;
bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
std::string word_;
std::string heredoc_;
};
@@ -327,6 +332,7 @@ class HighlightShell : public Highlight {
int t_ = 0;
int i_ = 0;
bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
std::string word_;
std::string heredoc_;
};
@@ -368,3 +374,24 @@ class HighlightCsharp : public Highlight {
int trips2_;
std::string word_;
};
+
+class HighlightRuby : public Highlight {
+ public:
+ HighlightRuby();
+ ~HighlightRuby() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ int i_ = 0;
+ int level_ = 0;
+ unsigned char q_ = 0;
+ unsigned char last_ = 0;
+ unsigned char opener_ = 0;
+ unsigned char closer_ = 0;
+ bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
+ std::string word_;
+ std::string heredoc_;
+};
diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cpp
index 2d43afc4..fe2465c5 100644
--- a/llamafile/highlight_perl.cpp
+++ b/llamafile/highlight_perl.cpp
@@ -34,6 +34,7 @@ enum {
LT,
LT_LT,
LT_LT_NAME,
+ LT_LT_QNAME,
HEREDOC_BOL,
HEREDOC,
};
@@ -193,6 +194,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
t_ = LT_LT;
heredoc_.clear();
pending_heredoc_ = false;
+ indented_heredoc_ = false;
} else {
t_ = NORMAL;
goto Normal;
@@ -200,8 +202,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
break;
case LT_LT:
- if (c == '\'') {
- t_ = LT_LT_NAME;
+ if (c == '-') {
+ indented_heredoc_ = true;
+ *r += c;
+ } else if (c == '\'') {
+ t_ = LT_LT_QNAME;
+ *r += HI_STRING;
*r += c;
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
@@ -229,6 +235,18 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
}
break;
+ case LT_LT_QNAME:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = HEREDOC_BOL;
+ pending_heredoc_ = true;
+ t_ = NORMAL;
+ } else {
+ heredoc_ += c;
+ }
+ break;
+
case HEREDOC_BOL:
*r += c;
if (c == '\n') {
@@ -237,13 +255,13 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
*r += HI_RESET;
}
i_ = 0;
- } else if (isalnum(c) || c == '_') {
- if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
- i_++;
- } else {
- t_ = HEREDOC;
- i_ = 0;
- }
+ } else if (c == '\t' && indented_heredoc_) {
+ // do nothing
+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
+ i_++;
+ } else {
+ t_ = HEREDOC;
+ i_ = 0;
}
break;
@@ -281,6 +299,7 @@ void HighlightPerl::flush(std::string *r) {
case COMMENT:
case HEREDOC_BOL:
case HEREDOC:
+ case LT_LT_QNAME:
*r += HI_RESET;
break;
default:
diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cpp
new file mode 100644
index 00000000..75f72e5b
--- /dev/null
+++ b/llamafile/highlight_ruby.cpp
@@ -0,0 +1,624 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+// ruby lexical syntax is bananas and isn't even formally documented
+//
+// known issues
+//
+// 1. we can't tell this backtick apart from command substitution
+//
+// def `(command)
+// return "just testing a backquote override"
+// end
+//
+// 2. we don't know <<arg isn't a heredoc. emacs doesn't get this right
+// either, but somehow vim does.
+//
+// when /\.*\.h/
+// options[:includes] <<arg; true
+// when /--(\w+)=\"?(.*)\"?/
+// options[$1.to_sym] = $2; true
+//
+
+enum {
+ NORMAL,
+ WORD,
+ EQUAL,
+ EQUAL_WORD,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ DQUOTE,
+ DQUOTE_BACKSLASH,
+ TICK,
+ TICK_BACKSLASH,
+ COMMENT,
+ LT,
+ LT_LT,
+ LT_LT_NAME,
+ LT_LT_QNAME,
+ HEREDOC_BOL,
+ HEREDOC,
+ COLON,
+ COLON_WORD,
+ AT,
+ AT_WORD,
+ DOLLAR,
+ DOLLAR_WORD,
+ PERCENT,
+ PERCENT2,
+ PERCENT_STRING,
+ MULTICOM,
+ MULTICOM_BOL,
+ REGEX,
+ REGEX_BACKSLASH,
+};
+
+static int mirror(int c) {
+ switch (c) {
+ case '(':
+ return ')';
+ case '{':
+ return '}';
+ case '[':
+ return ']';
+ case '<':
+ return '>';
+ default:
+ return c;
+ }
+}
+
+static bool isident(int c) {
+ return !isascii(c) || //
+ isalnum(c) || //
+ c == '!' || //
+ c == '$' || //
+ c == '%' || //
+ c == '&' || //
+ c == '-' || //
+ c == '/' || //
+ c == '=' || //
+ c == '?' || //
+ c == '@' || //
+ c == '^' || //
+ c == '_';
+}
+
+HighlightRuby::HighlightRuby() {
+}
+
+HighlightRuby::~HighlightRuby() {
+}
+
+void HighlightRuby::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+
+ if (!isblank(c) && c != '/' && c != '<')
+ last_ = c;
+
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == ':') {
+ t_ = COLON;
+ } else if (c == '@') {
+ t_ = AT;
+ } else if (c == '=') {
+ t_ = EQUAL;
+ } else if (c == '$') {
+ t_ = DOLLAR;
+ } else if (c == '%') {
+ t_ = PERCENT;
+ q_ = 0;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '`') {
+ t_ = TICK;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '#') {
+ *r += HI_COMMENT;
+ *r += c;
+ t_ = COMMENT;
+ } else if (c == '<' && !isalnum(last_)) {
+ *r += c;
+ t_ = LT;
+ } else if (c == '/' && !isalnum(last_)) {
+ t_ = REGEX;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '\n') {
+ *r += c;
+ if (pending_heredoc_) {
+ *r += HI_STRING;
+ pending_heredoc_ = false;
+ t_ = HEREDOC_BOL;
+ i_ = 0;
+ }
+ } else {
+ *r += c;
+ }
+ break;
+
+ case EQUAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = EQUAL_WORD;
+ word_ += c;
+ } else {
+ *r += '=';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case EQUAL_WORD:
+ if (isident(c)) {
+ word_ += c;
+ break;
+ } else if (word_ == "begin") {
+ *r += HI_COMMENT;
+ *r += "=begin";
+ *r += c;
+ if (c == '\n') {
+ t_ = MULTICOM_BOL;
+ i_ = 0;
+ } else {
+ t_ = MULTICOM;
+ }
+ word_.clear();
+ break;
+ } else {
+ *r += '=';
+ t_ = WORD;
+ }
+ // fallthrough
+
+ case WORD:
+ if (isident(c)) {
+ word_ += c;
+ } else {
+ if (is_keyword_ruby(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ last_ = 0;
+ } else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (!word_.empty() && isupper(word_[0])) {
+ *r += HI_CLASS;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case REGEX:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = REGEX_BACKSLASH;
+ }
+ break;
+
+ case REGEX_BACKSLASH:
+ *r += c;
+ t_ = REGEX;
+ break;
+
+ case MULTICOM:
+ *r += c;
+ if (c == '\n') {
+ t_ = MULTICOM_BOL;
+ i_ = 0;
+ }
+ break;
+
+ case MULTICOM_BOL:
+ *r += c;
+ if (c == "=end"[i_]) {
+ if (++i_ == 4) {
+ t_ = NORMAL;
+ *r += HI_RESET;
+ }
+ } else {
+ t_ = MULTICOM;
+ }
+ break;
+
+ case COLON:
+ if (isident(c)) {
+ *r += HI_LISPKW;
+ *r += ':';
+ *r += c;
+ t_ = COLON_WORD;
+ } else {
+ *r += ':';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case COLON_WORD:
+ if (isident(c)) {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case AT:
+ if (isident(c)) {
+ *r += HI_VAR;
+ *r += '@';
+ *r += c;
+ t_ = AT_WORD;
+ } else {
+ *r += '@';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case AT_WORD:
+ if (isident(c)) {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case PERCENT:
+ if (c == 'q' || c == 'Q') {
+ q_ = c;
+ t_ = PERCENT2;
+ } else if (ispunct(c)) {
+ level_ = 1;
+ opener_ = c;
+ closer_ = mirror(c);
+ *r += HI_STRING;
+ *r += '%';
+ *r += c;
+ t_ = PERCENT_STRING;
+ } else {
+ *r += '%';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case PERCENT2:
+ if (ispunct(c)) {
+ level_ = 1;
+ opener_ = c;
+ closer_ = mirror(c);
+ *r += HI_STRING;
+ *r += '%';
+ *r += q_;
+ *r += c;
+ t_ = PERCENT_STRING;
+ } else {
+ *r += '%';
+ *r += q_;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case PERCENT_STRING:
+ *r += c;
+ if (c == opener_ && opener_ != closer_) {
+ ++level_;
+ } else if (c == closer_) {
+ if (!--level_) {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ }
+ break;
+
+ case DOLLAR:
+ if (isdigit(c) || //
+ c == '!' || //
+ c == '#' || //
+ c == '$' || //
+ c == '&' || //
+ c == '-' || //
+ c == '/' || //
+ c == '<' || //
+ c == '=' || //
+ c == '>' || //
+ c == '@' || //
+ c == '\\' || //
+ c == '^' || //
+ c == '_' || //
+ c == '`') {
+ *r += HI_VAR;
+ *r += '$';
+ *r += c;
+ *r += HI_RESET;
+ t_ = AT_WORD;
+ } else if (isalpha(c)) {
+ *r += HI_VAR;
+ *r += '$';
+ *r += c;
+ t_ = DOLLAR_WORD;
+ } else {
+ *r += '$';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case DOLLAR_WORD:
+ if (isident(c)) {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case COMMENT:
+ if (c == '\n') {
+ *r += HI_RESET;
+ *r += c;
+ t_ = NORMAL;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE;
+ break;
+
+ case TICK:
+ *r += c;
+ if (c == '`') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = TICK_BACKSLASH;
+ }
+ break;
+
+ case TICK_BACKSLASH:
+ *r += c;
+ t_ = TICK;
+ break;
+
+ case LT:
+ if (c == '<') {
+ *r += c;
+ t_ = LT_LT;
+ heredoc_.clear();
+ pending_heredoc_ = false;
+ indented_heredoc_ = false;
+ } else {
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case LT_LT:
+ if (c == '-') {
+ indented_heredoc_ = true;
+ *r += c;
+ } else if (c == '\'' || c == '`' || c == '"') {
+ closer_ = c;
+ t_ = LT_LT_QNAME;
+ *r += HI_STRING;
+ *r += c;
+ } else if (isalpha(c) || c == '_') {
+ t_ = LT_LT_NAME;
+ heredoc_ += c;
+ *r += c;
+ } else if (!isblank(c)) {
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case LT_LT_NAME:
+ if (isalnum(c) || c == '_') {
+ t_ = LT_LT_NAME;
+ heredoc_ += c;
+ *r += c;
+ } else if (c == '\n') {
+ *r += c;
+ *r += HI_STRING;
+ t_ = HEREDOC_BOL;
+ } else {
+ pending_heredoc_ = true;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case LT_LT_QNAME:
+ *r += c;
+ if (c == closer_) {
+ *r += HI_RESET;
+ t_ = HEREDOC_BOL;
+ pending_heredoc_ = true;
+ t_ = NORMAL;
+ } else {
+ heredoc_ += c;
+ }
+ break;
+
+ case HEREDOC_BOL:
+ *r += c;
+ if (c == '\n') {
+ if (i_ == heredoc_.size()) {
+ t_ = NORMAL;
+ *r += HI_RESET;
+ }
+ i_ = 0;
+ } else if (c == '\t' && indented_heredoc_) {
+ // do nothing
+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
+ i_++;
+ } else {
+ t_ = HEREDOC;
+ i_ = 0;
+ }
+ break;
+
+ case HEREDOC:
+ *r += c;
+ if (c == '\n')
+ t_ = HEREDOC_BOL;
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightRuby::flush(std::string *r) {
+ switch (t_) {
+ case EQUAL_WORD:
+ *r += '=';
+ // fallthrough
+ case WORD:
+ if (is_keyword_ruby(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (!word_.empty() && isupper(word_[0])) {
+ *r += HI_CLASS;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case AT:
+ *r += '@';
+ break;
+ case EQUAL:
+ *r += '=';
+ break;
+ case COLON:
+ *r += ':';
+ break;
+ case DOLLAR:
+ *r += '$';
+ break;
+ case PERCENT:
+ *r += '%';
+ break;
+ case PERCENT2:
+ *r += '%';
+ *r += q_;
+ break;
+ case REGEX:
+ case REGEX_BACKSLASH:
+ case PERCENT_STRING:
+ case AT_WORD:
+ case DOLLAR_WORD:
+ case TICK:
+ case TICK_BACKSLASH:
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case DQUOTE:
+ case DQUOTE_BACKSLASH:
+ case COMMENT:
+ case HEREDOC_BOL:
+ case HEREDOC:
+ case LT_LT_QNAME:
+ case COLON_WORD:
+ case MULTICOM:
+ case MULTICOM_BOL:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cpp
index 91cded5b..5d6ad093 100644
--- a/llamafile/highlight_shell.cpp
+++ b/llamafile/highlight_shell.cpp
@@ -33,6 +33,7 @@ enum {
LT,
LT_LT,
LT_LT_NAME,
+ LT_LT_QNAME,
HEREDOC_BOL,
HEREDOC,
};
@@ -200,6 +201,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
t_ = LT_LT;
heredoc_.clear();
pending_heredoc_ = false;
+ indented_heredoc_ = false;
} else {
t_ = NORMAL;
goto Normal;
@@ -207,8 +209,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
break;
case LT_LT:
- if (c == '\'') {
- t_ = LT_LT_NAME;
+ if (c == '-') {
+ indented_heredoc_ = true;
+ *r += c;
+ } else if (c == '\'') {
+ t_ = LT_LT_QNAME;
+ *r += HI_STRING;
*r += c;
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
@@ -236,6 +242,18 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
}
break;
+ case LT_LT_QNAME:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = HEREDOC_BOL;
+ pending_heredoc_ = true;
+ t_ = NORMAL;
+ } else {
+ heredoc_ += c;
+ }
+ break;
+
case HEREDOC_BOL:
*r += c;
if (c == '\n') {
@@ -244,13 +262,13 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
*r += HI_RESET;
}
i_ = 0;
- } else if (isalnum(c) || c == '_') {
- if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
- i_++;
- } else {
- t_ = HEREDOC;
- i_ = 0;
- }
+ } else if (c == '\t' && indented_heredoc_) {
+ // do nothing
+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {
+ i_++;
+ } else {
+ t_ = HEREDOC;
+ i_ = 0;
}
break;
@@ -291,6 +309,7 @@ void HighlightShell::flush(std::string *r) {
case COMMENT:
case HEREDOC_BOL:
case HEREDOC:
+ case LT_LT_QNAME:
*r += HI_RESET;
break;
default:
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index b1de2d2b..0ab73bac 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -25,7 +25,7 @@
#define LENGTH 10
#define ITERATIONS 200000
-#define CHARSET "abc123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"
+#define CHARSET "aQq123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"
const char *const kLanguages[] = {
"ada", //
@@ -50,11 +50,12 @@ const char *const kLanguages[] = {
"php!", //
"plain", //
"python", //
+ "ruby", //
"rust", //
"shell", //
"sql", //
- "tcl", //
"swift", //
+ "tcl", //
"zig", //
};
diff --git a/llamafile/is_keyword_ruby.c b/llamafile/is_keyword_ruby.c
new file mode 100644
index 00000000..0ee566e1
--- /dev/null
+++ b/llamafile/is_keyword_ruby.c
@@ -0,0 +1,232 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_ruby.gperf */
+/* Computed positions: -k'1,$' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_ruby.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 36
+#define MIN_WORD_LENGTH 2
+#define MAX_WORD_LENGTH 8
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 60
+/* maximum key range = 58, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 15, 61, 61, 61, 30, 0, 61,
+ 0, 61, 61, 61, 61, 61, 61, 61, 25, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 30, 30,
+ 0, 5, 5, 15, 61, 61, 5, 20, 25, 61,
+ 15, 0, 10, 61, 5, 0, 15, 15, 0, 61,
+ 10, 40, 35, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
+ 61, 61, 61, 61, 61, 61, 61
+ };
+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]+1];
+}
+
+const char *
+is_keyword_ruby (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("END")];
+ char stringpool_str4[sizeof("redo")];
+ char stringpool_str5[sizeof("class")];
+ char stringpool_str6[sizeof("unless")];
+ char stringpool_str7[sizeof("do")];
+ char stringpool_str8[sizeof("end")];
+ char stringpool_str9[sizeof("case")];
+ char stringpool_str10[sizeof("undef")];
+ char stringpool_str11[sizeof("rescue")];
+ char stringpool_str12[sizeof("if")];
+ char stringpool_str13[sizeof("def")];
+ char stringpool_str14[sizeof("else")];
+ char stringpool_str15[sizeof("elsif")];
+ char stringpool_str16[sizeof("ensure")];
+ char stringpool_str17[sizeof("or")];
+ char stringpool_str18[sizeof("not")];
+ char stringpool_str19[sizeof("next")];
+ char stringpool_str20[sizeof("while")];
+ char stringpool_str21[sizeof("return")];
+ char stringpool_str22[sizeof("in")];
+ char stringpool_str23[sizeof("for")];
+ char stringpool_str24[sizeof("self")];
+ char stringpool_str25[sizeof("super")];
+ char stringpool_str26[sizeof("module")];
+ char stringpool_str28[sizeof("defined?")];
+ char stringpool_str29[sizeof("when")];
+ char stringpool_str30[sizeof("until")];
+ char stringpool_str33[sizeof("and")];
+ char stringpool_str34[sizeof("then")];
+ char stringpool_str35[sizeof("alias")];
+ char stringpool_str40[sizeof("yield")];
+ char stringpool_str44[sizeof("fail")];
+ char stringpool_str45[sizeof("retry")];
+ char stringpool_str50[sizeof("begin")];
+ char stringpool_str55[sizeof("break")];
+ char stringpool_str60[sizeof("BEGIN")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "END",
+ "redo",
+ "class",
+ "unless",
+ "do",
+ "end",
+ "case",
+ "undef",
+ "rescue",
+ "if",
+ "def",
+ "else",
+ "elsif",
+ "ensure",
+ "or",
+ "not",
+ "next",
+ "while",
+ "return",
+ "in",
+ "for",
+ "self",
+ "super",
+ "module",
+ "defined?",
+ "when",
+ "until",
+ "and",
+ "then",
+ "alias",
+ "yield",
+ "fail",
+ "retry",
+ "begin",
+ "break",
+ "BEGIN"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_ruby.gperf b/llamafile/is_keyword_ruby.gperf
new file mode 100644
index 00000000..c7306862
--- /dev/null
+++ b/llamafile/is_keyword_ruby.gperf
@@ -0,0 +1,45 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_ruby
+%%
+BEGIN
+END
+alias
+and
+begin
+break
+case
+class
+def
+defined?
+do
+else
+elsif
+end
+ensure
+fail
+for
+if
+in
+module
+next
+not
+or
+redo
+rescue
+retry
+return
+self
+super
+then
+undef
+unless
+until
+when
+while
+yield
diff --git a/llamafile/is_keyword_ruby_builtin.c b/llamafile/is_keyword_ruby_builtin.c
new file mode 100644
index 00000000..b4d25479
--- /dev/null
+++ b/llamafile/is_keyword_ruby_builtin.c
@@ -0,0 +1,369 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_ruby_builtin.gperf */
+/* Computed positions: -k'1-2,4,6' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_ruby_builtin.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 67
+#define MIN_WORD_LENGTH 1
+#define MAX_WORD_LENGTH 20
+#define MIN_HASH_VALUE 4
+#define MAX_HASH_VALUE 136
+/* maximum key range = 133, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 20, 137, 0, 45, 5,
+ 25, 0, 85, 0, 55, 35, 137, 0, 5, 5,
+ 10, 40, 5, 137, 0, 20, 0, 10, 40, 65,
+ 80, 20, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,
+ 137, 137, 137, 137, 137, 137
+ };
+ register unsigned int hval = len;
+
+ switch (hval)
+ {
+ default:
+ hval += asso_values[(unsigned char)str[5]];
+ /*FALLTHROUGH*/
+ case 5:
+ case 4:
+ hval += asso_values[(unsigned char)str[3]];
+ /*FALLTHROUGH*/
+ case 3:
+ case 2:
+ hval += asso_values[(unsigned char)str[1]];
+ /*FALLTHROUGH*/
+ case 1:
+ hval += asso_values[(unsigned char)str[0]];
+ break;
+ }
+ return hval;
+}
+
+const char *
+is_keyword_ruby_builtin (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str4[sizeof("attr")];
+ char stringpool_str6[sizeof("p")];
+ char stringpool_str9[sizeof("trap")];
+ char stringpool_str11[sizeof("attr_reader")];
+ char stringpool_str13[sizeof("attr_accessor")];
+ char stringpool_str14[sizeof("proc")];
+ char stringpool_str15[sizeof("catch")];
+ char stringpool_str16[sizeof("caller")];
+ char stringpool_str17[sizeof("require")];
+ char stringpool_str19[sizeof("protected")];
+ char stringpool_str20[sizeof("print")];
+ char stringpool_str21[sizeof("callcc")];
+ char stringpool_str24[sizeof("putc")];
+ char stringpool_str25[sizeof("raise")];
+ char stringpool_str26[sizeof("require_relative")];
+ char stringpool_str27[sizeof("prepend")];
+ char stringpool_str29[sizeof("rand")];
+ char stringpool_str30[sizeof("sleep")];
+ char stringpool_str31[sizeof("public")];
+ char stringpool_str34[sizeof("trace_var")];
+ char stringpool_str35[sizeof("srand")];
+ char stringpool_str36[sizeof("untrace_var")];
+ char stringpool_str37[sizeof("alias_method")];
+ char stringpool_str39[sizeof("puts")];
+ char stringpool_str40[sizeof("public_constant")];
+ char stringpool_str41[sizeof("refine")];
+ char stringpool_str42[sizeof("at_exit")];
+ char stringpool_str44[sizeof("public_class_method")];
+ char stringpool_str45[sizeof("using")];
+ char stringpool_str49[sizeof("eval")];
+ char stringpool_str50[sizeof("abort")];
+ char stringpool_str51[sizeof("system")];
+ char stringpool_str52[sizeof("private")];
+ char stringpool_str54[sizeof("loop")];
+ char stringpool_str55[sizeof("__callee__")];
+ char stringpool_str56[sizeof("lambda")];
+ char stringpool_str57[sizeof("syscall")];
+ char stringpool_str59[sizeof("open")];
+ char stringpool_str61[sizeof("private_constant")];
+ char stringpool_str65[sizeof("private_class_method")];
+ char stringpool_str67[sizeof("sprintf")];
+ char stringpool_str68[sizeof("readline")];
+ char stringpool_str69[sizeof("readlines")];
+ char stringpool_str70[sizeof("module_function")];
+ char stringpool_str71[sizeof("global_variables")];
+ char stringpool_str73[sizeof("define_method")];
+ char stringpool_str74[sizeof("load")];
+ char stringpool_str76[sizeof("attr_writer")];
+ char stringpool_str79[sizeof("warn")];
+ char stringpool_str80[sizeof("local_variables")];
+ char stringpool_str82[sizeof("include")];
+ char stringpool_str84[sizeof("exit")];
+ char stringpool_str85[sizeof("exit!")];
+ char stringpool_str87[sizeof("block_given?")];
+ char stringpool_str89[sizeof("exec")];
+ char stringpool_str94[sizeof("fail")];
+ char stringpool_str95[sizeof("spawn")];
+ char stringpool_str98[sizeof("autoload")];
+ char stringpool_str99[sizeof("autoload?")];
+ char stringpool_str100[sizeof("throw")];
+ char stringpool_str102[sizeof("__dir__")];
+ char stringpool_str105[sizeof("__method__")];
+ char stringpool_str106[sizeof("printf")];
+ char stringpool_str111[sizeof("extend")];
+ char stringpool_str122[sizeof("binding")];
+ char stringpool_str129[sizeof("fork")];
+ char stringpool_str136[sizeof("format")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "attr",
+ "p",
+ "trap",
+ "attr_reader",
+ "attr_accessor",
+ "proc",
+ "catch",
+ "caller",
+ "require",
+ "protected",
+ "print",
+ "callcc",
+ "putc",
+ "raise",
+ "require_relative",
+ "prepend",
+ "rand",
+ "sleep",
+ "public",
+ "trace_var",
+ "srand",
+ "untrace_var",
+ "alias_method",
+ "puts",
+ "public_constant",
+ "refine",
+ "at_exit",
+ "public_class_method",
+ "using",
+ "eval",
+ "abort",
+ "system",
+ "private",
+ "loop",
+ "__callee__",
+ "lambda",
+ "syscall",
+ "open",
+ "private_constant",
+ "private_class_method",
+ "sprintf",
+ "readline",
+ "readlines",
+ "module_function",
+ "global_variables",
+ "define_method",
+ "load",
+ "attr_writer",
+ "warn",
+ "local_variables",
+ "include",
+ "exit",
+ "exit!",
+ "block_given?",
+ "exec",
+ "fail",
+ "spawn",
+ "autoload",
+ "autoload?",
+ "throw",
+ "__dir__",
+ "__method__",
+ "printf",
+ "extend",
+ "binding",
+ "fork",
+ "format"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str56,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str65,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str80,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str84,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str87,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str89,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str94,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str95,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str98,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str100,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str105,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str106,
+ -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str111,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str129,
+ -1, -1, -1, -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_ruby_builtin.gperf b/llamafile/is_keyword_ruby_builtin.gperf
new file mode 100644
index 00000000..0e6dd677
--- /dev/null
+++ b/llamafile/is_keyword_ruby_builtin.gperf
@@ -0,0 +1,76 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_ruby_builtin
+%%
+__callee__
+__dir__
+__method__
+abort
+alias_method
+at_exit
+attr
+attr_accessor
+attr_reader
+attr_writer
+autoload
+autoload?
+binding
+block_given?
+callcc
+caller
+catch
+define_method
+eval
+exec
+exit
+exit!
+extend
+fail
+fork
+format
+global_variables
+include
+lambda
+load
+local_variables
+loop
+module_function
+open
+p
+prepend
+print
+printf
+private
+private_class_method
+private_constant
+proc
+protected
+public
+public_class_method
+public_constant
+putc
+puts
+raise
+rand
+readline
+readlines
+refine
+require
+require_relative
+sleep
+spawn
+sprintf
+srand
+syscall
+system
+throw
+trace_var
+trap
+untrace_var
+using
+warn
diff --git a/llamafile/is_keyword_ruby_constant.c b/llamafile/is_keyword_ruby_constant.c
new file mode 100644
index 00000000..b5bce6c1
--- /dev/null
+++ b/llamafile/is_keyword_ruby_constant.c
@@ -0,0 +1,137 @@
+/* ANSI-C code produced by gperf version 3.1 */
+/* Command-line: gperf llamafile/is_keyword_ruby_constant.gperf */
+/* Computed positions: -k'3' */
+
+#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \
+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \
+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \
+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \
+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \
+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \
+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \
+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \
+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \
+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \
+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \
+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \
+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \
+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \
+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \
+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \
+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \
+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \
+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \
+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \
+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \
+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \
+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))
+/* The character set is not based on ISO-646. */
+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."
+#endif
+
+#line 1 "llamafile/is_keyword_ruby_constant.gperf"
+
+#include <string.h>
+
+#define TOTAL_KEYWORDS 6
+#define MIN_WORD_LENGTH 3
+#define MAX_WORD_LENGTH 12
+#define MIN_HASH_VALUE 3
+#define MAX_HASH_VALUE 13
+/* maximum key range = 11, duplicates = 0 */
+
+#ifdef __GNUC__
+__inline
+#else
+#ifdef __cplusplus
+inline
+#endif
+#endif
+static unsigned int
+hash (register const char *str, register size_t len)
+{
+ static const unsigned char asso_values[] =
+ {
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 0,
+ 5, 14, 14, 14, 14, 14, 0, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 0, 14,
+ 14, 14, 14, 14, 14, 14, 14, 0, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 14, 14, 14
+ };
+ return len + asso_values[(unsigned char)str[2]];
+}
+
+const char *
+is_keyword_ruby_constant (register const char *str, register size_t len)
+{
+ struct stringpool_t
+ {
+ char stringpool_str3[sizeof("nil")];
+ char stringpool_str4[sizeof("true")];
+ char stringpool_str5[sizeof("false")];
+ char stringpool_str8[sizeof("__LINE__")];
+ char stringpool_str12[sizeof("__ENCODING__")];
+ char stringpool_str13[sizeof("__FILE__")];
+ };
+ static const struct stringpool_t stringpool_contents =
+ {
+ "nil",
+ "true",
+ "false",
+ "__LINE__",
+ "__ENCODING__",
+ "__FILE__"
+ };
+ #define stringpool ((const char *) &stringpool_contents)
+ static const int wordlist[] =
+ {
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,
+ -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,
+ -1, -1, -1,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,
+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13
+ };
+
+ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
+ {
+ register unsigned int key = hash (str, len);
+
+ if (key <= MAX_HASH_VALUE)
+ {
+ register int o = wordlist[key];
+ if (o >= 0)
+ {
+ register const char *s = o + stringpool;
+
+ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')
+ return s;
+ }
+ }
+ }
+ return 0;
+}
diff --git a/llamafile/is_keyword_ruby_constant.gperf b/llamafile/is_keyword_ruby_constant.gperf
new file mode 100644
index 00000000..66b6cf09
--- /dev/null
+++ b/llamafile/is_keyword_ruby_constant.gperf
@@ -0,0 +1,15 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_ruby_constant
+%%
+__ENCODING__
+__FILE__
+__LINE__
+false
+nil
+true