Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add Ruby syntax highlighting
Justine Tunney0f2c86b74441
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 8013d2c7..233b3b7d 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -106,9 +106,12 @@ Highlight *Highlight::create(const std::string_view &lang) {} else if (lang == "d") {return new HighlightC(is_keyword_d);} else if (lang == "zig") {- return new HighlightZig();+ return new HighlightZig;} else if (lang == "tcl") {- return new HighlightTcl();+ return new HighlightTcl;+ } else if (lang == "ruby" || //+ lang == "rb") {+ return new HighlightRuby;} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 1c2cc8fe..ca6f88b0 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -31,6 +31,7 @@#define HI_CONTIN "\033[33m" // yellow#define HI_LABEL "\033[33m" // yellow#define HI_TYPE "\033[36m" // cyan+#define HI_CLASS "\033[1;36m" // bold cyan#define HI_SELECTOR "\033[33m" // yellow#define HI_PROPERTY "\033[36m" // cyan#define HI_TAG "\033[33m" // yellow@@ -86,6 +87,9 @@ is_keyword_f is_keyword_zig_constant;is_keyword_f is_keyword_tcl;is_keyword_f is_keyword_tcl_type;is_keyword_f is_keyword_tcl_builtin;+is_keyword_f is_keyword_ruby;+is_keyword_f is_keyword_ruby_builtin;+is_keyword_f is_keyword_ruby_constant;}class Highlight {@@ -312,6 +316,7 @@ class HighlightPerl : public Highlight {int t_ = 0;int i_ = 0;bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;std::string word_;std::string heredoc_;};@@ -327,6 +332,7 @@ class HighlightShell : public Highlight {int t_ = 0;int i_ = 0;bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;std::string word_;std::string heredoc_;};@@ -368,3 +374,24 @@ class HighlightCsharp : public Highlight {int trips2_;std::string word_;};++class HighlightRuby : public Highlight {+ public:+ HighlightRuby();+ ~HighlightRuby() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ int i_ = 0;+ int level_ = 0;+ unsigned char q_ = 0;+ unsigned char last_ = 0;+ unsigned char opener_ = 0;+ unsigned char closer_ = 0;+ bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;+ std::string word_;+ std::string heredoc_;+};diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight_perl.cppindex 2d43afc4..fe2465c5 100644--- a/llamafile/highlight_perl.cpp+++ b/llamafile/highlight_perl.cpp@@ -34,6 +34,7 @@ enum {LT,LT_LT,LT_LT_NAME,+ LT_LT_QNAME,HEREDOC_BOL,HEREDOC,};@@ -193,6 +194,7 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {t_ = LT_LT;heredoc_.clear();pending_heredoc_ = false;+ indented_heredoc_ = false;} else {t_ = NORMAL;goto Normal;@@ -200,8 +202,12 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {break;case LT_LT:- if (c == '\'') {- t_ = LT_LT_NAME;+ if (c == '-') {+ indented_heredoc_ = true;+ *r += c;+ } else if (c == '\'') {+ t_ = LT_LT_QNAME;+ *r += HI_STRING;*r += c;} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;@@ -229,6 +235,18 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {}break;+ case LT_LT_QNAME:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = HEREDOC_BOL;+ pending_heredoc_ = true;+ t_ = NORMAL;+ } else {+ heredoc_ += c;+ }+ break;+case HEREDOC_BOL:*r += c;if (c == '\n') {@@ -237,13 +255,13 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {*r += HI_RESET;}i_ = 0;- } else if (isalnum(c) || c == '_') {- if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {- i_++;- } else {- t_ = HEREDOC;- i_ = 0;- }+ } else if (c == '\t' && indented_heredoc_) {+ // do nothing+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {+ i_++;+ } else {+ t_ = HEREDOC;+ i_ = 0;}break;@@ -281,6 +299,7 @@ void HighlightPerl::flush(std::string *r) {case COMMENT:case HEREDOC_BOL:case HEREDOC:+ case LT_LT_QNAME:*r += HI_RESET;break;default:diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cppnew file mode 100644index 00000000..75f72e5b--- /dev/null+++ b/llamafile/highlight_ruby.cpp@@ -0,0 +1,624 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++// ruby lexical syntax is bananas and isn't even formally documented+//+// known issues+//+// 1. we can't tell this backtick apart from command substitution+//+// def `(command)+// return "just testing a backquote override"+// end+//+// 2. we don't know <<arg isn't a heredoc. emacs doesn't get this right+// either, but somehow vim does.+//+// when /\.*\.h/+// options[:includes] <<arg; true+// when /--(\w+)=\"?(.*)\"?/+// options[$1.to_sym] = $2; true+//++enum {+ NORMAL,+ WORD,+ EQUAL,+ EQUAL_WORD,+ QUOTE,+ QUOTE_BACKSLASH,+ DQUOTE,+ DQUOTE_BACKSLASH,+ TICK,+ TICK_BACKSLASH,+ COMMENT,+ LT,+ LT_LT,+ LT_LT_NAME,+ LT_LT_QNAME,+ HEREDOC_BOL,+ HEREDOC,+ COLON,+ COLON_WORD,+ AT,+ AT_WORD,+ DOLLAR,+ DOLLAR_WORD,+ PERCENT,+ PERCENT2,+ PERCENT_STRING,+ MULTICOM,+ MULTICOM_BOL,+ REGEX,+ REGEX_BACKSLASH,+};++static int mirror(int c) {+ switch (c) {+ case '(':+ return ')';+ case '{':+ return '}';+ case '[':+ return ']';+ case '<':+ return '>';+ default:+ return c;+ }+}++static bool isident(int c) {+ return !isascii(c) || //+ isalnum(c) || //+ c == '!' || //+ c == '$' || //+ c == '%' || //+ c == '&' || //+ c == '-' || //+ c == '/' || //+ c == '=' || //+ c == '?' || //+ c == '@' || //+ c == '^' || //+ c == '_';+}++HighlightRuby::HighlightRuby() {+}++HighlightRuby::~HighlightRuby() {+}++void HighlightRuby::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;++ if (!isblank(c) && c != '/' && c != '<')+ last_ = c;++ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == ':') {+ t_ = COLON;+ } else if (c == '@') {+ t_ = AT;+ } else if (c == '=') {+ t_ = EQUAL;+ } else if (c == '$') {+ t_ = DOLLAR;+ } else if (c == '%') {+ t_ = PERCENT;+ q_ = 0;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '`') {+ t_ = TICK;+ *r += HI_STRING;+ *r += c;+ } else if (c == '#') {+ *r += HI_COMMENT;+ *r += c;+ t_ = COMMENT;+ } else if (c == '<' && !isalnum(last_)) {+ *r += c;+ t_ = LT;+ } else if (c == '/' && !isalnum(last_)) {+ t_ = REGEX;+ *r += HI_STRING;+ *r += c;+ } else if (c == '\n') {+ *r += c;+ if (pending_heredoc_) {+ *r += HI_STRING;+ pending_heredoc_ = false;+ t_ = HEREDOC_BOL;+ i_ = 0;+ }+ } else {+ *r += c;+ }+ break;++ case EQUAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = EQUAL_WORD;+ word_ += c;+ } else {+ *r += '=';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case EQUAL_WORD:+ if (isident(c)) {+ word_ += c;+ break;+ } else if (word_ == "begin") {+ *r += HI_COMMENT;+ *r += "=begin";+ *r += c;+ if (c == '\n') {+ t_ = MULTICOM_BOL;+ i_ = 0;+ } else {+ t_ = MULTICOM;+ }+ word_.clear();+ break;+ } else {+ *r += '=';+ t_ = WORD;+ }+ // fallthrough++ case WORD:+ if (isident(c)) {+ word_ += c;+ } else {+ if (is_keyword_ruby(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ last_ = 0;+ } else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else if (!word_.empty() && isupper(word_[0])) {+ *r += HI_CLASS;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case REGEX:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = REGEX_BACKSLASH;+ }+ break;++ case REGEX_BACKSLASH:+ *r += c;+ t_ = REGEX;+ break;++ case MULTICOM:+ *r += c;+ if (c == '\n') {+ t_ = MULTICOM_BOL;+ i_ = 0;+ }+ break;++ case MULTICOM_BOL:+ *r += c;+ if (c == "=end"[i_]) {+ if (++i_ == 4) {+ t_ = NORMAL;+ *r += HI_RESET;+ }+ } else {+ t_ = MULTICOM;+ }+ break;++ case COLON:+ if (isident(c)) {+ *r += HI_LISPKW;+ *r += ':';+ *r += c;+ t_ = COLON_WORD;+ } else {+ *r += ':';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case COLON_WORD:+ if (isident(c)) {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case AT:+ if (isident(c)) {+ *r += HI_VAR;+ *r += '@';+ *r += c;+ t_ = AT_WORD;+ } else {+ *r += '@';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case AT_WORD:+ if (isident(c)) {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case PERCENT:+ if (c == 'q' || c == 'Q') {+ q_ = c;+ t_ = PERCENT2;+ } else if (ispunct(c)) {+ level_ = 1;+ opener_ = c;+ closer_ = mirror(c);+ *r += HI_STRING;+ *r += '%';+ *r += c;+ t_ = PERCENT_STRING;+ } else {+ *r += '%';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case PERCENT2:+ if (ispunct(c)) {+ level_ = 1;+ opener_ = c;+ closer_ = mirror(c);+ *r += HI_STRING;+ *r += '%';+ *r += q_;+ *r += c;+ t_ = PERCENT_STRING;+ } else {+ *r += '%';+ *r += q_;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case PERCENT_STRING:+ *r += c;+ if (c == opener_ && opener_ != closer_) {+ ++level_;+ } else if (c == closer_) {+ if (!--level_) {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ }+ break;++ case DOLLAR:+ if (isdigit(c) || //+ c == '!' || //+ c == '#' || //+ c == '$' || //+ c == '&' || //+ c == '-' || //+ c == '/' || //+ c == '<' || //+ c == '=' || //+ c == '>' || //+ c == '@' || //+ c == '\\' || //+ c == '^' || //+ c == '_' || //+ c == '`') {+ *r += HI_VAR;+ *r += '$';+ *r += c;+ *r += HI_RESET;+ t_ = AT_WORD;+ } else if (isalpha(c)) {+ *r += HI_VAR;+ *r += '$';+ *r += c;+ t_ = DOLLAR_WORD;+ } else {+ *r += '$';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case DOLLAR_WORD:+ if (isident(c)) {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case COMMENT:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE_BACKSLASH;+ }+ break;++ case DQUOTE_BACKSLASH:+ *r += c;+ t_ = DQUOTE;+ break;++ case TICK:+ *r += c;+ if (c == '`') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = TICK_BACKSLASH;+ }+ break;++ case TICK_BACKSLASH:+ *r += c;+ t_ = TICK;+ break;++ case LT:+ if (c == '<') {+ *r += c;+ t_ = LT_LT;+ heredoc_.clear();+ pending_heredoc_ = false;+ indented_heredoc_ = false;+ } else {+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case LT_LT:+ if (c == '-') {+ indented_heredoc_ = true;+ *r += c;+ } else if (c == '\'' || c == '`' || c == '"') {+ closer_ = c;+ t_ = LT_LT_QNAME;+ *r += HI_STRING;+ *r += c;+ } else if (isalpha(c) || c == '_') {+ t_ = LT_LT_NAME;+ heredoc_ += c;+ *r += c;+ } else if (!isblank(c)) {+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case LT_LT_NAME:+ if (isalnum(c) || c == '_') {+ t_ = LT_LT_NAME;+ heredoc_ += c;+ *r += c;+ } else if (c == '\n') {+ *r += c;+ *r += HI_STRING;+ t_ = HEREDOC_BOL;+ } else {+ pending_heredoc_ = true;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case LT_LT_QNAME:+ *r += c;+ if (c == closer_) {+ *r += HI_RESET;+ t_ = HEREDOC_BOL;+ pending_heredoc_ = true;+ t_ = NORMAL;+ } else {+ heredoc_ += c;+ }+ break;++ case HEREDOC_BOL:+ *r += c;+ if (c == '\n') {+ if (i_ == heredoc_.size()) {+ t_ = NORMAL;+ *r += HI_RESET;+ }+ i_ = 0;+ } else if (c == '\t' && indented_heredoc_) {+ // do nothing+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {+ i_++;+ } else {+ t_ = HEREDOC;+ i_ = 0;+ }+ break;++ case HEREDOC:+ *r += c;+ if (c == '\n')+ t_ = HEREDOC_BOL;+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightRuby::flush(std::string *r) {+ switch (t_) {+ case EQUAL_WORD:+ *r += '=';+ // fallthrough+ case WORD:+ if (is_keyword_ruby(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else if (!word_.empty() && isupper(word_[0])) {+ *r += HI_CLASS;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case AT:+ *r += '@';+ break;+ case EQUAL:+ *r += '=';+ break;+ case COLON:+ *r += ':';+ break;+ case DOLLAR:+ *r += '$';+ break;+ case PERCENT:+ *r += '%';+ break;+ case PERCENT2:+ *r += '%';+ *r += q_;+ break;+ case REGEX:+ case REGEX_BACKSLASH:+ case PERCENT_STRING:+ case AT_WORD:+ case DOLLAR_WORD:+ case TICK:+ case TICK_BACKSLASH:+ case QUOTE:+ case QUOTE_BACKSLASH:+ case DQUOTE:+ case DQUOTE_BACKSLASH:+ case COMMENT:+ case HEREDOC_BOL:+ case HEREDOC:+ case LT_LT_QNAME:+ case COLON_WORD:+ case MULTICOM:+ case MULTICOM_BOL:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight_shell.cppindex 91cded5b..5d6ad093 100644--- a/llamafile/highlight_shell.cpp+++ b/llamafile/highlight_shell.cpp@@ -33,6 +33,7 @@ enum {LT,LT_LT,LT_LT_NAME,+ LT_LT_QNAME,HEREDOC_BOL,HEREDOC,};@@ -200,6 +201,7 @@ void HighlightShell::feed(std::string *r, std::string_view input) {t_ = LT_LT;heredoc_.clear();pending_heredoc_ = false;+ indented_heredoc_ = false;} else {t_ = NORMAL;goto Normal;@@ -207,8 +209,12 @@ void HighlightShell::feed(std::string *r, std::string_view input) {break;case LT_LT:- if (c == '\'') {- t_ = LT_LT_NAME;+ if (c == '-') {+ indented_heredoc_ = true;+ *r += c;+ } else if (c == '\'') {+ t_ = LT_LT_QNAME;+ *r += HI_STRING;*r += c;} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;@@ -236,6 +242,18 @@ void HighlightShell::feed(std::string *r, std::string_view input) {}break;+ case LT_LT_QNAME:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = HEREDOC_BOL;+ pending_heredoc_ = true;+ t_ = NORMAL;+ } else {+ heredoc_ += c;+ }+ break;+case HEREDOC_BOL:*r += c;if (c == '\n') {@@ -244,13 +262,13 @@ void HighlightShell::feed(std::string *r, std::string_view input) {*r += HI_RESET;}i_ = 0;- } else if (isalnum(c) || c == '_') {- if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {- i_++;- } else {- t_ = HEREDOC;- i_ = 0;- }+ } else if (c == '\t' && indented_heredoc_) {+ // do nothing+ } else if (i_ < heredoc_.size() && (heredoc_[i_] & 255) == c) {+ i_++;+ } else {+ t_ = HEREDOC;+ i_ = 0;}break;@@ -291,6 +309,7 @@ void HighlightShell::flush(std::string *r) {case COMMENT:case HEREDOC_BOL:case HEREDOC:+ case LT_LT_QNAME:*r += HI_RESET;break;default:diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex b1de2d2b..0ab73bac 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -25,7 +25,7 @@#define LENGTH 10#define ITERATIONS 200000-#define CHARSET "abc123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"+#define CHARSET "aQq123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"const char *const kLanguages[] = {"ada", //@@ -50,11 +50,12 @@ const char *const kLanguages[] = {"php!", //"plain", //"python", //+ "ruby", //"rust", //"shell", //"sql", //- "tcl", //"swift", //+ "tcl", //"zig", //};diff --git a/llamafile/is_keyword_ruby.c b/llamafile/is_keyword_ruby.cnew file mode 100644index 00000000..0ee566e1--- /dev/null+++ b/llamafile/is_keyword_ruby.c@@ -0,0 +1,232 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_ruby.gperf */+/* Computed positions: -k'1,$' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_ruby.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 36+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 8+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 60+/* maximum key range = 58, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 15, 61, 61, 61, 30, 0, 61,+ 0, 61, 61, 61, 61, 61, 61, 61, 25, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 30, 30,+ 0, 5, 5, 15, 61, 61, 5, 20, 25, 61,+ 15, 0, 10, 61, 5, 0, 15, 15, 0, 61,+ 10, 40, 35, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,+ 61, 61, 61, 61, 61, 61, 61+ };+ return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsigned char)str[0]+1];+}++const char *+is_keyword_ruby (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("END")];+ char stringpool_str4[sizeof("redo")];+ char stringpool_str5[sizeof("class")];+ char stringpool_str6[sizeof("unless")];+ char stringpool_str7[sizeof("do")];+ char stringpool_str8[sizeof("end")];+ char stringpool_str9[sizeof("case")];+ char stringpool_str10[sizeof("undef")];+ char stringpool_str11[sizeof("rescue")];+ char stringpool_str12[sizeof("if")];+ char stringpool_str13[sizeof("def")];+ char stringpool_str14[sizeof("else")];+ char stringpool_str15[sizeof("elsif")];+ char stringpool_str16[sizeof("ensure")];+ char stringpool_str17[sizeof("or")];+ char stringpool_str18[sizeof("not")];+ char stringpool_str19[sizeof("next")];+ char stringpool_str20[sizeof("while")];+ char stringpool_str21[sizeof("return")];+ char stringpool_str22[sizeof("in")];+ char stringpool_str23[sizeof("for")];+ char stringpool_str24[sizeof("self")];+ char stringpool_str25[sizeof("super")];+ char stringpool_str26[sizeof("module")];+ char stringpool_str28[sizeof("defined?")];+ char stringpool_str29[sizeof("when")];+ char stringpool_str30[sizeof("until")];+ char stringpool_str33[sizeof("and")];+ char stringpool_str34[sizeof("then")];+ char stringpool_str35[sizeof("alias")];+ char stringpool_str40[sizeof("yield")];+ char stringpool_str44[sizeof("fail")];+ char stringpool_str45[sizeof("retry")];+ char stringpool_str50[sizeof("begin")];+ char stringpool_str55[sizeof("break")];+ char stringpool_str60[sizeof("BEGIN")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "END",+ "redo",+ "class",+ "unless",+ "do",+ "end",+ "case",+ "undef",+ "rescue",+ "if",+ "def",+ "else",+ "elsif",+ "ensure",+ "or",+ "not",+ "next",+ "while",+ "return",+ "in",+ "for",+ "self",+ "super",+ "module",+ "defined?",+ "when",+ "until",+ "and",+ "then",+ "alias",+ "yield",+ "fail",+ "retry",+ "begin",+ "break",+ "BEGIN"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str7,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str10,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str18,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str22,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str23,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str28,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str33,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str60+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_ruby.gperf b/llamafile/is_keyword_ruby.gperfnew file mode 100644index 00000000..c7306862--- /dev/null+++ b/llamafile/is_keyword_ruby.gperf@@ -0,0 +1,45 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_ruby+%%+BEGIN+END+alias+and+begin+break+case+class+def+defined?+do+else+elsif+end+ensure+fail+for+if+in+module+next+not+or+redo+rescue+retry+return+self+super+then+undef+unless+until+when+while+yielddiff --git a/llamafile/is_keyword_ruby_builtin.c b/llamafile/is_keyword_ruby_builtin.cnew file mode 100644index 00000000..b4d25479--- /dev/null+++ b/llamafile/is_keyword_ruby_builtin.c@@ -0,0 +1,369 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_ruby_builtin.gperf */+/* Computed positions: -k'1-2,4,6' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_ruby_builtin.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 67+#define MIN_WORD_LENGTH 1+#define MAX_WORD_LENGTH 20+#define MIN_HASH_VALUE 4+#define MAX_HASH_VALUE 136+/* maximum key range = 133, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 20, 137, 0, 45, 5,+ 25, 0, 85, 0, 55, 35, 137, 0, 5, 5,+ 10, 40, 5, 137, 0, 20, 0, 10, 40, 65,+ 80, 20, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137, 137, 137, 137, 137,+ 137, 137, 137, 137, 137, 137+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[5]];+ /*FALLTHROUGH*/+ case 5:+ case 4:+ hval += asso_values[(unsigned char)str[3]];+ /*FALLTHROUGH*/+ case 3:+ case 2:+ hval += asso_values[(unsigned char)str[1]];+ /*FALLTHROUGH*/+ case 1:+ hval += asso_values[(unsigned char)str[0]];+ break;+ }+ return hval;+}++const char *+is_keyword_ruby_builtin (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str4[sizeof("attr")];+ char stringpool_str6[sizeof("p")];+ char stringpool_str9[sizeof("trap")];+ char stringpool_str11[sizeof("attr_reader")];+ char stringpool_str13[sizeof("attr_accessor")];+ char stringpool_str14[sizeof("proc")];+ char stringpool_str15[sizeof("catch")];+ char stringpool_str16[sizeof("caller")];+ char stringpool_str17[sizeof("require")];+ char stringpool_str19[sizeof("protected")];+ char stringpool_str20[sizeof("print")];+ char stringpool_str21[sizeof("callcc")];+ char stringpool_str24[sizeof("putc")];+ char stringpool_str25[sizeof("raise")];+ char stringpool_str26[sizeof("require_relative")];+ char stringpool_str27[sizeof("prepend")];+ char stringpool_str29[sizeof("rand")];+ char stringpool_str30[sizeof("sleep")];+ char stringpool_str31[sizeof("public")];+ char stringpool_str34[sizeof("trace_var")];+ char stringpool_str35[sizeof("srand")];+ char stringpool_str36[sizeof("untrace_var")];+ char stringpool_str37[sizeof("alias_method")];+ char stringpool_str39[sizeof("puts")];+ char stringpool_str40[sizeof("public_constant")];+ char stringpool_str41[sizeof("refine")];+ char stringpool_str42[sizeof("at_exit")];+ char stringpool_str44[sizeof("public_class_method")];+ char stringpool_str45[sizeof("using")];+ char stringpool_str49[sizeof("eval")];+ char stringpool_str50[sizeof("abort")];+ char stringpool_str51[sizeof("system")];+ char stringpool_str52[sizeof("private")];+ char stringpool_str54[sizeof("loop")];+ char stringpool_str55[sizeof("__callee__")];+ char stringpool_str56[sizeof("lambda")];+ char stringpool_str57[sizeof("syscall")];+ char stringpool_str59[sizeof("open")];+ char stringpool_str61[sizeof("private_constant")];+ char stringpool_str65[sizeof("private_class_method")];+ char stringpool_str67[sizeof("sprintf")];+ char stringpool_str68[sizeof("readline")];+ char stringpool_str69[sizeof("readlines")];+ char stringpool_str70[sizeof("module_function")];+ char stringpool_str71[sizeof("global_variables")];+ char stringpool_str73[sizeof("define_method")];+ char stringpool_str74[sizeof("load")];+ char stringpool_str76[sizeof("attr_writer")];+ char stringpool_str79[sizeof("warn")];+ char stringpool_str80[sizeof("local_variables")];+ char stringpool_str82[sizeof("include")];+ char stringpool_str84[sizeof("exit")];+ char stringpool_str85[sizeof("exit!")];+ char stringpool_str87[sizeof("block_given?")];+ char stringpool_str89[sizeof("exec")];+ char stringpool_str94[sizeof("fail")];+ char stringpool_str95[sizeof("spawn")];+ char stringpool_str98[sizeof("autoload")];+ char stringpool_str99[sizeof("autoload?")];+ char stringpool_str100[sizeof("throw")];+ char stringpool_str102[sizeof("__dir__")];+ char stringpool_str105[sizeof("__method__")];+ char stringpool_str106[sizeof("printf")];+ char stringpool_str111[sizeof("extend")];+ char stringpool_str122[sizeof("binding")];+ char stringpool_str129[sizeof("fork")];+ char stringpool_str136[sizeof("format")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "attr",+ "p",+ "trap",+ "attr_reader",+ "attr_accessor",+ "proc",+ "catch",+ "caller",+ "require",+ "protected",+ "print",+ "callcc",+ "putc",+ "raise",+ "require_relative",+ "prepend",+ "rand",+ "sleep",+ "public",+ "trace_var",+ "srand",+ "untrace_var",+ "alias_method",+ "puts",+ "public_constant",+ "refine",+ "at_exit",+ "public_class_method",+ "using",+ "eval",+ "abort",+ "system",+ "private",+ "loop",+ "__callee__",+ "lambda",+ "syscall",+ "open",+ "private_constant",+ "private_class_method",+ "sprintf",+ "readline",+ "readlines",+ "module_function",+ "global_variables",+ "define_method",+ "load",+ "attr_writer",+ "warn",+ "local_variables",+ "include",+ "exit",+ "exit!",+ "block_given?",+ "exec",+ "fail",+ "spawn",+ "autoload",+ "autoload?",+ "throw",+ "__dir__",+ "__method__",+ "printf",+ "extend",+ "binding",+ "fork",+ "format"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str6,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str9,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str11,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str14,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str15,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str16,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str17,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str19,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str20,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str21,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str24,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str25,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str26,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str27,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str29,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str30,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str31,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str34,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str35,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str36,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str37,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str39,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str40,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str41,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str42,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str44,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str45,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str49,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str50,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str51,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str52,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str54,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str55,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str56,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str57,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str59,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str61,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str65,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str67,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str68,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str69,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str70,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str71,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str73,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str74,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str76,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str79,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str80,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str82,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str84,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str85,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str87,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str89,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str94,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str95,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str98,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str99,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str100,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str102,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str105,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str106,+ -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str111,+ -1, -1, -1, -1, -1, -1, -1, -1, -1,+ -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str122,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str129,+ -1, -1, -1, -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str136+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_ruby_builtin.gperf b/llamafile/is_keyword_ruby_builtin.gperfnew file mode 100644index 00000000..0e6dd677--- /dev/null+++ b/llamafile/is_keyword_ruby_builtin.gperf@@ -0,0 +1,76 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_ruby_builtin+%%+__callee__+__dir__+__method__+abort+alias_method+at_exit+attr+attr_accessor+attr_reader+attr_writer+autoload+autoload?+binding+block_given?+callcc+caller+catch+define_method+eval+exec+exit+exit!+extend+fail+fork+format+global_variables+include+lambda+load+local_variables+loop+module_function+open+p+prepend+printf+private+private_class_method+private_constant+proc+protected+public+public_class_method+public_constant+putc+puts+raise+rand+readline+readlines+refine+require+require_relative+sleep+spawn+sprintf+srand+syscall+system+throw+trace_var+trap+untrace_var+using+warndiff --git a/llamafile/is_keyword_ruby_constant.c b/llamafile/is_keyword_ruby_constant.cnew file mode 100644index 00000000..b5bce6c1--- /dev/null+++ b/llamafile/is_keyword_ruby_constant.c@@ -0,0 +1,137 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_ruby_constant.gperf */+/* Computed positions: -k'3' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_ruby_constant.gperf"++#include <string.h>++#define TOTAL_KEYWORDS 6+#define MIN_WORD_LENGTH 3+#define MAX_WORD_LENGTH 12+#define MIN_HASH_VALUE 3+#define MAX_HASH_VALUE 13+/* maximum key range = 11, duplicates = 0 */++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 0,+ 5, 14, 14, 14, 14, 14, 0, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 0, 14,+ 14, 14, 14, 14, 14, 14, 14, 0, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,+ 14, 14, 14, 14, 14, 14+ };+ return len + asso_values[(unsigned char)str[2]];+}++const char *+is_keyword_ruby_constant (register const char *str, register size_t len)+{+ struct stringpool_t+ {+ char stringpool_str3[sizeof("nil")];+ char stringpool_str4[sizeof("true")];+ char stringpool_str5[sizeof("false")];+ char stringpool_str8[sizeof("__LINE__")];+ char stringpool_str12[sizeof("__ENCODING__")];+ char stringpool_str13[sizeof("__FILE__")];+ };+ static const struct stringpool_t stringpool_contents =+ {+ "nil",+ "true",+ "false",+ "__LINE__",+ "__ENCODING__",+ "__FILE__"+ };+ #define stringpool ((const char *) &stringpool_contents)+ static const int wordlist[] =+ {+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str3,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str4,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str5,+ -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str8,+ -1, -1, -1,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str12,+ (int)(size_t)&((struct stringpool_t *)0)->stringpool_str13+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register int o = wordlist[key];+ if (o >= 0)+ {+ register const char *s = o + stringpool;++ if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0')+ return s;+ }+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_ruby_constant.gperf b/llamafile/is_keyword_ruby_constant.gperfnew file mode 100644index 00000000..66b6cf09--- /dev/null+++ b/llamafile/is_keyword_ruby_constant.gperf@@ -0,0 +1,15 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_ruby_constant+%%+__ENCODING__+__FILE__+__LINE__+false+nil+true