Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Improve Ruby syntax highlighting
Justine Tunney5b0fff1cdbaa
diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 99d7be15..96a638d5 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -543,15 +543,20 @@ class HighlightRuby : public Highlight {void flush(std::string *result) override;private:+ int c_ = 0;+ int u_ = 0;int t_ = 0;int i_ = 0;int level_ = 0;+ int nesti_ = 0;+ int expect_ = 0;unsigned char q_ = 0;- unsigned char last_ = 0;unsigned char opener_ = 0;unsigned char closer_ = 0;+ bool is_definition_ = 0;bool pending_heredoc_ = false;bool indented_heredoc_ = false;+ unsigned char nest_[16];std::string word_;std::string heredoc_;};diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight_js.cppindex af845118..fe2a0a41 100644--- a/llamafile/highlight_js.cpp+++ b/llamafile/highlight_js.cpp@@ -377,5 +377,6 @@ void HighlightJs::flush(std::string *r) {c_ = 0;u_ = 0;t_ = NORMAL;+ expect_ = 0;nesti_ = 0;}diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cppindex 49101efa..73e85639 100644--- a/llamafile/highlight_ruby.cpp+++ b/llamafile/highlight_ruby.cpp@@ -16,7 +16,8 @@// limitations under the License.#include "highlight.h"-+#include "string.h"+#include <cosmo.h>#include <ctype.h>// ruby lexical syntax is bananas and isn't even formally documented@@ -46,6 +47,9 @@ enum {QUOTE,QUOTE_BACKSLASH,DQUOTE,+ DQUOTE_HASH,+ DQUOTE_HASH_DOLLAR,+ DQUOTE_HASH_DOLLAR_WORD,DQUOTE_BACKSLASH,TICK,TICK_BACKSLASH,@@ -68,7 +72,17 @@ enum {MULTICOM,MULTICOM_BOL,REGEX,+ REGEX_HASH,+ REGEX_HASH_DOLLAR,+ REGEX_HASH_DOLLAR_WORD,REGEX_BACKSLASH,+ QUESTION,+ QUESTION_BACKSLASH,+};++enum {+ EXPECT_VALUE,+ EXPECT_OPERATOR,};static int mirror(int c) {@@ -86,6 +100,61 @@ static int mirror(int c) {}}+static bool ispunct_overridable(int c) {+ switch (c) {+ case '%':+ case '&':+ case '*':+ case '+':+ case '-':+ case '/':+ case '<':+ case '>':+ case '^':+ case '_':+ case '`':+ case '|':+ case '~':+ return true;+ default:+ return false;+ }+}++static bool is_dollar_one(int c) {+ switch (c) {+ case '!':+ case '"':+ case '#':+ case '$':+ case '&':+ case '-':+ case '/':+ case '0':+ case '1':+ case '2':+ case '3':+ case '4':+ case '5':+ case '6':+ case '7':+ case '8':+ case '9':+ case '<':+ case '=':+ case '>':+ case '@':+ case '\'':+ case '\\':+ case '^':+ case '_':+ case '`':+ return true;+ default:+ return false;+ }+}+static bool isident(int c) {return !isascii(c) || //isalnum(c) || //@@ -111,69 +180,131 @@ HighlightRuby::~HighlightRuby() {void HighlightRuby::feed(std::string *r, std::string_view input) {int c;for (size_t i = 0; i < input.size(); ++i) {- c = input[i] & 255;-- if (!isblank(c) && c != '/' && c != '<')- last_ = c;-+ wchar_t c;+ int b = input[i] & 255;+ if (!u_) {+ if (b < 0300) {+ c = b;+ } else {+ c_ = ThomPikeByte(b);+ u_ = ThomPikeLen(b) - 1;+ continue;+ }+ } else if (ThomPikeCont(b)) {+ c = c_ = ThomPikeMerge(c_, b);+ if (--u_)+ continue;+ } else {+ u_ = 0;+ c = b;+ }+ if (c == '\r')+ continue;+ if (c == 0xFEFF)+ continue; // utf-8 bomswitch (t_) {Normal:case NORMAL:- if (!isascii(c) || isalpha(c) || c == '_') {+ if (!isascii(c) || isalpha(c) || c == '_' ||+ (is_definition_ && ispunct_overridable(c))) {t_ = WORD;- word_ += c;+ lf::append_wchar(&word_, c);+ is_definition_ = false;} else if (c == ':') {t_ = COLON;+ expect_ = EXPECT_OPERATOR;+ is_definition_ = false;} else if (c == '@') {t_ = AT;+ is_definition_ = false;} else if (c == '=') {t_ = EQUAL;+ expect_ = EXPECT_VALUE;+ is_definition_ = false;+ } else if (c == '?' && expect_ == EXPECT_VALUE) {+ t_ = QUESTION;+ is_definition_ = false;} else if (c == '$') {t_ = DOLLAR;- } else if (c == '%') {+ expect_ = EXPECT_OPERATOR;+ is_definition_ = false;+ } else if (c == '%' && expect_ == EXPECT_VALUE) {t_ = PERCENT;q_ = 0;+ expect_ = EXPECT_OPERATOR;} else if (c == '\'') {t_ = QUOTE;*r += HI_STRING;- *r += c;+ lf::append_wchar(r, c);+ expect_ = EXPECT_OPERATOR;+ is_definition_ = false;} else if (c == '"') {t_ = DQUOTE;*r += HI_STRING;- *r += c;+ lf::append_wchar(r, c);+ expect_ = EXPECT_OPERATOR;+ is_definition_ = false;} else if (c == '`') {t_ = TICK;*r += HI_STRING;- *r += c;+ lf::append_wchar(r, c);+ expect_ = EXPECT_OPERATOR;} else if (c == '#') {*r += HI_COMMENT;- *r += c;+ lf::append_wchar(r, c);t_ = COMMENT;- } else if (c == '<' && !isalnum(last_)) {- *r += c;+ } else if (c == '<' && expect_ == EXPECT_VALUE) {+ lf::append_wchar(r, c);t_ = LT;- } else if (c == '/' && !isalnum(last_)) {+ } else if (c == '/' && expect_ == EXPECT_VALUE) {t_ = REGEX;*r += HI_STRING;- *r += c;+ lf::append_wchar(r, c);+ } else if (c == '{' && nesti_ && nesti_ < sizeof(nest_)) {+ expect_ = EXPECT_VALUE;+ *r += '{';+ nest_[nesti_++] = NORMAL;+ is_definition_ = false;+ } else if (c == '}' && nesti_) {+ if ((t_ = nest_[--nesti_]) != NORMAL)+ *r += HI_STRING;+ *r += '}';+ expect_ = EXPECT_OPERATOR;+ is_definition_ = false;} else if (c == '\n') {- *r += c;+ expect_ = EXPECT_VALUE;+ lf::append_wchar(r, c);if (pending_heredoc_) {*r += HI_STRING;pending_heredoc_ = false;t_ = HEREDOC_BOL;i_ = 0;}+ } else if (c == ']') {+ expect_ = EXPECT_OPERATOR;+ *r += ']';+ is_definition_ = false;+ } else if (ispunct(c)) {+ expect_ = EXPECT_VALUE;+ lf::append_wchar(r, c);+ is_definition_ = false;+ } else if (isdigit(c) || c == '.') {+ expect_ = EXPECT_OPERATOR;+ lf::append_wchar(r, c);+ is_definition_ = false;+ } else if (isspace(c)) {+ lf::append_wchar(r, c);} else {- *r += c;+ lf::append_wchar(r, c);+ is_definition_ = false;}break;case EQUAL:if (!isascii(c) || isalpha(c) || c == '_') {t_ = EQUAL_WORD;- word_ += c;+ lf::append_wchar(&word_, c);} else {*r += '=';t_ = NORMAL;@@ -183,12 +314,12 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case EQUAL_WORD:if (isident(c)) {- word_ += c;+ lf::append_wchar(&word_, c);break;} else if (word_ == "begin") {*r += HI_COMMENT;*r += "=begin";- *r += c;+ lf::append_wchar(r, c);if (c == '\n') {t_ = MULTICOM_BOL;i_ = 0;@@ -205,27 +336,34 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case WORD:if (isident(c)) {- word_ += c;+ lf::append_wchar(&word_, c);} else {if (is_keyword_ruby(word_.data(), word_.size())) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- last_ = 0;+ expect_ = EXPECT_VALUE;+ if (word_ == "def") {+ is_definition_ = true;+ }} else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;+ expect_ = EXPECT_VALUE;} else if (is_keyword_ruby_constant(word_.data(), word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;+ expect_ = EXPECT_OPERATOR;} else if (!word_.empty() && isupper(word_[0])) {*r += HI_CLASS;*r += word_;*r += HI_RESET;+ expect_ = EXPECT_OPERATOR;} else {*r += word_;+ expect_ = EXPECT_OPERATOR;}word_.clear();t_ = NORMAL;@@ -233,23 +371,81 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {}break;+ Regex:case REGEX:- *r += c;if (c == '/') {+ lf::append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;+ } else if (c == '#') {+ t_ = REGEX_HASH;} else if (c == '\\') {+ lf::append_wchar(r, c);t_ = REGEX_BACKSLASH;+ } else {+ lf::append_wchar(r, c);+ }+ break;++ case REGEX_HASH:+ if (c == '{' && nesti_ < sizeof(nest_)) {+ *r += HI_BOLD;+ *r += '#';+ *r += HI_UNBOLD;+ *r += HI_STRING;+ *r += '{';+ *r += HI_RESET;+ expect_ = EXPECT_VALUE;+ nest_[nesti_++] = REGEX;+ t_ = NORMAL;+ } else if (c == '$') {+ t_ = REGEX_HASH_DOLLAR;+ } else {+ *r += '#';+ t_ = REGEX;+ goto Regex;+ }+ break;++ case REGEX_HASH_DOLLAR:+ if (is_dollar_one(c)) {+ *r += '#';+ *r += HI_BOLD;+ *r += '$';+ lf::append_wchar(r, c);+ *r += HI_UNBOLD;+ t_ = REGEX;+ } else if (isalpha(c)) {+ *r += '#';+ *r += HI_BOLD;+ *r += '$';+ lf::append_wchar(r, c);+ t_ = REGEX_HASH_DOLLAR_WORD;+ } else {+ *r += '#';+ *r += '$';+ t_ = REGEX;+ goto Regex;+ }+ break;++ case REGEX_HASH_DOLLAR_WORD:+ if (isident(c)) {+ lf::append_wchar(r, c);+ } else {+ *r += HI_UNBOLD;+ t_ = REGEX;+ goto Regex;}break;case REGEX_BACKSLASH:- *r += c;+ lf::append_wchar(r, c);t_ = REGEX;break;case MULTICOM:- *r += c;+ lf::append_wchar(r, c);if (c == '\n') {t_ = MULTICOM_BOL;i_ = 0;@@ -257,7 +453,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case MULTICOM_BOL:- *r += c;+ lf::append_wchar(r, c);if (c == "=end"[i_]) {if (++i_ == 4) {t_ = NORMAL;@@ -272,7 +468,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {if (isident(c)) {*r += HI_LISPKW;*r += ':';- *r += c;+ lf::append_wchar(r, c);t_ = COLON_WORD;} else {*r += ':';@@ -283,7 +479,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case COLON_WORD:if (isident(c)) {- *r += c;+ lf::append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -295,7 +491,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {if (isident(c)) {*r += HI_VAR;*r += '@';- *r += c;+ lf::append_wchar(r, c);t_ = AT_WORD;} else {*r += '@';@@ -306,7 +502,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case AT_WORD:if (isident(c)) {- *r += c;+ lf::append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -324,7 +520,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {closer_ = mirror(c);*r += HI_STRING;*r += '%';- *r += c;+ lf::append_wchar(r, c);t_ = PERCENT_STRING;} else {*r += '%';@@ -341,7 +537,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {*r += HI_STRING;*r += '%';*r += q_;- *r += c;+ lf::append_wchar(r, c);t_ = PERCENT_STRING;} else {*r += '%';@@ -352,7 +548,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case PERCENT_STRING:- *r += c;+ lf::append_wchar(r, c);if (c == opener_ && opener_ != closer_) {++level_;} else if (c == closer_) {@@ -364,32 +560,16 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case DOLLAR:- if (isdigit(c) || //- c == '!' || //- c == '"' || //- c == '#' || //- c == '$' || //- c == '&' || //- c == '-' || //- c == '/' || //- c == '<' || //- c == '=' || //- c == '>' || //- c == '@' || //- c == '\'' || //- c == '\\' || //- c == '^' || //- c == '_' || //- c == '`') {+ if (is_dollar_one(c)) {*r += HI_VAR;*r += '$';- *r += c;+ lf::append_wchar(r, c);*r += HI_RESET;t_ = NORMAL;} else if (isalpha(c)) {*r += HI_VAR;*r += '$';- *r += c;+ lf::append_wchar(r, c);t_ = DOLLAR_WORD;} else {*r += '$';@@ -400,7 +580,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case DOLLAR_WORD:if (isident(c)) {- *r += c;+ lf::append_wchar(r, c);} else {*r += HI_RESET;t_ = NORMAL;@@ -409,7 +589,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case COMMENT:- *r += c;+ lf::append_wchar(r, c);if (c == '\n') {*r += HI_RESET;t_ = NORMAL;@@ -417,7 +597,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case QUOTE:- *r += c;+ lf::append_wchar(r, c);if (c == '\'') {*r += HI_RESET;t_ = NORMAL;@@ -427,27 +607,85 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case QUOTE_BACKSLASH:- *r += c;+ lf::append_wchar(r, c);t_ = QUOTE;break;+ Dquote:case DQUOTE:- *r += c;if (c == '"') {+ *r += '"';*r += HI_RESET;t_ = NORMAL;+ } else if (c == '#') {+ t_ = DQUOTE_HASH;} else if (c == '\\') {t_ = DQUOTE_BACKSLASH;+ *r += '\\';+ } else {+ lf::append_wchar(r, c);+ }+ break;++ case DQUOTE_HASH:+ if (c == '{' && nesti_ < sizeof(nest_)) {+ *r += HI_BOLD;+ *r += '#';+ *r += HI_UNBOLD;+ *r += HI_STRING;+ *r += '{';+ *r += HI_RESET;+ expect_ = EXPECT_VALUE;+ nest_[nesti_++] = DQUOTE;+ t_ = NORMAL;+ } else if (c == '$') {+ t_ = DQUOTE_HASH_DOLLAR;+ } else {+ *r += '#';+ t_ = DQUOTE;+ goto Dquote;}break;case DQUOTE_BACKSLASH:- *r += c;+ lf::append_wchar(r, c);t_ = DQUOTE;break;+ case DQUOTE_HASH_DOLLAR:+ if (is_dollar_one(c)) {+ *r += '#';+ *r += HI_BOLD;+ *r += '$';+ lf::append_wchar(r, c);+ *r += HI_UNBOLD;+ t_ = DQUOTE;+ } else if (isalpha(c)) {+ *r += '#';+ *r += HI_BOLD;+ *r += '$';+ lf::append_wchar(r, c);+ t_ = DQUOTE_HASH_DOLLAR_WORD;+ } else {+ *r += '#';+ *r += '$';+ t_ = DQUOTE;+ goto Dquote;+ }+ break;++ case DQUOTE_HASH_DOLLAR_WORD:+ if (isident(c)) {+ lf::append_wchar(r, c);+ } else {+ *r += HI_UNBOLD;+ t_ = DQUOTE;+ goto Dquote;+ }+ break;+case TICK:- *r += c;+ lf::append_wchar(r, c);if (c == '`') {*r += HI_RESET;t_ = NORMAL;@@ -457,13 +695,13 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case TICK_BACKSLASH:- *r += c;+ lf::append_wchar(r, c);t_ = TICK;break;case LT:if (c == '<') {- *r += c;+ lf::append_wchar(r, c);t_ = LT_LT;heredoc_.clear();pending_heredoc_ = false;@@ -477,16 +715,16 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {case LT_LT:if (c == '-') {indented_heredoc_ = true;- *r += c;+ lf::append_wchar(r, c);} else if (c == '\'' || c == '`' || c == '"') {closer_ = c;t_ = LT_LT_QNAME;*r += HI_STRING;- *r += c;+ lf::append_wchar(r, c);} else if (isalpha(c) || c == '_') {t_ = LT_LT_NAME;heredoc_ += c;- *r += c;+ lf::append_wchar(r, c);} else {t_ = NORMAL;goto Normal;@@ -497,9 +735,9 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {if (isalnum(c) || c == '_') {t_ = LT_LT_NAME;heredoc_ += c;- *r += c;+ lf::append_wchar(r, c);} else if (c == '\n') {- *r += c;+ lf::append_wchar(r, c);*r += HI_STRING;t_ = HEREDOC_BOL;} else {@@ -510,7 +748,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case LT_LT_QNAME:- *r += c;+ lf::append_wchar(r, c);if (c == closer_) {*r += HI_RESET;t_ = HEREDOC_BOL;@@ -522,7 +760,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case HEREDOC_BOL:- *r += c;+ lf::append_wchar(r, c);if (c == '\n') {if (i_ == heredoc_.size()) {t_ = NORMAL;@@ -540,11 +778,35 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {break;case HEREDOC:- *r += c;+ lf::append_wchar(r, c);if (c == '\n')t_ = HEREDOC_BOL;break;+ case QUESTION:+ if (c == '\\') {+ t_ = QUESTION_BACKSLASH;+ } else if (isspace(c)) {+ *r += '?';+ t_ = NORMAL;+ goto Normal;+ } else {+ *r += HI_ESCAPE;+ *r += '?';+ lf::append_wchar(r, c);+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case QUESTION_BACKSLASH:+ *r += HI_ESCAPE;+ *r += "?\\";+ lf::append_wchar(r, c);+ *r += HI_RESET;+ t_ = NORMAL;+ break;+default:__builtin_unreachable();}@@ -597,8 +859,31 @@ void HighlightRuby::flush(std::string *r) {*r += '%';*r += q_;break;+ case QUESTION:+ *r += '?';+ break;+ case QUESTION_BACKSLASH:+ *r += "?\\";+ break;+ case DQUOTE_HASH:+ *r += '#';+ *r += HI_RESET;+ break;+ case DQUOTE_HASH_DOLLAR:+ *r += "#$";+ *r += HI_RESET;+ break;+ case REGEX_HASH:+ *r += '#';+ *r += HI_RESET;+ break;+ case REGEX_HASH_DOLLAR:+ *r += "#$";+ *r += HI_RESET;+ break;case REGEX:case REGEX_BACKSLASH:+ case REGEX_HASH_DOLLAR_WORD:case PERCENT_STRING:case AT_WORD:case DOLLAR_WORD:@@ -608,6 +893,7 @@ void HighlightRuby::flush(std::string *r) {case QUOTE_BACKSLASH:case DQUOTE:case DQUOTE_BACKSLASH:+ case DQUOTE_HASH_DOLLAR_WORD:case COMMENT:case HEREDOC_BOL:case HEREDOC:@@ -620,5 +906,10 @@ void HighlightRuby::flush(std::string *r) {default:break;}+ c_ = 0;+ u_ = 0;t_ = NORMAL;+ is_definition_ = 0;+ expect_ = EXPECT_VALUE;+ nesti_ = 0;}diff --git a/llamafile/server/www/ctype.js b/llamafile/server/www/ctype.jsindex 6a50ffb3..a26a5999 100644--- a/llamafile/server/www/ctype.js+++ b/llamafile/server/www/ctype.js@@ -92,7 +92,6 @@ function ispunct(c) {case '|':case '}':case '~':- case '}':return true;default:return false;diff --git a/llamafile/server/www/highlight_markdown.js b/llamafile/server/www/highlight_markdown.jsindex 9ca50ad7..8f28e986 100644--- a/llamafile/server/www/highlight_markdown.js+++ b/llamafile/server/www/highlight_markdown.js@@ -45,6 +45,32 @@ class HighlightMarkdown extends Highlighter {static STYLE_EMPHASIS = 1;static STYLE_LIST = 2;+ static is_escapable(c) {+ switch (c) {+ case '!':+ case '#':+ case '(':+ case ')':+ case '*':+ case '+':+ case '-':+ case '.':+ case '<':+ case '>':+ case '[':+ case '\\':+ case ']':+ case '_':+ case '`':+ case '{':+ case '|':+ case '}':+ return true;+ default:+ return false;+ }+ }+constructor(delegate) {super(delegate);this.bol = true;@@ -150,6 +176,7 @@ class HighlightMarkdown extends Highlighter {case HighlightMarkdown.HYPHEN:if (isblank(c)) {+ // - handle list itemthis.li('-');this.append(c);this.state = HighlightMarkdown.NORMAL;@@ -160,13 +187,19 @@ class HighlightMarkdown extends Highlighter {break;case HighlightMarkdown.BACKSLASH:- this.append(c);- this.state = HighlightMarkdown.NORMAL;+ if (HighlightMarkdown.is_escapable(c)) {+ this.append(c);+ this.state = HighlightMarkdown.NORMAL;+ } else {+ this.append('\\');+ this.epsilon(HighlightMarkdown.NORMAL);+ }break;case HighlightMarkdown.STAR:if (c == '*') {// handle **strong** text+ // we don't call this.got() in case this is *** barthis.push('strong', '');this.state = HighlightMarkdown.NORMAL;this.style.push(HighlightMarkdown.STYLE_STRONG);@@ -360,6 +393,8 @@ class HighlightMarkdown extends Highlighter {break;case HighlightMarkdown.LAB_BACKSLASH:+ if (!HighlightMarkdown.is_escapable(c))+ this.href += '\\';this.href += c;this.state = HighlightMarkdown.LAB;break;@@ -375,6 +410,8 @@ class HighlightMarkdown extends Highlighter {break;case HighlightMarkdown.LSB_BACKSLASH:+ if (!HighlightMarkdown.is_escapable(c))+ this.text += '\\';this.text += c;this.state = HighlightMarkdown.LSB;break;@@ -406,6 +443,8 @@ class HighlightMarkdown extends Highlighter {break;case HighlightMarkdown.LSB_RSB_LPAREN_BACKSLASH:+ if (!HighlightMarkdown.is_escapable(c))+ this.href += '\\';this.href += c;this.state = HighlightMarkdown.LSB_RSB_LPAREN;break;@@ -492,12 +531,29 @@ class HighlightMarkdown extends Highlighter {setupCodeBlock(pre) {const copyButton = document.createElement('button');copyButton.className = 'copy-button';- copyButton.innerHTML = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path></svg>`;+ copyButton.innerHTML =+ `<svg xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24" fill="none"+ stroke="currentColor"+ stroke-width="2"+ stroke-linecap="round"+ stroke-linejoin="round">+ <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>+ <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>+ </svg>`;copyButton.addEventListener('click', function() {try {copyTextToClipboard(pre.innerText);const originalInnerHTML = copyButton.innerHTML;- copyButton.innerHTML = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="green" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"></polyline></svg>`;+ copyButton.innerHTML =+ `<svg xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24" fill="none"+ stroke="green"+ stroke-width="2"+ stroke-linecap="round"+ stroke-linejoin="round">+ <polyline points="20 6 9 17 4 12"></polyline>+ </svg>`;setTimeout(() => {copyButton.innerHTML = originalInnerHTML;}, 2000);diff --git a/llamafile/server/www/highlight_ruby.js b/llamafile/server/www/highlight_ruby.jsindex 79b87de6..84fd5c48 100644--- a/llamafile/server/www/highlight_ruby.js+++ b/llamafile/server/www/highlight_ruby.js@@ -162,6 +162,18 @@ class HighlightRuby extends Highlighter {static MULTICOM_BOL = 27;static REGEX = 28;static REGEX_BACKSLASH = 29;+ static DQUOTE_HASH = 30;+ static QUESTION = 31;+ static QUESTION_BACKSLASH = 32;+ static REGEX_HASH = 33;+ static REGEX_HASH_DOLLAR = 34;+ static REGEX_HASH_DOLLAR_WORD = 35;+ static DQUOTE_HASH = 36;+ static DQUOTE_HASH_DOLLAR = 37;+ static DQUOTE_HASH_DOLLAR_WORD = 38;++ static EXPECT_VALUE = 0;+ static EXPECT_OPERATOR = 1;static mirror(c) {switch (c) {@@ -178,6 +190,61 @@ class HighlightRuby extends Highlighter {}}+ static ispunct_overridable(c) {+ switch (c) {+ case '%':+ case '&':+ case '*':+ case '+':+ case '-':+ case '/':+ case '<':+ case '>':+ case '^':+ case '_':+ case '`':+ case '|':+ case '~':+ return true;+ default:+ return false;+ }+ }++ static is_dollar_one(c) {+ switch (c) {+ case '!':+ case '"':+ case '#':+ case '$':+ case '&':+ case '-':+ case '/':+ case '0':+ case '1':+ case '2':+ case '3':+ case '4':+ case '5':+ case '6':+ case '7':+ case '8':+ case '9':+ case '<':+ case '=':+ case '>':+ case '@':+ case '\'':+ case '\\':+ case '^':+ case '_':+ case '`':+ return true;+ default:+ return false;+ }+ }+static isident(c) {return !isascii(c) ||isalnum(c) ||@@ -198,7 +265,6 @@ class HighlightRuby extends Highlighter {super(delegate);this.i = 0;this.q = 0;- this.last = 0;this.level = 0;this.opener = 0;this.closer = 0;@@ -206,57 +272,86 @@ class HighlightRuby extends Highlighter {this.indented_heredoc = false;this.heredoc = '';this.word = '';+ this.nest = [];+ this.is_definition = false;+ this.expect = HighlightRuby.EXPECT_VALUE;}feed(input) {for (let i = 0; i < input.length; i += this.delta) {this.delta = 1;let c = input[i];-- if (!isblank(c) && c != '/' && c != '<')- this.last = c;-switch (this.state) {case HighlightRuby.NORMAL:- if (!isascii(c) || isalpha(c) || c == '_') {+ if (!isascii(c) || isalpha(c) || c == '_' ||+ (this.is_definition && HighlightRuby.ispunct_overridable(c))) {this.state = HighlightRuby.WORD;this.word += c;+ this.is_definition = false;} else if (c == ':') {this.state = HighlightRuby.COLON;+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.is_definition = false;} else if (c == '@') {this.state = HighlightRuby.AT;+ this.is_definition = false;} else if (c == '=') {this.state = HighlightRuby.EQUAL;+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.is_definition = false;+ } else if (c == '?' && this.expect == HighlightRuby.EXPECT_VALUE) {+ this.state = HighlightRuby.QUESTION;+ this.is_definition = false;} else if (c == '$') {this.state = HighlightRuby.DOLLAR;- } else if (c == '%') {+ this.expect = HighlightRuby.EXPECT_OPERATOR;+ this.is_definition = false;+ } else if (c == '%' && this.expect == HighlightRuby.EXPECT_VALUE) {this.state = HighlightRuby.PERCENT;+ this.expect = HighlightRuby.EXPECT_OPERATOR;this.q = 0;} else if (c == '\'') {this.state = HighlightRuby.QUOTE;this.push("span", "string");this.append(c);+ this.expect = HighlightRuby.EXPECT_OPERATOR;+ this.is_definition = false;} else if (c == '"') {this.state = HighlightRuby.DQUOTE;+ this.expect = HighlightRuby.EXPECT_OPERATOR;this.push("span", "string");this.append(c);+ this.is_definition = false;} else if (c == '`') {this.state = HighlightRuby.TICK;+ this.expect = HighlightRuby.EXPECT_OPERATOR;this.push("span", "string");this.append(c);} else if (c == '#') {this.push("span", "comment");this.append(c);this.state = HighlightRuby.COMMENT;- } else if (c == '<' && !isalnum(this.last)) {+ } else if (c == '<' && this.expect == HighlightRuby.EXPECT_VALUE) {this.append(c);this.state = HighlightRuby.LT;- } else if (c == '/' && !isalnum(this.last)) {+ } else if (c == '/' && this.expect == HighlightRuby.EXPECT_VALUE) {this.state = HighlightRuby.REGEX;this.push("span", "string");this.append(c);+ } else if (c == '{' && this.nest.length) {+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.append('{');+ this.nest.push(HighlightRuby.NORMAL);+ this.is_definition = false;+ } else if (c == '}' && this.nest.length) {+ if ((this.state = this.nest.pop()) != HighlightRuby.NORMAL)+ this.push("span", "string");+ this.append('}');+ this.expect = HighlightRuby.EXPECT_OPERATOR;+ this.is_definition = false;} else if (c == '\n') {+ this.expect = HighlightRuby.EXPECT_VALUE;this.append(c);if (this.pending_heredoc) {this.push("span", "string");@@ -264,8 +359,23 @@ class HighlightRuby extends Highlighter {this.state = HighlightRuby.HEREDOC_BOL;this.i = 0;}+ } else if (c == ']') {+ this.expect = HighlightRuby.EXPECT_OPERATOR;+ this.append(']');+ this.is_definition = false;+ } else if (ispunct(c)) {+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.append(c);+ this.is_definition = false;+ } else if (isdigit(c) || c == '.') {+ this.expect = HighlightRuby.EXPECT_OPERATOR;+ this.append(c);+ this.is_definition = false;+ } else if (isspace(c)) {+ this.append(c);} else {this.append(c);+ this.is_definition = false;}break;@@ -309,21 +419,28 @@ class HighlightRuby extends Highlighter {this.push("span", "keyword");this.append(this.word);this.pop();- this.last = 0;+ this.expect = HighlightRuby.EXPECT_VALUE;+ if (this.word == 'def') {+ this.is_definition = true;+ }} else if (RUBY_BUILTINS.has(this.word)) {this.push("span", "builtin");this.append(this.word);this.pop();+ this.expect = HighlightRuby.EXPECT_VALUE;} else if (RUBY_CONSTANTS.has(this.word)) {this.push("span", "constant");this.append(this.word);this.pop();+ this.expect = HighlightRuby.EXPECT_OPERATOR;} else if (this.word && this.word[0] == this.word[0].toUpperCase()) {this.push("span", "class");this.append(this.word);this.pop();+ this.expect = HighlightRuby.EXPECT_OPERATOR;} else {this.append(this.word);+ this.expect = HighlightRuby.EXPECT_OPERATOR;}this.word = '';this.epsilon(HighlightRuby.NORMAL);@@ -331,12 +448,65 @@ class HighlightRuby extends Highlighter {break;case HighlightRuby.REGEX:- this.append(c);if (c == '/') {+ this.append(c);this.pop();this.state = HighlightRuby.NORMAL;+ } else if (c == '#') {+ this.state = HighlightRuby.REGEX_HASH;} else if (c == '\\') {+ this.append(c);this.state = HighlightRuby.REGEX_BACKSLASH;+ } else {+ this.append(c);+ }+ break;++ case HighlightRuby.REGEX_HASH:+ if (c == '{') {+ this.push("strong", "");+ this.append('#');+ this.pop();+ this.append('{');+ this.pop();+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.nest.push(HighlightRuby.REGEX);+ this.state = HighlightRuby.NORMAL;+ } else if (c == '$') {+ this.state = HighlightRuby.REGEX_HASH_DOLLAR;+ } else {+ this.append('#');+ this.epsilon(HighlightRuby.REGEX);+ }+ break;++ case HighlightRuby.REGEX_HASH_DOLLAR:+ if (HighlightRuby.is_dollar_one(c)) {+ this.append('#');+ this.push("strong", "");+ this.append('$');+ this.append(c);+ this.pop();+ this.state = HighlightRuby.REGEX;+ } else if (isalpha(c)) {+ this.append('#');+ this.push("strong", "");+ this.append('$');+ this.append(c);+ this.state = HighlightRuby.REGEX_HASH_DOLLAR_WORD;+ } else {+ this.append('#');+ this.append('$');+ this.epsilon(HighlightRuby.REGEX);+ }+ break;++ case HighlightRuby.REGEX_HASH_DOLLAR_WORD:+ if (HighlightRuby.isident(c)) {+ this.append(c);+ } else {+ this.pop();+ this.epsilon(HighlightRuby.REGEX);}break;@@ -455,23 +625,7 @@ class HighlightRuby extends Highlighter {break;case HighlightRuby.DOLLAR:- if (isdigit(c) || //- c == '!' || //- c == '"' || //- c == '#' || //- c == '$' || //- c == '&' || //- c == '-' || //- c == '/' || //- c == '<' || //- c == '=' || //- c == '>' || //- c == '@' || //- c == '\'' || //- c == '\\' || //- c == '^' || //- c == '_' || //- c == '`') {+ if (HighlightRuby.is_dollar_one(c)) {this.push("span", "var");this.append('$');this.append(c);@@ -521,12 +675,65 @@ class HighlightRuby extends Highlighter {break;case HighlightRuby.DQUOTE:- this.append(c);if (c == '"') {+ this.append(c);this.pop();this.state = HighlightRuby.NORMAL;+ } else if (c == '#') {+ this.state = HighlightRuby.DQUOTE_HASH;} else if (c == '\\') {this.state = HighlightRuby.DQUOTE_BACKSLASH;+ this.append(c);+ } else {+ this.append(c);+ }+ break;++ case HighlightRuby.DQUOTE_HASH:+ if (c == '{') {+ this.push("strong", "");+ this.append('#');+ this.pop();+ this.append('{');+ this.pop();+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.nest.push(HighlightRuby.DQUOTE);+ this.state = HighlightRuby.NORMAL;+ } else if (c == '$') {+ this.state = HighlightRuby.DQUOTE_HASH_DOLLAR;+ } else {+ this.append('#');+ this.epsilon(HighlightRuby.DQUOTE);+ }+ break;++ case HighlightRuby.DQUOTE_HASH_DOLLAR:+ if (HighlightRuby.is_dollar_one(c)) {+ this.append('#');+ this.push("strong", "");+ this.append('$');+ this.append(c);+ this.pop();+ this.state = HighlightRuby.DQUOTE;+ } else if (isalpha(c)) {+ this.append('#');+ this.push("strong", "");+ this.append('$');+ this.append(c);+ this.state = HighlightRuby.DQUOTE_HASH_DOLLAR_WORD;+ } else {+ this.append('#');+ this.append('$');+ this.epsilon(HighlightRuby.DQUOTE);+ }+ break;++ case HighlightRuby.DQUOTE_HASH_DOLLAR_WORD:+ if (HighlightRuby.isident(c)) {+ this.append(c);+ } else {+ this.pop();+ this.epsilon(HighlightRuby.DQUOTE);}break;@@ -631,6 +838,29 @@ class HighlightRuby extends Highlighter {this.state = HighlightRuby.HEREDOC_BOL;break;+ case HighlightRuby.QUESTION:+ if (c == '\\') {+ this.state = HighlightRuby.QUESTION_BACKSLASH;+ } else if (isspace(c)) {+ this.append('?');+ this.epsilon(HighlightRuby.NORMAL);+ } else {+ this.push('span', 'escape');+ this.append('?');+ this.append(c);+ this.pop();+ this.state = HighlightRuby.NORMAL;+ }+ break;++ case HighlightRuby.QUESTION_BACKSLASH:+ this.push('span', 'escape');+ this.append("?\\");+ this.append(c);+ this.pop();+ this.state = HighlightRuby.NORMAL;+ break;+default:throw new Error('Invalid state');}@@ -683,8 +913,31 @@ class HighlightRuby extends Highlighter {this.append('%');this.append(this.q);break;+ case HighlightRuby.QUESTION:+ this.append('?');+ break;+ case HighlightRuby.QUESTION_BACKSLASH:+ this.append("?\\");+ break;+ case HighlightRuby.DQUOTE_HASH:+ this.append('#');+ this.pop();+ break;+ case HighlightRuby.DQUOTE_HASH_DOLLAR:+ this.append("#$");+ this.pop();+ break;+ case HighlightRuby.REGEX_HASH:+ this.append('#');+ this.pop();+ break;+ case HighlightRuby.REGEX_HASH_DOLLAR:+ this.append("#$");+ this.pop();+ break;case HighlightRuby.REGEX:case HighlightRuby.REGEX_BACKSLASH:+ case HighlightRuby.REGEX_HASH_DOLLAR_WORD:case HighlightRuby.PERCENT_STRING:case HighlightRuby.AT_WORD:case HighlightRuby.DOLLAR_WORD:@@ -694,6 +947,7 @@ class HighlightRuby extends Highlighter {case HighlightRuby.QUOTE_BACKSLASH:case HighlightRuby.DQUOTE:case HighlightRuby.DQUOTE_BACKSLASH:+ case HighlightRuby.DQUOTE_HASH_DOLLAR_WORD:case HighlightRuby.COMMENT:case HighlightRuby.HEREDOC_BOL:case HighlightRuby.HEREDOC:@@ -706,6 +960,9 @@ class HighlightRuby extends Highlighter {default:break;}+ this.nest = [];+ this.expect = HighlightRuby.EXPECT_VALUE;+ this.is_definition = false;this.state = HighlightRuby.NORMAL;this.delegate.flush();this.delta = 1;