Skip to content

Distribute and run LLMs with a single file.

Improve Ruby syntax highlighting

Justine Tunney5b0fff1cdbaa
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 99d7be15..96a638d5 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -543,15 +543,20 @@ class HighlightRuby : public Highlight {
void flush(std::string *result) override;
private:
+ int c_ = 0;
+ int u_ = 0;
int t_ = 0;
int i_ = 0;
int level_ = 0;
+ int nesti_ = 0;
+ int expect_ = 0;
unsigned char q_ = 0;
- unsigned char last_ = 0;
unsigned char opener_ = 0;
unsigned char closer_ = 0;
+ bool is_definition_ = 0;
bool pending_heredoc_ = false;
bool indented_heredoc_ = false;
+ unsigned char nest_[16];
std::string word_;
std::string heredoc_;
};
diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight_js.cpp
index af845118..fe2a0a41 100644
--- a/llamafile/highlight_js.cpp
+++ b/llamafile/highlight_js.cpp
@@ -377,5 +377,6 @@ void HighlightJs::flush(std::string *r) {
c_ = 0;
u_ = 0;
t_ = NORMAL;
+ expect_ = 0;
nesti_ = 0;
}
diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight_ruby.cpp
index 49101efa..73e85639 100644
--- a/llamafile/highlight_ruby.cpp
+++ b/llamafile/highlight_ruby.cpp
@@ -16,7 +16,8 @@
// limitations under the License.
#include "highlight.h"
-
+#include "string.h"
+#include <cosmo.h>
#include <ctype.h>
// ruby lexical syntax is bananas and isn't even formally documented
@@ -46,6 +47,9 @@ enum {
QUOTE,
QUOTE_BACKSLASH,
DQUOTE,
+ DQUOTE_HASH,
+ DQUOTE_HASH_DOLLAR,
+ DQUOTE_HASH_DOLLAR_WORD,
DQUOTE_BACKSLASH,
TICK,
TICK_BACKSLASH,
@@ -68,7 +72,17 @@ enum {
MULTICOM,
MULTICOM_BOL,
REGEX,
+ REGEX_HASH,
+ REGEX_HASH_DOLLAR,
+ REGEX_HASH_DOLLAR_WORD,
REGEX_BACKSLASH,
+ QUESTION,
+ QUESTION_BACKSLASH,
+};
+
+enum {
+ EXPECT_VALUE,
+ EXPECT_OPERATOR,
};
static int mirror(int c) {
@@ -86,6 +100,61 @@ static int mirror(int c) {
}
}
+static bool ispunct_overridable(int c) {
+ switch (c) {
+ case '%':
+ case '&':
+ case '*':
+ case '+':
+ case '-':
+ case '/':
+ case '<':
+ case '>':
+ case '^':
+ case '_':
+ case '`':
+ case '|':
+ case '~':
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool is_dollar_one(int c) {
+ switch (c) {
+ case '!':
+ case '"':
+ case '#':
+ case '$':
+ case '&':
+ case '-':
+ case '/':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case '<':
+ case '=':
+ case '>':
+ case '@':
+ case '\'':
+ case '\\':
+ case '^':
+ case '_':
+ case '`':
+ return true;
+ default:
+ return false;
+ }
+}
+
static bool isident(int c) {
return !isascii(c) || //
isalnum(c) || //
@@ -111,69 +180,131 @@ HighlightRuby::~HighlightRuby() {
void HighlightRuby::feed(std::string *r, std::string_view input) {
int c;
for (size_t i = 0; i < input.size(); ++i) {
- c = input[i] & 255;
-
- if (!isblank(c) && c != '/' && c != '<')
- last_ = c;
-
+ wchar_t c;
+ int b = input[i] & 255;
+ if (!u_) {
+ if (b < 0300) {
+ c = b;
+ } else {
+ c_ = ThomPikeByte(b);
+ u_ = ThomPikeLen(b) - 1;
+ continue;
+ }
+ } else if (ThomPikeCont(b)) {
+ c = c_ = ThomPikeMerge(c_, b);
+ if (--u_)
+ continue;
+ } else {
+ u_ = 0;
+ c = b;
+ }
+ if (c == '\r')
+ continue;
+ if (c == 0xFEFF)
+ continue; // utf-8 bom
switch (t_) {
Normal:
case NORMAL:
- if (!isascii(c) || isalpha(c) || c == '_') {
+ if (!isascii(c) || isalpha(c) || c == '_' ||
+ (is_definition_ && ispunct_overridable(c))) {
t_ = WORD;
- word_ += c;
+ lf::append_wchar(&word_, c);
+ is_definition_ = false;
} else if (c == ':') {
t_ = COLON;
+ expect_ = EXPECT_OPERATOR;
+ is_definition_ = false;
} else if (c == '@') {
t_ = AT;
+ is_definition_ = false;
} else if (c == '=') {
t_ = EQUAL;
+ expect_ = EXPECT_VALUE;
+ is_definition_ = false;
+ } else if (c == '?' && expect_ == EXPECT_VALUE) {
+ t_ = QUESTION;
+ is_definition_ = false;
} else if (c == '$') {
t_ = DOLLAR;
- } else if (c == '%') {
+ expect_ = EXPECT_OPERATOR;
+ is_definition_ = false;
+ } else if (c == '%' && expect_ == EXPECT_VALUE) {
t_ = PERCENT;
q_ = 0;
+ expect_ = EXPECT_OPERATOR;
} else if (c == '\'') {
t_ = QUOTE;
*r += HI_STRING;
- *r += c;
+ lf::append_wchar(r, c);
+ expect_ = EXPECT_OPERATOR;
+ is_definition_ = false;
} else if (c == '"') {
t_ = DQUOTE;
*r += HI_STRING;
- *r += c;
+ lf::append_wchar(r, c);
+ expect_ = EXPECT_OPERATOR;
+ is_definition_ = false;
} else if (c == '`') {
t_ = TICK;
*r += HI_STRING;
- *r += c;
+ lf::append_wchar(r, c);
+ expect_ = EXPECT_OPERATOR;
} else if (c == '#') {
*r += HI_COMMENT;
- *r += c;
+ lf::append_wchar(r, c);
t_ = COMMENT;
- } else if (c == '<' && !isalnum(last_)) {
- *r += c;
+ } else if (c == '<' && expect_ == EXPECT_VALUE) {
+ lf::append_wchar(r, c);
t_ = LT;
- } else if (c == '/' && !isalnum(last_)) {
+ } else if (c == '/' && expect_ == EXPECT_VALUE) {
t_ = REGEX;
*r += HI_STRING;
- *r += c;
+ lf::append_wchar(r, c);
+ } else if (c == '{' && nesti_ && nesti_ < sizeof(nest_)) {
+ expect_ = EXPECT_VALUE;
+ *r += '{';
+ nest_[nesti_++] = NORMAL;
+ is_definition_ = false;
+ } else if (c == '}' && nesti_) {
+ if ((t_ = nest_[--nesti_]) != NORMAL)
+ *r += HI_STRING;
+ *r += '}';
+ expect_ = EXPECT_OPERATOR;
+ is_definition_ = false;
} else if (c == '\n') {
- *r += c;
+ expect_ = EXPECT_VALUE;
+ lf::append_wchar(r, c);
if (pending_heredoc_) {
*r += HI_STRING;
pending_heredoc_ = false;
t_ = HEREDOC_BOL;
i_ = 0;
}
+ } else if (c == ']') {
+ expect_ = EXPECT_OPERATOR;
+ *r += ']';
+ is_definition_ = false;
+ } else if (ispunct(c)) {
+ expect_ = EXPECT_VALUE;
+ lf::append_wchar(r, c);
+ is_definition_ = false;
+ } else if (isdigit(c) || c == '.') {
+ expect_ = EXPECT_OPERATOR;
+ lf::append_wchar(r, c);
+ is_definition_ = false;
+ } else if (isspace(c)) {
+ lf::append_wchar(r, c);
} else {
- *r += c;
+ lf::append_wchar(r, c);
+ is_definition_ = false;
}
break;
case EQUAL:
if (!isascii(c) || isalpha(c) || c == '_') {
t_ = EQUAL_WORD;
- word_ += c;
+ lf::append_wchar(&word_, c);
} else {
*r += '=';
t_ = NORMAL;
@@ -183,12 +314,12 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case EQUAL_WORD:
if (isident(c)) {
- word_ += c;
+ lf::append_wchar(&word_, c);
break;
} else if (word_ == "begin") {
*r += HI_COMMENT;
*r += "=begin";
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\n') {
t_ = MULTICOM_BOL;
i_ = 0;
@@ -205,27 +336,34 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case WORD:
if (isident(c)) {
- word_ += c;
+ lf::append_wchar(&word_, c);
} else {
if (is_keyword_ruby(word_.data(), word_.size())) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- last_ = 0;
+ expect_ = EXPECT_VALUE;
+ if (word_ == "def") {
+ is_definition_ = true;
+ }
} else if (is_keyword_ruby_builtin(word_.data(), word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
+ expect_ = EXPECT_VALUE;
} else if (is_keyword_ruby_constant(word_.data(), word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
+ expect_ = EXPECT_OPERATOR;
} else if (!word_.empty() && isupper(word_[0])) {
*r += HI_CLASS;
*r += word_;
*r += HI_RESET;
+ expect_ = EXPECT_OPERATOR;
} else {
*r += word_;
+ expect_ = EXPECT_OPERATOR;
}
word_.clear();
t_ = NORMAL;
@@ -233,23 +371,81 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
}
break;
+ Regex:
case REGEX:
- *r += c;
if (c == '/') {
+ lf::append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
+ } else if (c == '#') {
+ t_ = REGEX_HASH;
} else if (c == '\\') {
+ lf::append_wchar(r, c);
t_ = REGEX_BACKSLASH;
+ } else {
+ lf::append_wchar(r, c);
+ }
+ break;
+
+ case REGEX_HASH:
+ if (c == '{' && nesti_ < sizeof(nest_)) {
+ *r += HI_BOLD;
+ *r += '#';
+ *r += HI_UNBOLD;
+ *r += HI_STRING;
+ *r += '{';
+ *r += HI_RESET;
+ expect_ = EXPECT_VALUE;
+ nest_[nesti_++] = REGEX;
+ t_ = NORMAL;
+ } else if (c == '$') {
+ t_ = REGEX_HASH_DOLLAR;
+ } else {
+ *r += '#';
+ t_ = REGEX;
+ goto Regex;
+ }
+ break;
+
+ case REGEX_HASH_DOLLAR:
+ if (is_dollar_one(c)) {
+ *r += '#';
+ *r += HI_BOLD;
+ *r += '$';
+ lf::append_wchar(r, c);
+ *r += HI_UNBOLD;
+ t_ = REGEX;
+ } else if (isalpha(c)) {
+ *r += '#';
+ *r += HI_BOLD;
+ *r += '$';
+ lf::append_wchar(r, c);
+ t_ = REGEX_HASH_DOLLAR_WORD;
+ } else {
+ *r += '#';
+ *r += '$';
+ t_ = REGEX;
+ goto Regex;
+ }
+ break;
+
+ case REGEX_HASH_DOLLAR_WORD:
+ if (isident(c)) {
+ lf::append_wchar(r, c);
+ } else {
+ *r += HI_UNBOLD;
+ t_ = REGEX;
+ goto Regex;
}
break;
case REGEX_BACKSLASH:
- *r += c;
+ lf::append_wchar(r, c);
t_ = REGEX;
break;
case MULTICOM:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\n') {
t_ = MULTICOM_BOL;
i_ = 0;
@@ -257,7 +453,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case MULTICOM_BOL:
- *r += c;
+ lf::append_wchar(r, c);
if (c == "=end"[i_]) {
if (++i_ == 4) {
t_ = NORMAL;
@@ -272,7 +468,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
if (isident(c)) {
*r += HI_LISPKW;
*r += ':';
- *r += c;
+ lf::append_wchar(r, c);
t_ = COLON_WORD;
} else {
*r += ':';
@@ -283,7 +479,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case COLON_WORD:
if (isident(c)) {
- *r += c;
+ lf::append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -295,7 +491,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
if (isident(c)) {
*r += HI_VAR;
*r += '@';
- *r += c;
+ lf::append_wchar(r, c);
t_ = AT_WORD;
} else {
*r += '@';
@@ -306,7 +502,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case AT_WORD:
if (isident(c)) {
- *r += c;
+ lf::append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -324,7 +520,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
closer_ = mirror(c);
*r += HI_STRING;
*r += '%';
- *r += c;
+ lf::append_wchar(r, c);
t_ = PERCENT_STRING;
} else {
*r += '%';
@@ -341,7 +537,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
*r += HI_STRING;
*r += '%';
*r += q_;
- *r += c;
+ lf::append_wchar(r, c);
t_ = PERCENT_STRING;
} else {
*r += '%';
@@ -352,7 +548,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case PERCENT_STRING:
- *r += c;
+ lf::append_wchar(r, c);
if (c == opener_ && opener_ != closer_) {
++level_;
} else if (c == closer_) {
@@ -364,32 +560,16 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case DOLLAR:
- if (isdigit(c) || //
- c == '!' || //
- c == '"' || //
- c == '#' || //
- c == '$' || //
- c == '&' || //
- c == '-' || //
- c == '/' || //
- c == '<' || //
- c == '=' || //
- c == '>' || //
- c == '@' || //
- c == '\'' || //
- c == '\\' || //
- c == '^' || //
- c == '_' || //
- c == '`') {
+ if (is_dollar_one(c)) {
*r += HI_VAR;
*r += '$';
- *r += c;
+ lf::append_wchar(r, c);
*r += HI_RESET;
t_ = NORMAL;
} else if (isalpha(c)) {
*r += HI_VAR;
*r += '$';
- *r += c;
+ lf::append_wchar(r, c);
t_ = DOLLAR_WORD;
} else {
*r += '$';
@@ -400,7 +580,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case DOLLAR_WORD:
if (isident(c)) {
- *r += c;
+ lf::append_wchar(r, c);
} else {
*r += HI_RESET;
t_ = NORMAL;
@@ -409,7 +589,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case COMMENT:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\n') {
*r += HI_RESET;
t_ = NORMAL;
@@ -417,7 +597,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case QUOTE:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\'') {
*r += HI_RESET;
t_ = NORMAL;
@@ -427,27 +607,85 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case QUOTE_BACKSLASH:
- *r += c;
+ lf::append_wchar(r, c);
t_ = QUOTE;
break;
+ Dquote:
case DQUOTE:
- *r += c;
if (c == '"') {
+ *r += '"';
*r += HI_RESET;
t_ = NORMAL;
+ } else if (c == '#') {
+ t_ = DQUOTE_HASH;
} else if (c == '\\') {
t_ = DQUOTE_BACKSLASH;
+ *r += '\\';
+ } else {
+ lf::append_wchar(r, c);
+ }
+ break;
+
+ case DQUOTE_HASH:
+ if (c == '{' && nesti_ < sizeof(nest_)) {
+ *r += HI_BOLD;
+ *r += '#';
+ *r += HI_UNBOLD;
+ *r += HI_STRING;
+ *r += '{';
+ *r += HI_RESET;
+ expect_ = EXPECT_VALUE;
+ nest_[nesti_++] = DQUOTE;
+ t_ = NORMAL;
+ } else if (c == '$') {
+ t_ = DQUOTE_HASH_DOLLAR;
+ } else {
+ *r += '#';
+ t_ = DQUOTE;
+ goto Dquote;
}
break;
case DQUOTE_BACKSLASH:
- *r += c;
+ lf::append_wchar(r, c);
t_ = DQUOTE;
break;
+ case DQUOTE_HASH_DOLLAR:
+ if (is_dollar_one(c)) {
+ *r += '#';
+ *r += HI_BOLD;
+ *r += '$';
+ lf::append_wchar(r, c);
+ *r += HI_UNBOLD;
+ t_ = DQUOTE;
+ } else if (isalpha(c)) {
+ *r += '#';
+ *r += HI_BOLD;
+ *r += '$';
+ lf::append_wchar(r, c);
+ t_ = DQUOTE_HASH_DOLLAR_WORD;
+ } else {
+ *r += '#';
+ *r += '$';
+ t_ = DQUOTE;
+ goto Dquote;
+ }
+ break;
+
+ case DQUOTE_HASH_DOLLAR_WORD:
+ if (isident(c)) {
+ lf::append_wchar(r, c);
+ } else {
+ *r += HI_UNBOLD;
+ t_ = DQUOTE;
+ goto Dquote;
+ }
+ break;
+
case TICK:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '`') {
*r += HI_RESET;
t_ = NORMAL;
@@ -457,13 +695,13 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case TICK_BACKSLASH:
- *r += c;
+ lf::append_wchar(r, c);
t_ = TICK;
break;
case LT:
if (c == '<') {
- *r += c;
+ lf::append_wchar(r, c);
t_ = LT_LT;
heredoc_.clear();
pending_heredoc_ = false;
@@ -477,16 +715,16 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
case LT_LT:
if (c == '-') {
indented_heredoc_ = true;
- *r += c;
+ lf::append_wchar(r, c);
} else if (c == '\'' || c == '`' || c == '"') {
closer_ = c;
t_ = LT_LT_QNAME;
*r += HI_STRING;
- *r += c;
+ lf::append_wchar(r, c);
} else if (isalpha(c) || c == '_') {
t_ = LT_LT_NAME;
heredoc_ += c;
- *r += c;
+ lf::append_wchar(r, c);
} else {
t_ = NORMAL;
goto Normal;
@@ -497,9 +735,9 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
if (isalnum(c) || c == '_') {
t_ = LT_LT_NAME;
heredoc_ += c;
- *r += c;
+ lf::append_wchar(r, c);
} else if (c == '\n') {
- *r += c;
+ lf::append_wchar(r, c);
*r += HI_STRING;
t_ = HEREDOC_BOL;
} else {
@@ -510,7 +748,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case LT_LT_QNAME:
- *r += c;
+ lf::append_wchar(r, c);
if (c == closer_) {
*r += HI_RESET;
t_ = HEREDOC_BOL;
@@ -522,7 +760,7 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case HEREDOC_BOL:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\n') {
if (i_ == heredoc_.size()) {
t_ = NORMAL;
@@ -540,11 +778,35 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
break;
case HEREDOC:
- *r += c;
+ lf::append_wchar(r, c);
if (c == '\n')
t_ = HEREDOC_BOL;
break;
+ case QUESTION:
+ if (c == '\\') {
+ t_ = QUESTION_BACKSLASH;
+ } else if (isspace(c)) {
+ *r += '?';
+ t_ = NORMAL;
+ goto Normal;
+ } else {
+ *r += HI_ESCAPE;
+ *r += '?';
+ lf::append_wchar(r, c);
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ break;
+
+ case QUESTION_BACKSLASH:
+ *r += HI_ESCAPE;
+ *r += "?\\";
+ lf::append_wchar(r, c);
+ *r += HI_RESET;
+ t_ = NORMAL;
+ break;
+
default:
__builtin_unreachable();
}
@@ -597,8 +859,31 @@ void HighlightRuby::flush(std::string *r) {
*r += '%';
*r += q_;
break;
+ case QUESTION:
+ *r += '?';
+ break;
+ case QUESTION_BACKSLASH:
+ *r += "?\\";
+ break;
+ case DQUOTE_HASH:
+ *r += '#';
+ *r += HI_RESET;
+ break;
+ case DQUOTE_HASH_DOLLAR:
+ *r += "#$";
+ *r += HI_RESET;
+ break;
+ case REGEX_HASH:
+ *r += '#';
+ *r += HI_RESET;
+ break;
+ case REGEX_HASH_DOLLAR:
+ *r += "#$";
+ *r += HI_RESET;
+ break;
case REGEX:
case REGEX_BACKSLASH:
+ case REGEX_HASH_DOLLAR_WORD:
case PERCENT_STRING:
case AT_WORD:
case DOLLAR_WORD:
@@ -608,6 +893,7 @@ void HighlightRuby::flush(std::string *r) {
case QUOTE_BACKSLASH:
case DQUOTE:
case DQUOTE_BACKSLASH:
+ case DQUOTE_HASH_DOLLAR_WORD:
case COMMENT:
case HEREDOC_BOL:
case HEREDOC:
@@ -620,5 +906,10 @@ void HighlightRuby::flush(std::string *r) {
default:
break;
}
+ c_ = 0;
+ u_ = 0;
t_ = NORMAL;
+ is_definition_ = 0;
+ expect_ = EXPECT_VALUE;
+ nesti_ = 0;
}
diff --git a/llamafile/server/www/ctype.js b/llamafile/server/www/ctype.js
index 6a50ffb3..a26a5999 100644
--- a/llamafile/server/www/ctype.js
+++ b/llamafile/server/www/ctype.js
@@ -92,7 +92,6 @@ function ispunct(c) {
case '|':
case '}':
case '~':
- case '}':
return true;
default:
return false;
diff --git a/llamafile/server/www/highlight_markdown.js b/llamafile/server/www/highlight_markdown.js
index 9ca50ad7..8f28e986 100644
--- a/llamafile/server/www/highlight_markdown.js
+++ b/llamafile/server/www/highlight_markdown.js
@@ -45,6 +45,32 @@ class HighlightMarkdown extends Highlighter {
static STYLE_EMPHASIS = 1;
static STYLE_LIST = 2;
+ static is_escapable(c) {
+ switch (c) {
+ case '!':
+ case '#':
+ case '(':
+ case ')':
+ case '*':
+ case '+':
+ case '-':
+ case '.':
+ case '<':
+ case '>':
+ case '[':
+ case '\\':
+ case ']':
+ case '_':
+ case '`':
+ case '{':
+ case '|':
+ case '}':
+ return true;
+ default:
+ return false;
+ }
+ }
+
constructor(delegate) {
super(delegate);
this.bol = true;
@@ -150,6 +176,7 @@ class HighlightMarkdown extends Highlighter {
case HighlightMarkdown.HYPHEN:
if (isblank(c)) {
+ // - handle list item
this.li('-');
this.append(c);
this.state = HighlightMarkdown.NORMAL;
@@ -160,13 +187,19 @@ class HighlightMarkdown extends Highlighter {
break;
case HighlightMarkdown.BACKSLASH:
- this.append(c);
- this.state = HighlightMarkdown.NORMAL;
+ if (HighlightMarkdown.is_escapable(c)) {
+ this.append(c);
+ this.state = HighlightMarkdown.NORMAL;
+ } else {
+ this.append('\\');
+ this.epsilon(HighlightMarkdown.NORMAL);
+ }
break;
case HighlightMarkdown.STAR:
if (c == '*') {
// handle **strong** text
+ // we don't call this.got() in case this is *** bar
this.push('strong', '');
this.state = HighlightMarkdown.NORMAL;
this.style.push(HighlightMarkdown.STYLE_STRONG);
@@ -360,6 +393,8 @@ class HighlightMarkdown extends Highlighter {
break;
case HighlightMarkdown.LAB_BACKSLASH:
+ if (!HighlightMarkdown.is_escapable(c))
+ this.href += '\\';
this.href += c;
this.state = HighlightMarkdown.LAB;
break;
@@ -375,6 +410,8 @@ class HighlightMarkdown extends Highlighter {
break;
case HighlightMarkdown.LSB_BACKSLASH:
+ if (!HighlightMarkdown.is_escapable(c))
+ this.text += '\\';
this.text += c;
this.state = HighlightMarkdown.LSB;
break;
@@ -406,6 +443,8 @@ class HighlightMarkdown extends Highlighter {
break;
case HighlightMarkdown.LSB_RSB_LPAREN_BACKSLASH:
+ if (!HighlightMarkdown.is_escapable(c))
+ this.href += '\\';
this.href += c;
this.state = HighlightMarkdown.LSB_RSB_LPAREN;
break;
@@ -492,12 +531,29 @@ class HighlightMarkdown extends Highlighter {
setupCodeBlock(pre) {
const copyButton = document.createElement('button');
copyButton.className = 'copy-button';
- copyButton.innerHTML = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path></svg>`;
+ copyButton.innerHTML =
+ `<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24" fill="none"
+ stroke="currentColor"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round">
+ <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>
+ <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>
+ </svg>`;
copyButton.addEventListener('click', function() {
try {
copyTextToClipboard(pre.innerText);
const originalInnerHTML = copyButton.innerHTML;
- copyButton.innerHTML = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="green" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"></polyline></svg>`;
+ copyButton.innerHTML =
+ `<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 24 24" fill="none"
+ stroke="green"
+ stroke-width="2"
+ stroke-linecap="round"
+ stroke-linejoin="round">
+ <polyline points="20 6 9 17 4 12"></polyline>
+ </svg>`;
setTimeout(() => {
copyButton.innerHTML = originalInnerHTML;
}, 2000);
diff --git a/llamafile/server/www/highlight_ruby.js b/llamafile/server/www/highlight_ruby.js
index 79b87de6..84fd5c48 100644
--- a/llamafile/server/www/highlight_ruby.js
+++ b/llamafile/server/www/highlight_ruby.js
@@ -162,6 +162,18 @@ class HighlightRuby extends Highlighter {
static MULTICOM_BOL = 27;
static REGEX = 28;
static REGEX_BACKSLASH = 29;
+ static DQUOTE_HASH = 30;
+ static QUESTION = 31;
+ static QUESTION_BACKSLASH = 32;
+ static REGEX_HASH = 33;
+ static REGEX_HASH_DOLLAR = 34;
+ static REGEX_HASH_DOLLAR_WORD = 35;
+ static DQUOTE_HASH = 36;
+ static DQUOTE_HASH_DOLLAR = 37;
+ static DQUOTE_HASH_DOLLAR_WORD = 38;
+
+ static EXPECT_VALUE = 0;
+ static EXPECT_OPERATOR = 1;
static mirror(c) {
switch (c) {
@@ -178,6 +190,61 @@ class HighlightRuby extends Highlighter {
}
}
+ static ispunct_overridable(c) {
+ switch (c) {
+ case '%':
+ case '&':
+ case '*':
+ case '+':
+ case '-':
+ case '/':
+ case '<':
+ case '>':
+ case '^':
+ case '_':
+ case '`':
+ case '|':
+ case '~':
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static is_dollar_one(c) {
+ switch (c) {
+ case '!':
+ case '"':
+ case '#':
+ case '$':
+ case '&':
+ case '-':
+ case '/':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case '<':
+ case '=':
+ case '>':
+ case '@':
+ case '\'':
+ case '\\':
+ case '^':
+ case '_':
+ case '`':
+ return true;
+ default:
+ return false;
+ }
+ }
+
static isident(c) {
return !isascii(c) ||
isalnum(c) ||
@@ -198,7 +265,6 @@ class HighlightRuby extends Highlighter {
super(delegate);
this.i = 0;
this.q = 0;
- this.last = 0;
this.level = 0;
this.opener = 0;
this.closer = 0;
@@ -206,57 +272,86 @@ class HighlightRuby extends Highlighter {
this.indented_heredoc = false;
this.heredoc = '';
this.word = '';
+ this.nest = [];
+ this.is_definition = false;
+ this.expect = HighlightRuby.EXPECT_VALUE;
}
feed(input) {
for (let i = 0; i < input.length; i += this.delta) {
this.delta = 1;
let c = input[i];
-
- if (!isblank(c) && c != '/' && c != '<')
- this.last = c;
-
switch (this.state) {
case HighlightRuby.NORMAL:
- if (!isascii(c) || isalpha(c) || c == '_') {
+ if (!isascii(c) || isalpha(c) || c == '_' ||
+ (this.is_definition && HighlightRuby.ispunct_overridable(c))) {
this.state = HighlightRuby.WORD;
this.word += c;
+ this.is_definition = false;
} else if (c == ':') {
this.state = HighlightRuby.COLON;
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.is_definition = false;
} else if (c == '@') {
this.state = HighlightRuby.AT;
+ this.is_definition = false;
} else if (c == '=') {
this.state = HighlightRuby.EQUAL;
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.is_definition = false;
+ } else if (c == '?' && this.expect == HighlightRuby.EXPECT_VALUE) {
+ this.state = HighlightRuby.QUESTION;
+ this.is_definition = false;
} else if (c == '$') {
this.state = HighlightRuby.DOLLAR;
- } else if (c == '%') {
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
+ this.is_definition = false;
+ } else if (c == '%' && this.expect == HighlightRuby.EXPECT_VALUE) {
this.state = HighlightRuby.PERCENT;
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
this.q = 0;
} else if (c == '\'') {
this.state = HighlightRuby.QUOTE;
this.push("span", "string");
this.append(c);
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
+ this.is_definition = false;
} else if (c == '"') {
this.state = HighlightRuby.DQUOTE;
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
this.push("span", "string");
this.append(c);
+ this.is_definition = false;
} else if (c == '`') {
this.state = HighlightRuby.TICK;
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
this.push("span", "string");
this.append(c);
} else if (c == '#') {
this.push("span", "comment");
this.append(c);
this.state = HighlightRuby.COMMENT;
- } else if (c == '<' && !isalnum(this.last)) {
+ } else if (c == '<' && this.expect == HighlightRuby.EXPECT_VALUE) {
this.append(c);
this.state = HighlightRuby.LT;
- } else if (c == '/' && !isalnum(this.last)) {
+ } else if (c == '/' && this.expect == HighlightRuby.EXPECT_VALUE) {
this.state = HighlightRuby.REGEX;
this.push("span", "string");
this.append(c);
+ } else if (c == '{' && this.nest.length) {
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.append('{');
+ this.nest.push(HighlightRuby.NORMAL);
+ this.is_definition = false;
+ } else if (c == '}' && this.nest.length) {
+ if ((this.state = this.nest.pop()) != HighlightRuby.NORMAL)
+ this.push("span", "string");
+ this.append('}');
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
+ this.is_definition = false;
} else if (c == '\n') {
+ this.expect = HighlightRuby.EXPECT_VALUE;
this.append(c);
if (this.pending_heredoc) {
this.push("span", "string");
@@ -264,8 +359,23 @@ class HighlightRuby extends Highlighter {
this.state = HighlightRuby.HEREDOC_BOL;
this.i = 0;
}
+ } else if (c == ']') {
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
+ this.append(']');
+ this.is_definition = false;
+ } else if (ispunct(c)) {
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.append(c);
+ this.is_definition = false;
+ } else if (isdigit(c) || c == '.') {
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
+ this.append(c);
+ this.is_definition = false;
+ } else if (isspace(c)) {
+ this.append(c);
} else {
this.append(c);
+ this.is_definition = false;
}
break;
@@ -309,21 +419,28 @@ class HighlightRuby extends Highlighter {
this.push("span", "keyword");
this.append(this.word);
this.pop();
- this.last = 0;
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ if (this.word == 'def') {
+ this.is_definition = true;
+ }
} else if (RUBY_BUILTINS.has(this.word)) {
this.push("span", "builtin");
this.append(this.word);
this.pop();
+ this.expect = HighlightRuby.EXPECT_VALUE;
} else if (RUBY_CONSTANTS.has(this.word)) {
this.push("span", "constant");
this.append(this.word);
this.pop();
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
} else if (this.word && this.word[0] == this.word[0].toUpperCase()) {
this.push("span", "class");
this.append(this.word);
this.pop();
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
} else {
this.append(this.word);
+ this.expect = HighlightRuby.EXPECT_OPERATOR;
}
this.word = '';
this.epsilon(HighlightRuby.NORMAL);
@@ -331,12 +448,65 @@ class HighlightRuby extends Highlighter {
break;
case HighlightRuby.REGEX:
- this.append(c);
if (c == '/') {
+ this.append(c);
this.pop();
this.state = HighlightRuby.NORMAL;
+ } else if (c == '#') {
+ this.state = HighlightRuby.REGEX_HASH;
} else if (c == '\\') {
+ this.append(c);
this.state = HighlightRuby.REGEX_BACKSLASH;
+ } else {
+ this.append(c);
+ }
+ break;
+
+ case HighlightRuby.REGEX_HASH:
+ if (c == '{') {
+ this.push("strong", "");
+ this.append('#');
+ this.pop();
+ this.append('{');
+ this.pop();
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.nest.push(HighlightRuby.REGEX);
+ this.state = HighlightRuby.NORMAL;
+ } else if (c == '$') {
+ this.state = HighlightRuby.REGEX_HASH_DOLLAR;
+ } else {
+ this.append('#');
+ this.epsilon(HighlightRuby.REGEX);
+ }
+ break;
+
+ case HighlightRuby.REGEX_HASH_DOLLAR:
+ if (HighlightRuby.is_dollar_one(c)) {
+ this.append('#');
+ this.push("strong", "");
+ this.append('$');
+ this.append(c);
+ this.pop();
+ this.state = HighlightRuby.REGEX;
+ } else if (isalpha(c)) {
+ this.append('#');
+ this.push("strong", "");
+ this.append('$');
+ this.append(c);
+ this.state = HighlightRuby.REGEX_HASH_DOLLAR_WORD;
+ } else {
+ this.append('#');
+ this.append('$');
+ this.epsilon(HighlightRuby.REGEX);
+ }
+ break;
+
+ case HighlightRuby.REGEX_HASH_DOLLAR_WORD:
+ if (HighlightRuby.isident(c)) {
+ this.append(c);
+ } else {
+ this.pop();
+ this.epsilon(HighlightRuby.REGEX);
}
break;
@@ -455,23 +625,7 @@ class HighlightRuby extends Highlighter {
break;
case HighlightRuby.DOLLAR:
- if (isdigit(c) || //
- c == '!' || //
- c == '"' || //
- c == '#' || //
- c == '$' || //
- c == '&' || //
- c == '-' || //
- c == '/' || //
- c == '<' || //
- c == '=' || //
- c == '>' || //
- c == '@' || //
- c == '\'' || //
- c == '\\' || //
- c == '^' || //
- c == '_' || //
- c == '`') {
+ if (HighlightRuby.is_dollar_one(c)) {
this.push("span", "var");
this.append('$');
this.append(c);
@@ -521,12 +675,65 @@ class HighlightRuby extends Highlighter {
break;
case HighlightRuby.DQUOTE:
- this.append(c);
if (c == '"') {
+ this.append(c);
this.pop();
this.state = HighlightRuby.NORMAL;
+ } else if (c == '#') {
+ this.state = HighlightRuby.DQUOTE_HASH;
} else if (c == '\\') {
this.state = HighlightRuby.DQUOTE_BACKSLASH;
+ this.append(c);
+ } else {
+ this.append(c);
+ }
+ break;
+
+ case HighlightRuby.DQUOTE_HASH:
+ if (c == '{') {
+ this.push("strong", "");
+ this.append('#');
+ this.pop();
+ this.append('{');
+ this.pop();
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.nest.push(HighlightRuby.DQUOTE);
+ this.state = HighlightRuby.NORMAL;
+ } else if (c == '$') {
+ this.state = HighlightRuby.DQUOTE_HASH_DOLLAR;
+ } else {
+ this.append('#');
+ this.epsilon(HighlightRuby.DQUOTE);
+ }
+ break;
+
+ case HighlightRuby.DQUOTE_HASH_DOLLAR:
+ if (HighlightRuby.is_dollar_one(c)) {
+ this.append('#');
+ this.push("strong", "");
+ this.append('$');
+ this.append(c);
+ this.pop();
+ this.state = HighlightRuby.DQUOTE;
+ } else if (isalpha(c)) {
+ this.append('#');
+ this.push("strong", "");
+ this.append('$');
+ this.append(c);
+ this.state = HighlightRuby.DQUOTE_HASH_DOLLAR_WORD;
+ } else {
+ this.append('#');
+ this.append('$');
+ this.epsilon(HighlightRuby.DQUOTE);
+ }
+ break;
+
+ case HighlightRuby.DQUOTE_HASH_DOLLAR_WORD:
+ if (HighlightRuby.isident(c)) {
+ this.append(c);
+ } else {
+ this.pop();
+ this.epsilon(HighlightRuby.DQUOTE);
}
break;
@@ -631,6 +838,29 @@ class HighlightRuby extends Highlighter {
this.state = HighlightRuby.HEREDOC_BOL;
break;
+ case HighlightRuby.QUESTION:
+ if (c == '\\') {
+ this.state = HighlightRuby.QUESTION_BACKSLASH;
+ } else if (isspace(c)) {
+ this.append('?');
+ this.epsilon(HighlightRuby.NORMAL);
+ } else {
+ this.push('span', 'escape');
+ this.append('?');
+ this.append(c);
+ this.pop();
+ this.state = HighlightRuby.NORMAL;
+ }
+ break;
+
+ case HighlightRuby.QUESTION_BACKSLASH:
+ this.push('span', 'escape');
+ this.append("?\\");
+ this.append(c);
+ this.pop();
+ this.state = HighlightRuby.NORMAL;
+ break;
+
default:
throw new Error('Invalid state');
}
@@ -683,8 +913,31 @@ class HighlightRuby extends Highlighter {
this.append('%');
this.append(this.q);
break;
+ case HighlightRuby.QUESTION:
+ this.append('?');
+ break;
+ case HighlightRuby.QUESTION_BACKSLASH:
+ this.append("?\\");
+ break;
+ case HighlightRuby.DQUOTE_HASH:
+ this.append('#');
+ this.pop();
+ break;
+ case HighlightRuby.DQUOTE_HASH_DOLLAR:
+ this.append("#$");
+ this.pop();
+ break;
+ case HighlightRuby.REGEX_HASH:
+ this.append('#');
+ this.pop();
+ break;
+ case HighlightRuby.REGEX_HASH_DOLLAR:
+ this.append("#$");
+ this.pop();
+ break;
case HighlightRuby.REGEX:
case HighlightRuby.REGEX_BACKSLASH:
+ case HighlightRuby.REGEX_HASH_DOLLAR_WORD:
case HighlightRuby.PERCENT_STRING:
case HighlightRuby.AT_WORD:
case HighlightRuby.DOLLAR_WORD:
@@ -694,6 +947,7 @@ class HighlightRuby extends Highlighter {
case HighlightRuby.QUOTE_BACKSLASH:
case HighlightRuby.DQUOTE:
case HighlightRuby.DQUOTE_BACKSLASH:
+ case HighlightRuby.DQUOTE_HASH_DOLLAR_WORD:
case HighlightRuby.COMMENT:
case HighlightRuby.HEREDOC_BOL:
case HighlightRuby.HEREDOC:
@@ -706,6 +960,9 @@ class HighlightRuby extends Highlighter {
default:
break;
}
+ this.nest = [];
+ this.expect = HighlightRuby.EXPECT_VALUE;
+ this.is_definition = false;
this.state = HighlightRuby.NORMAL;
this.delegate.flush();
this.delta = 1;