Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Put more work into markdown rendering
Justine Tunneyc0622daa4f2d
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 254add21..31b59bbc 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -173,19 +173,20 @@ o/$(MODE)/llamafile/highlight_test: \o/$(MODE)/llamafile/highlight_test.o \o/$(MODE)/llama.cpp/llama.cpp.a \-o/$(MODE)/llamafile/highlight_c_test: \- o/$(MODE)/llamafile/highlight_c_test.o \- o/$(MODE)/llamafile/highlight_c.o \- o/$(MODE)/llamafile/is_keyword_c.o \+o/$(MODE)/llamafile/highlight_c_test: \+ o/$(MODE)/llamafile/highlight_c_test.o \+ o/$(MODE)/llamafile/highlight_c.o \+ o/$(MODE)/llamafile/is_keyword_c.o \o/$(MODE)/llamafile/is_keyword_c_constant.o \- o/$(MODE)/llamafile/is_keyword_c_type.o \- o/$(MODE)/llamafile/is_keyword_c_pod.o \- o/$(MODE)/llamafile/is_keyword_cpp.o \+ o/$(MODE)/llamafile/is_keyword_c_type.o \+ o/$(MODE)/llamafile/is_keyword_c_pod.o \+ o/$(MODE)/llamafile/is_keyword_cpp.o \o/$(MODE)/llamafile/highlight_python_test: \o/$(MODE)/llamafile/highlight_python_test.o \o/$(MODE)/llamafile/highlight_python.o \o/$(MODE)/llamafile/is_keyword_python.o \+ o/$(MODE)/llamafile/is_keyword_python_builtin.o \o/$(MODE)/llamafile/is_keyword_python_constant.o \o/$(MODE)/llamafile/high: \diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex e5c859d1..97c85030 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -138,6 +138,7 @@ Highlight *Highlight::create(const std::string_view &lang) {lang == "cl" || //lang == "clisp" || //lang == "scheme" || //+ lang == "racket" || //lang == "clojure")return new HighlightLisp;diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 394eb07b..c1fe519e 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -69,6 +69,7 @@ is_keyword_f is_keyword_js_constant;is_keyword_f is_keyword_java;is_keyword_f is_keyword_java_constant;is_keyword_f is_keyword_python;+is_keyword_f is_keyword_python_builtin;is_keyword_f is_keyword_python_constant;is_keyword_f is_keyword_rust;is_keyword_f is_keyword_rust_type;@@ -715,6 +716,7 @@ class HighlightSwift : public Highlight {int hash1_ = 0;int hash2_ = 0;int nesti_ = 0;+ int expect_ = 0;std::string word_;unsigned char nest_[16];unsigned char hash_[16];diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cppindex 83bd1505..06f0f950 100644--- a/llamafile/highlight_python.cpp+++ b/llamafile/highlight_python.cpp@@ -84,6 +84,10 @@ void HighlightPython::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;+ } else if (is_keyword_python_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;} else if (is_keyword_python_constant(word_.data(), word_.size())) {*r += HI_CONSTANT;*r += word_;@@ -268,6 +272,10 @@ void HighlightPython::flush(std::string *r) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;+ } else if (is_keyword_python_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;} else if (is_keyword_python_constant(word_.data(), word_.size())) {*r += HI_CONSTANT;*r += word_;diff --git a/llamafile/highlight_swift.cpp b/llamafile/highlight_swift.cppindex 2a799372..8af3d2d1 100644--- a/llamafile/highlight_swift.cpp+++ b/llamafile/highlight_swift.cpp@@ -58,6 +58,11 @@ enum {REGEX_BACKSLASH,};+enum {+ EXPECT_VALUE,+ EXPECT_OPERATOR,+};+HighlightSwift::HighlightSwift() {}@@ -100,21 +105,31 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {*r += HI_STRING;*r += '"';hash1_ = 0;+ expect_ = EXPECT_OPERATOR;} else if (c == '#') {t_ = HASH;hash1_ = 1;+ expect_ = EXPECT_OPERATOR;} else if (c == '(' && nesti_ && nesti_ < sizeof(nest_)) {*r += '(';nest_[nesti_] = NORMAL;hash_[nesti_] = 0;nesti_++;+ expect_ = EXPECT_VALUE;} else if (c == ')' && nesti_) {+ expect_ = EXPECT_OPERATOR;--nesti_;t_ = nest_[nesti_];hash1_ = hash_[nesti_];if (t_ != NORMAL)*r += HI_STRING;*r += ')';+ } else if (c == ')' || c == ']' || isdigit(c) || c == '.') {+ expect_ = EXPECT_OPERATOR;+ lf::append_wchar(r, c);+ } else if (ispunct(c) || c == '\n') {+ expect_ = EXPECT_VALUE;+ lf::append_wchar(r, c);} else {lf::append_wchar(r, c);}@@ -159,12 +174,16 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {*r += HI_COMMENT;*r += "/*";t_ = SLASH_STAR;- } else {+ } else if (expect_ == EXPECT_VALUE) {*r += HI_STRING;*r += '/';hash1_ = 0;t_ = REGEX;goto Regex;+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;}break;@@ -452,5 +471,6 @@ void HighlightSwift::flush(std::string *r) {c_ = 0;u_ = 0;t_ = NORMAL;+ expect_ = 0;nesti_ = 0;}diff --git a/llamafile/is_keyword_python_builtin.gperf b/llamafile/is_keyword_python_builtin.gperfnew file mode 100644index 00000000..4c9b001d--- /dev/null+++ b/llamafile/is_keyword_python_builtin.gperf@@ -0,0 +1,80 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_python_builtin+%%+__import__+abs+aiter+all+anext+any+ascii+bin+bool+breakpoint+bytearray+bytes+callable+chr+classmethod+compile+complex+delattr+dict+dir+divmod+enumerate+eval+exec+filter+float+format+frozenset+getattr+globals+hasattr+hash+help+hex+id+input+int+isinstance+issubclass+iter+len+list+locals+map+max+memoryview+min+next+object+oct+open+ord+pow+property+range+repr+reversed+round+set+setattr+slice+sorted+staticmethod+str+sum+super+tuple+type+vars+zipdiff --git a/llamafile/server/www/chatbot.css b/llamafile/server/www/chatbot.cssindex 365269e1..771b1085 100644--- a/llamafile/server/www/chatbot.css+++ b/llamafile/server/www/chatbot.css@@ -207,6 +207,23 @@ ul li:first-child {position: relative;}+.message blockquote {+ border-left: 4px solid #8b5cf6; /* Purple accent line */+ margin: 1rem 0;+ padding: 0.75rem 1rem;+ background: rgba(139, 92, 246, 0.05); /* Very light purple background */+ border-radius: 0.375rem;+ font-style: italic;+ color: #4b5563; /* Slightly muted text */+}++/* Optional: Style nested blockquotes differently */+.message blockquote blockquote {+ border-left-color: #6d28d9; /* Darker purple for nested quotes */+ background: rgba(109, 40, 217, 0.05);+ margin-left: 1rem;+}+.copy-button {position: absolute;top: 0.5rem;diff --git a/llamafile/server/www/chatbot.js b/llamafile/server/www/chatbot.jsindex e7843952..cf54fdab 100644--- a/llamafile/server/www/chatbot.js+++ b/llamafile/server/www/chatbot.js@@ -37,7 +37,7 @@ function createMessageElement(content, role) {const messageDiv = document.createElement("div");messageDiv.classList.add("message", role);let hdom = new HighlightDom(messageDiv);- const high = new HighlightMarkdown(hdom);+ const high = new RenderMarkdown(hdom);high.feed(content);high.flush();return messageDiv;@@ -73,7 +73,7 @@ async function handleChatStream(response) {let currentMessageElement = createMessageElement("", "assistant");chatMessages.appendChild(currentMessageElement);let hdom = new HighlightDom(currentMessageElement);- const high = new HighlightMarkdown(hdom);+ const high = new RenderMarkdown(hdom);streamingMessageContent = [];try {diff --git a/llamafile/server/www/ctype.js b/llamafile/server/www/ctype.jsindex 8365c4ff..5856956f 100644--- a/llamafile/server/www/ctype.js+++ b/llamafile/server/www/ctype.js@@ -17,6 +17,11 @@ function isascii(c) {return 0 <= i && i <= 127;}+function isgraph(c) {+ let i = c.charCodeAt(0);+ return 0x21 <= i && i <= 0x7E;+}+function isdigit(c) {let i = c.charCodeAt(0);return (48 <= i && i <= 57);diff --git a/llamafile/server/www/highlight_lisp.js b/llamafile/server/www/highlight_lisp.jsindex 8fff4265..3773c775 100644--- a/llamafile/server/www/highlight_lisp.js+++ b/llamafile/server/www/highlight_lisp.js@@ -276,6 +276,7 @@ class HighlightLisp extends Highlighter {}Highlighter.REGISTRY['clojure'] = HighlightLisp;+Highlighter.REGISTRY['racket'] = HighlightLisp;Highlighter.REGISTRY['scheme'] = HighlightLisp;Highlighter.REGISTRY['elisp'] = HighlightLisp;Highlighter.REGISTRY['clisp'] = HighlightLisp;diff --git a/llamafile/server/www/highlight_markdown.js b/llamafile/server/www/highlight_markdown.jsindex 8d270a26..dba9ca74 100644--- a/llamafile/server/www/highlight_markdown.js+++ b/llamafile/server/www/highlight_markdown.js@@ -17,150 +17,31 @@ class HighlightMarkdown extends Highlighter {static NORMAL = 0;static TICK = 1;static TICK_TICK = 2;- static LANG = 3;static CODE = 4;static CODE_TICK = 5;static CODE_TICK_TICK = 6;static STAR = 7;- static EMPHASIS_STAR = 8;- static STRONG_STAR = 9;- static BACKSLASH = 10;- static INCODE = 11;- static INCODE2 = 12;- static INCODE2_TICK = 13;- static LANG2 = 14;- static NEWLINE = 15;- static EAT_NEWLINE = 16;- static INCODE2_TICK2 = 17;- static LSB = 18;- static LSB_BACKSLASH = 19;- static LSB_RSB = 20;- static LSB_RSB_LPAREN = 21;- static LSB_RSB_LPAREN_BACKSLASH = 22;- static LAB = 23;- static LAB_BACKSLASH = 24;- static HYPHEN = 25;- static TILDE = 26;- static EMPHASIS_STAR2 = 27;- static OLNAME = 28;- static OLNAME_DOT = 29;-- static STYLE_SPAN = 16;- static STYLE_STRONG = 16;- static STYLE_EMPHASIS = 17;- static STYLE_STRIKE = 18;-- static STYLE_LIST = 32;- static STYLE_UL = 32;- static STYLE_OL = 33;-- static STYLE_DIV = 64;- static STYLE_LI = 64;-- static get_style_tag(t) {- switch (t) {- case HighlightMarkdown.STYLE_UL:- return 'UL';- case HighlightMarkdown.STYLE_OL:- return 'OL';- default:- throw new Error('bad style');- }- }-- static is_escapable(c) {- switch (c) {- case '!':- case '#':- case '(':- case ')':- case '*':- case '+':- case '-':- case '.':- case '<':- case '>':- case '[':- case '\\':- case ']':- case '_':- case '`':- case '{':- case '|':- case '}':- return true;- default:- return false;- }- }-- static is_sublist(depth, next) {- if (!depth) {- return next > 0;- } else {- return next >= depth + 4;- }- }+ static STRONG = 8;+ static STRONG_BACKSLASH = 9;+ static STRONG_STAR = 10;+ static BACKSLASH = 11;+ static INCODE = 12;+ static INCODE2 = 13;+ static INCODE2_TICK = 14;+ static INCODE2_TICK2 = 15;+ static EMPHASIS = 16;+ static EMPHASIS_BACKSLASH = 17;constructor(delegate) {super(delegate);this.bol = true;this.tail = false;- this.lang = '';- this.highlighter = null;- this.newlines = 0;- this.spaces = 0;- this.text = '';- this.href = '';- this.style = [];- this.trailing_spaces = 0;- this.olname = '';- }-- inside(style) {- return this.style.length && this.style[this.style.length - 1][0] == style;- }-- li(t) {- while (this.style.length &&- (this.style[this.style.length - 1][0] == HighlightMarkdown.STYLE_LI ||- (this.style[this.style.length - 1][0] & HighlightMarkdown.STYLE_SPAN) ||- ((this.style[this.style.length - 1][0] & HighlightMarkdown.STYLE_LIST) &&- (this.spaces < this.style[this.style.length - 1][1] ||- (this.style[this.style.length - 1][0] != t &&- !HighlightMarkdown.is_sublist(this.style[this.style.length - 1][1], this.spaces)))))) {- this.pop();- this.style.pop();- }- if (!this.style.length ||- ((this.style[this.style.length - 1][0] & HighlightMarkdown.STYLE_LIST) &&- HighlightMarkdown.is_sublist(this.style[this.style.length - 1][1], this.spaces))) {- this.style.push([t, this.spaces]);- this.push(HighlightMarkdown.get_style_tag(t), '');- }- this.style.push([HighlightMarkdown.STYLE_LI, this.spaces]);- this.push('LI', '');- this.bol = false;- }-- got() {- if (this.bol) {- this.bol = false;- while (this.style.length &&- (this.style[this.style.length - 1][0] == HighlightMarkdown.STYLE_LI ||- (this.style[this.style.length - 1][0] & HighlightMarkdown.STYLE_LIST)) &&- this.spaces <= this.style[this.style.length - 1][1]) {- this.pop();- this.style.pop();- }- }- this.spaces = 0;}feed(input) {for (let i = 0; i < input.length; i += this.delta) {this.delta = 1;- const c = input[i];+ let c = input[i];switch (this.state) {case HighlightMarkdown.NORMAL:@@ -168,230 +49,99 @@ class HighlightMarkdown extends Highlighter {this.state = HighlightMarkdown.TICK;break;} else if (c == '*') {- if (this.inside(HighlightMarkdown.STYLE_EMPHASIS)) {- this.state = HighlightMarkdown.EMPHASIS_STAR;- } else if (this.inside(HighlightMarkdown.STYLE_STRONG)) {- this.state = HighlightMarkdown.STRONG_STAR;- } else {- this.state = HighlightMarkdown.STAR;- }- break;- } else if (c == '-' && this.bol) {- this.state = HighlightMarkdown.HYPHEN;- break;- } else if (isdigit(c) && this.bol) {- this.epsilon(HighlightMarkdown.OLNAME);- } else if (c == '\n') {- this.bol = true;- this.tail = false;- this.state = HighlightMarkdown.NEWLINE;- this.newlines = 1;- this.trailing_spaces = this.spaces;- this.spaces = 0;+ this.state = HighlightMarkdown.STAR;break;- } else if (c == '~') {- this.state = HighlightMarkdown.TILDE;- this.got();- } else if (c == '[') {- this.state = HighlightMarkdown.LSB;- this.got();- } else if (c == '<') {- this.state = HighlightMarkdown.LAB;- this.got();} else if (c == '\\') {+ // handle \*\*not bold\*\* etc.this.state = HighlightMarkdown.BACKSLASH;- this.got();- } else if (c == ' ') {- ++this.spaces;- this.append(c);- } else if (c == '\t') {- this.spaces += 4;- this.append(c);+ this.append('\\');+ this.bol = false;} else {- this.got();this.append(c);}- this.tail = true;- break;-- case HighlightMarkdown.NEWLINE:- if (c == '\n') {- ++this.newlines;- this.spaces = 0;- } else if (c == ' ') {- ++this.spaces;- } else if (c == '\t') {- this.spaces += 4;- } else {- if (this.newlines >= 2) {- this.push("p", "");- this.pop();- } else if (this.trailing_spaces >= 2) {- this.push('br', '');- this.pop();- } else {- this.append('\n');- }- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.SPACE_SPACE:if (c == '\n') {- this.state = HighlightMarkdown.NORMAL;- this.push('br', '');- this.pop();- this.append('\n');- } else {- this.append(" ");- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.HYPHEN:- if (isblank(c)) {- // - handle list item- this.li(HighlightMarkdown.STYLE_UL);- this.append(c);- this.state = HighlightMarkdown.NORMAL;- } else {- this.append('-');- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.OLNAME:- if (isdigit(c)) {- this.olname += c;- } else if (c == '.') {- this.olname += '.';- this.state = HighlightMarkdown.OLNAME_DOT;- } else {- this.got();- this.append(this.olname);- this.olname = '';- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.OLNAME_DOT:- if (isspace(c)) {- this.li(HighlightMarkdown.STYLE_OL);- this.olname = '';- this.state = HighlightMarkdown.NORMAL;+ this.bol = true;+ this.tail = false;} else {- this.got();- this.append(this.olname);- this.olname = '';- this.epsilon(HighlightMarkdown.NORMAL);+ this.tail = true;+ if (!isblank(c))+ this.bol = false;}break;case HighlightMarkdown.BACKSLASH:- if (HighlightMarkdown.is_escapable(c)) {- this.append(c);- this.state = HighlightMarkdown.NORMAL;- } else {- this.append('\\');- this.epsilon(HighlightMarkdown.NORMAL);- }+ this.append(c);+ this.state = HighlightMarkdown.NORMAL;break;case HighlightMarkdown.STAR:if (c == '*') {// handle **strong** text- // we don't call this.got() in case this is *** bar- this.got();- this.push('strong', '');- this.state = HighlightMarkdown.NORMAL;- this.style.push([HighlightMarkdown.STYLE_STRONG, 0]);+ this.state = HighlightMarkdown.STRONG;+ this.push("STRONG", "");+ this.append("**");} else if (this.bol && isblank(c)) {- // * handle list item- this.li(HighlightMarkdown.STYLE_UL);+ this.append('*');this.append(c);this.state = HighlightMarkdown.NORMAL;} else {- // handle *emphasis* text- this.got();- this.push('em', '');- this.style.push([HighlightMarkdown.STYLE_EMPHASIS, 0]);- this.epsilon(HighlightMarkdown.NORMAL);+ // handle *emphasized* text+ // inverted because \e[3m has a poorly supported western bias+ this.append('*');+ this.push("EM", "");+ this.append(c);+ this.state = HighlightMarkdown.EMPHASIS;+ if (c == '\\')+ this.state = HighlightMarkdown.EMPHASIS_BACKSLASH;}+ this.bol = false;break;- case HighlightMarkdown.EMPHASIS_STAR:+ case HighlightMarkdown.EMPHASIS:+ // this is for *emphasized* textif (c == '*') {- this.state = HighlightMarkdown.EMPHASIS_STAR2;- } else {- // leave *italic* text- // ^- this.style.pop();+ this.state = HighlightMarkdown.NORMAL;this.pop();- this.epsilon(HighlightMarkdown.NORMAL);+ this.append('*');+ } else if (c == '\\') {+ this.state = HighlightMarkdown.EMPHASIS_BACKSLASH;+ this.append('\\');+ } else {+ this.append(c);}break;- case HighlightMarkdown.EMPHASIS_STAR2:- if (c == '*') {- this.style.pop();- this.pop();- if (this.inside(HighlightMarkdown.STYLE_STRONG)) {- // handle ***strong emphasis*** closing- // ^- this.style.pop();- this.pop();- this.state = HighlightMarkdown.NORMAL;- } else {- // handle *emphasis***strong**- // ^- this.push('strong', '');- this.state = HighlightMarkdown.NORMAL;- this.style.push([HighlightMarkdown.STYLE_STRONG, 0]);- this.state = HighlightMarkdown.NORMAL;- }- } else {- // handle *italic **strong** text*- // ^- this.push('strong', '');- this.style.push([HighlightMarkdown.STYLE_STRONG, 0]);- this.epsilon(HighlightMarkdown.NORMAL);- }+ case HighlightMarkdown.EMPHASIS_BACKSLASH:+ // so we can say *unbroken \* italic* and have it work+ this.append(c);+ this.state = HighlightMarkdown.EMPHASIS;break;- case HighlightMarkdown.STRONG_STAR:+ case HighlightMarkdown.STRONG:+ this.append(c);if (c == '*') {- this.state = HighlightMarkdown.NORMAL;- this.style.pop();- this.pop();- } else if (c == '\n' && !this.tail) {- // handle *** line break- this.got();- this.style.pop();- this.pop();- this.push('hr', '');- this.pop();- this.epsilon(HighlightMarkdown.NORMAL);- } else {- this.push('em', '');- this.style.push([HighlightMarkdown.STYLE_EMPHASIS, 0]);- this.epsilon(HighlightMarkdown.NORMAL);+ this.state = HighlightMarkdown.STRONG_STAR;+ } else if (c == '\\') {+ this.state = HighlightMarkdown.STRONG_BACKSLASH;}break;- case HighlightMarkdown.TILDE:- if (c == '~') {- // handle ~~strikethrough text~~- if (this.inside(HighlightMarkdown.STYLE_STRIKE)) {- this.pop();- this.style.pop();- } else {- this.push('s', '');- this.style.push([HighlightMarkdown.STYLE_STRIKE, 0]);- }+ case HighlightMarkdown.STRONG_BACKSLASH:+ // so we can say **unbroken \*\* bold** and have it work+ this.append(c);+ this.state = HighlightMarkdown.STRONG;+ break;++ case HighlightMarkdown.STRONG_STAR:+ this.append(c);+ if (c == '*' || // handle **bold** ending+ (c == '\n' && !this.tail)) { // handle *** line breakthis.state = HighlightMarkdown.NORMAL;+ this.pop();+ } else if (c == '\\') {+ this.state = HighlightMarkdown.STRONG_BACKSLASH;} else {- this.append('~');- this.epsilon(HighlightMarkdown.NORMAL);+ this.state = HighlightMarkdown.STRONG;}break;@@ -400,44 +150,43 @@ class HighlightMarkdown extends Highlighter {if (this.bol) {this.state = HighlightMarkdown.TICK_TICK;} else {- this.push("code", "");+ this.push("SPAN", "incode");+ this.append("``");this.state = HighlightMarkdown.INCODE2;}} else {- this.push("code", "");+ this.push("SPAN", "incode");+ this.append('`');this.append(c);this.state = HighlightMarkdown.INCODE;}- this.got();+ this.bol = false;break;case HighlightMarkdown.INCODE:// this is for `inline code` like that// no backslash escapes are supported here+ this.append(c);if (c == '`') {this.pop();this.state = HighlightMarkdown.NORMAL;- } else {- this.append(c);}break;case HighlightMarkdown.INCODE2:// this is for ``inline ` code`` like that// it lets you put backtick inside the code+ this.append(c);if (c == '`') {this.state = HighlightMarkdown.INCODE2_TICK;- } else {- this.append(c);}break;case HighlightMarkdown.INCODE2_TICK:+ this.append(c);if (c == '`') {this.state = HighlightMarkdown.INCODE2_TICK2;} else {- this.append('`');- this.append(c);this.state = HighlightMarkdown.INCODE2;}break;@@ -453,40 +202,21 @@ class HighlightMarkdown extends Highlighter {case HighlightMarkdown.TICK_TICK:if (c == '`') {- this.state = HighlightMarkdown.LANG;+ this.state = HighlightMarkdown.CODE;+ this.append("```");} else {- this.push('code', '');+ this.push("SPAN", "incode");+ this.append("``");this.append(c);this.state = HighlightMarkdown.INCODE2;}break;- case HighlightMarkdown.LANG:- if (!isascii(c) || !isspace(c)) {- this.lang += c.toLowerCase();- } else {- this.epsilon(HighlightMarkdown.LANG2);- }- break;-- case HighlightMarkdown.LANG2:- if (c == "\n") {- this.flush();- let pre = this.push('pre', '');- this.setupCodeBlock(pre);- let hdom = new HighlightDom(pre);- if (!(this.highlighter = Highlighter.create(this.lang, hdom)))- this.highlighter = Highlighter.create('txt', hdom);- this.state = HighlightMarkdown.CODE;- this.lang = '';- }- break;-case HighlightMarkdown.CODE:if (c == '`') {this.state = HighlightMarkdown.CODE_TICK;} else {- this.highlighter.feed(c);+ this.append(c);}break;@@ -494,236 +224,58 @@ class HighlightMarkdown extends Highlighter {if (c == '`') {this.state = HighlightMarkdown.CODE_TICK_TICK;} else {- this.highlighter.feed("`" + c);- this.state = HighlightMarkdown.CODE;+ this.append('`');+ this.epsilon(HighlightMarkdown.CODE);}break;case HighlightMarkdown.CODE_TICK_TICK:if (c == '`') {- this.state = HighlightMarkdown.EAT_NEWLINE;- this.highlighter.flush();- this.highlighter = null;- this.pop();- } else {- this.highlighter.feed("``" + c);- this.state = HighlightMarkdown.CODE;- }- break;-- case HighlightMarkdown.EAT_NEWLINE:- if (c != '\n') {- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.LAB:- if (c == '>') {- let a = this.push('a', '');- a.innerText = this.href;- a.href = this.href;- this.pop();- this.href = '';- this.state = HighlightMarkdown.NORMAL;- } else if (c == '\\') {- this.state = HighlightMarkdown.LAB_BACKSLASH;- } else {- this.href += c;- if (this.href.length <= 4 && !"http".startsWith(this.href)) {- this.append('<' + this.href);- this.href = '';- this.state = HighlightMarkdown.NORMAL;- }- }- break;-- case HighlightMarkdown.LAB_BACKSLASH:- if (!HighlightMarkdown.is_escapable(c))- this.href += '\\';- this.href += c;- this.state = HighlightMarkdown.LAB;- break;-- case HighlightMarkdown.LSB:- if (c == ']') {- this.state = HighlightMarkdown.LSB_RSB;- } else if (c == '\\') {- this.state = HighlightMarkdown.LSB_BACKSLASH;- } else {- this.text += c;- }- break;-- case HighlightMarkdown.LSB_BACKSLASH:- if (!HighlightMarkdown.is_escapable(c))- this.text += '\\';- this.text += c;- this.state = HighlightMarkdown.LSB;- break;-- case HighlightMarkdown.LSB_RSB:- if (c == '(') {- this.state = HighlightMarkdown.LSB_RSB_LPAREN;- } else {- this.append('[' + this.text + ']');- this.text = '';- this.epsilon(HighlightMarkdown.NORMAL);- }- break;-- case HighlightMarkdown.LSB_RSB_LPAREN:- if (c == ')') {- let a = this.push('a', '');- a.innerText = this.text;- a.href = this.href;- this.pop();- this.href = '';- this.text = '';this.state = HighlightMarkdown.NORMAL;- } else if (c == '\\') {- this.state = HighlightMarkdown.LSB_RSB_LPAREN_BACKSLASH;+ this.append("```");} else {- this.href += c;+ this.append("``");+ this.epsilon(HighlightMarkdown.CODE);}break;- case HighlightMarkdown.LSB_RSB_LPAREN_BACKSLASH:- if (!HighlightMarkdown.is_escapable(c))- this.href += '\\';- this.href += c;- this.state = HighlightMarkdown.LSB_RSB_LPAREN;- break;-default:- throw new Error('Invalid state: ' + this.state);+ throw new Error('Invalid state');}}}flush() {switch (this.state) {- case HighlightMarkdown.LANG:- this.lang = '';- break;case HighlightMarkdown.STAR:- case HighlightMarkdown.STRONG_STAR:- case HighlightMarkdown.EMPHASIS_STAR:this.append('*');break;- case HighlightMarkdown.EMPHASIS_STAR2:- this.append("**");- break;case HighlightMarkdown.TICK:+ case HighlightMarkdown.CODE_TICK:this.append('`');break;case HighlightMarkdown.TICK_TICK:+ case HighlightMarkdown.CODE_TICK_TICK:this.append("``");break;case HighlightMarkdown.INCODE:case HighlightMarkdown.INCODE2:case HighlightMarkdown.INCODE2_TICK:case HighlightMarkdown.INCODE2_TICK2:+ case HighlightMarkdown.STRONG:+ case HighlightMarkdown.STRONG_BACKSLASH:+ case HighlightMarkdown.STRONG_STAR:+ case HighlightMarkdown.EMPHASIS:+ case HighlightMarkdown.EMPHASIS_BACKSLASH:this.pop();break;- case HighlightMarkdown.CODE:- this.highlighter.flush();- this.highlighter = null;- this.pop();- break;- case HighlightMarkdown.CODE_TICK:- this.append('`');- this.highlighter.flush();- this.highlighter = null;- this.pop();- break;- case HighlightMarkdown.CODE_TICK_TICK:- this.append('``');- this.highlighter.flush();- this.highlighter = null;- this.pop();- break;- case HighlightMarkdown.LAB:- case HighlightMarkdown.LAB_BACKSLASH:- this.append('<' + this.href);- this.href = '';- break;- case HighlightMarkdown.LSB:- case HighlightMarkdown.LSB_BACKSLASH:- this.append('[' + this.text);- this.text = '';- break;- case HighlightMarkdown.LSB_RSB:- this.append('[' + this.text + ']');- this.text = '';- break;- case HighlightMarkdown.LSB_RSB_LPAREN:- case HighlightMarkdown.LSB_RSB_LPAREN_BACKSLASH:- this.append('[' + this.text + '](' + this.href);- this.text = '';- this.href = '';- break;- case HighlightMarkdown.TILDE:- this.append('~');- break;- case HighlightMarkdown.SPACE:- for (let i = 0; i < this.spaces; ++i)- this.append(' ');- break;- case HighlightMarkdown.OLNAME:- case HighlightMarkdown.OLNAME_DOT:- this.append(this.olname);- this.olname = '';- break;default:break;}- while (this.style.length) {- this.style.pop();- this.pop();- }this.state = HighlightMarkdown.NORMAL;this.bol = true;- this.spaces = 0;this.tail = false;- this.newlines = 0;this.delegate.flush();- this.delta = 1;- }-- setupCodeBlock(pre) {- const copyButton = document.createElement('button');- copyButton.className = 'copy-button';- copyButton.innerHTML =- `<svg xmlns="http://www.w3.org/2000/svg"- viewBox="0 0 24 24" fill="none"- stroke="currentColor"- stroke-width="2"- stroke-linecap="round"- stroke-linejoin="round">- <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>- <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>- </svg>`;- copyButton.addEventListener('click', function() {- try {- copyTextToClipboard(pre.innerText);- const originalInnerHTML = copyButton.innerHTML;- copyButton.innerHTML =- `<svg xmlns="http://www.w3.org/2000/svg"- viewBox="0 0 24 24" fill="none"- stroke="green"- stroke-width="2"- stroke-linecap="round"- stroke-linejoin="round">- <polyline points="20 6 9 17 4 12"></polyline>- </svg>`;- setTimeout(() => {- copyButton.innerHTML = originalInnerHTML;- }, 2000);- } catch (err) {- console.error('Failed to copy text:', err);- }- });- pre.appendChild(copyButton);}}diff --git a/llamafile/server/www/highlight_python.js b/llamafile/server/www/highlight_python.jsindex 50091d17..161bab13 100644--- a/llamafile/server/www/highlight_python.js+++ b/llamafile/server/www/highlight_python.js@@ -53,6 +53,80 @@ const PYTHON_CONSTANTS = new Set(['True',]);+const PYTHON_BUILTINS = new Set([+ '__import__',+ 'abs',+ 'aiter',+ 'all',+ 'anext',+ 'any',+ 'ascii',+ 'bin',+ 'bool',+ 'breakpoint',+ 'bytearray',+ 'bytes',+ 'callable',+ 'chr',+ 'classmethod',+ 'compile',+ 'complex',+ 'delattr',+ 'dict',+ 'dir',+ 'divmod',+ 'enumerate',+ 'eval',+ 'exec',+ 'filter',+ 'float',+ 'format',+ 'frozenset',+ 'getattr',+ 'globals',+ 'hasattr',+ 'hash',+ 'help',+ 'hex',+ 'id',+ 'input',+ 'int',+ 'isinstance',+ 'issubclass',+ 'iter',+ 'len',+ 'list',+ 'locals',+ 'map',+ 'max',+ 'memoryview',+ 'min',+ 'next',+ 'object',+ 'oct',+ 'open',+ 'ord',+ 'pow',+ 'print',+ 'property',+ 'range',+ 'repr',+ 'reversed',+ 'round',+ 'set',+ 'setattr',+ 'slice',+ 'sorted',+ 'staticmethod',+ 'str',+ 'sum',+ 'super',+ 'tuple',+ 'type',+ 'vars',+ 'zip',+]);+class HighlightPython extends Highlighter {static NORMAL = 0;@@ -114,6 +188,10 @@ class HighlightPython extends Highlighter {this.push("span", "keyword");this.append(this.word);this.pop();+ } else if (PYTHON_BUILTINS.has(this.word)) {+ this.push("span", "builtin");+ this.append(this.word);+ this.pop();} else if (PYTHON_CONSTANTS.has(this.word)) {this.push("span", "constant");this.append(this.word);@@ -293,6 +371,10 @@ class HighlightPython extends Highlighter {this.push("span", "keyword");this.append(this.word);this.pop();+ } else if (PYTHON_BUILTINS.has(this.word)) {+ this.push("span", "builtin");+ this.append(this.word);+ this.pop();} else if (PYTHON_CONSTANTS.has(this.word)) {this.push("span", "constant");this.append(this.word);diff --git a/llamafile/server/www/highlight_swift.js b/llamafile/server/www/highlight_swift.jsindex ce2fb38c..3fcdbf33 100644--- a/llamafile/server/www/highlight_swift.js+++ b/llamafile/server/www/highlight_swift.js@@ -988,6 +988,9 @@ class HighlightSwift extends Highlighter {static REGEX_END = 19;static REGEX_BACKSLASH = 20;+ static EXPECT_VALUE = 0;+ static EXPECT_OPERATOR = 1;+constructor(delegate) {super(delegate);this.hash1 = 0;@@ -995,6 +998,7 @@ class HighlightSwift extends Highlighter {this.word = '';this.nest = [];this.hash = [];+ this.expect = HighlightSwift.EXPECT_VALUE;}feed(input) {@@ -1014,19 +1018,29 @@ class HighlightSwift extends Highlighter {this.push("span", "string");this.append('"');this.hash1 = 0;+ this.expect = HighlightSwift.EXPECT_OPERATOR;} else if (c == '#') {this.state = HighlightSwift.HASH;this.hash1 = 1;+ this.expect = HighlightSwift.EXPECT_OPERATOR;} else if (c == '(' && this.nest.length) {this.append('(');this.nest.push(HighlightSwift.NORMAL);this.hash.push(0);+ this.expect = HighlightSwift.EXPECT_VALUE;} else if (c == ')' && this.nest.length) {+ this.expect = HighlightSwift.EXPECT_OPERATOR;this.state = this.nest.pop();this.hash1 = this.hash.pop();if (this.state != HighlightSwift.NORMAL)this.push("span", "string");this.append(')');+ } else if (c == ')' || c == ']' || isdigit(c) || c == '.') {+ this.expect = HighlightSwift.EXPECT_OPERATOR;+ this.append(c);+ } else if (ispunct(c) || c == '\n') {+ this.expect = HighlightSwift.EXPECT_VALUE;+ this.append(c);} else {this.append(c);}@@ -1069,11 +1083,14 @@ class HighlightSwift extends Highlighter {this.push("span", "comment");this.append("/*");this.state = HighlightSwift.SLASH_STAR;- } else {+ } else if (expect_ == EXPECT_VALUE) {this.push("span", "string");this.append('/');this.hash1 = 0;this.epsilon(HighlightSwift.REGEX);+ } else {+ this.append('/');+ this.epsilon(HighlightSwift.NORMAL);}break;@@ -1344,6 +1361,7 @@ class HighlightSwift extends Highlighter {this.hash = [];this.delegate.flush();this.delta = 1;+ this.expect = HighlightSwift.EXPECT_VALUE;}}diff --git a/llamafile/server/www/index.html b/llamafile/server/www/index.htmlindex 4883e559..c193a834 100644--- a/llamafile/server/www/index.html+++ b/llamafile/server/www/index.html@@ -74,5 +74,6 @@<script src="highlight_typescript.js"></script><script src="highlight_zig.js"></script><script src="highlight_cmake.js"></script>+<script src="render_markdown.js"></script><script src="chatbot.js"></script>diff --git a/llamafile/server/www/render_markdown.js b/llamafile/server/www/render_markdown.jsnew file mode 100644index 00000000..ee539b10--- /dev/null+++ b/llamafile/server/www/render_markdown.js@@ -0,0 +1,929 @@+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++class RenderMarkdown extends Highlighter {++ static NORMAL = 0;+ static TICK = 1;+ static TICK_TICK = 2;+ static LANG = 3;+ static CODE = 4;+ static CODE_TICK = 5;+ static STAR = 7;+ static EMPHASIS_STAR = 8;+ static STRONG_STAR = 9;+ static BACKSLASH = 10;+ static INCODE = 11;+ static INCODE2 = 12;+ static INCODE2_TICK = 13;+ static LANG2 = 14;+ static NEWLINE = 15;+ static INCODE2_TICK2 = 17;+ static LSB = 18;+ static LSB_BACKSLASH = 19;+ static LSB_RSB = 20;+ static LSB_RSB_LPAREN = 21;+ static LSB_RSB_LPAREN_BACKSLASH = 22;+ static LAB = 23;+ static LAB_BACKSLASH = 24;+ static HYPHEN = 25;+ static TILDE = 26;+ static EMPHASIS_STAR2 = 27;+ static OLNAME = 28;+ static OLNAME_DOT = 29;+ static HASH = 30;+ static HYPHEN_HYPHEN = 31;+ static HYPHEN_HYPHEN_HYPHEN = 32;+ static OLDCODE = 33;+ static BLOCKQUOTE = 34;+ static TICK_TICK_TICK = 35;++ static STYLE_SPAN = 16;+ static STYLE_STRONG = 16;+ static STYLE_EMPHASIS = 17;+ static STYLE_STRIKE = 18;++ static STYLE_LIST = 32;+ static STYLE_UL = 32;+ static STYLE_OL = 33;++ static STYLE_DIV = 64;+ static STYLE_LI = 64;+ static STYLE_HEADER = 65;+ static STYLE_BLOCKQUOTE = 66;++ static get_style_tag(t) {+ switch (t) {+ case RenderMarkdown.STYLE_UL:+ return 'UL';+ case RenderMarkdown.STYLE_OL:+ return 'OL';+ default:+ throw new Error('bad style');+ }+ }++ static is_sublist(depth, next) {+ if (!depth) {+ return next > 0;+ } else {+ return next >= depth + 4;+ }+ }++ constructor(delegate) {+ super(delegate);+ this.bol = true;+ this.tail = false;+ this.lang = '';+ this.highlighter = null;+ this.newlines = 0;+ this.spaces = 0;+ this.text = '';+ this.href = '';+ this.style = [];+ this.trailing_spaces = 0;+ this.olname = '';+ this.dedent = 0;+ this.hashes = 0;+ this.bqline = false;+ this.tick1 = 0;+ this.tick2 = 0;+ }++ inside(style) {+ return this.style.length && this.style[this.style.length - 1][0] == style;+ }++ blockquoted() {+ for (let i = 0; i < this.style.length; ++i)+ if (this.style[i][0] == RenderMarkdown.STYLE_BLOCKQUOTE)+ return true;+ return false;+ }++ li(t) {+ // plan out <li> insertion+ let need_list = true;+ while (this.style.length) {+ let e = this.style[this.style.length - 1];++ // eliminate <b>, <i>, and <s>+ if (e[0] & RenderMarkdown.STYLE_SPAN) {+ this.pop();+ this.style.pop();+ continue;+ }++ // eliminate <blockquote> if we're beneath it+ if (e[0] == RenderMarkdown.STYLE_BLOCKQUOTE && this.spaces < e[1]) {+ this.pop();+ this.style.pop();+ continue;+ }++ // otherwise it must be <ul><li> or <ol><li>+ if (e[0] != RenderMarkdown.STYLE_LI)+ break;+ if (this.style.length < 2)+ break;+ let list = this.style[this.style.length - 2];+ if (!(list[0] & RenderMarkdown.STYLE_LIST))+ break;++ // pop if new list is beneath the current one, e.g.+ //+ // - one+ // - two+ // - three+ //+ if (this.spaces < list[1]) {+ this.pop();+ this.style.pop();+ this.pop();+ this.style.pop();+ continue;+ }++ // don't pop if we're adding a sublist, e.g.+ //+ // - one+ // - two+ //+ if (RenderMarkdown.is_sublist(list[1], this.spaces)) {+ break;+ }++ // pop if list kind is changing and we're at the same level, e.g.+ //+ // 1. one+ // - two+ //+ if (list[0] != t) {+ this.pop();+ this.style.pop();+ this.pop();+ this.style.pop();+ break;+ }++ // pop <li> if we're adding a sibling, e.g.+ //+ // - one+ // - two+ //+ this.pop();+ this.style.pop();+ need_list = false;+ break;+ }++ // create new <li> element+ if (need_list) {+ this.style.push([t, this.spaces]);+ this.push(RenderMarkdown.get_style_tag(t), '');+ }+ this.style.push([RenderMarkdown.STYLE_LI, this.spaces]);+ this.push("LI", "");+ this.bol = false;+ }++ got() {+ if (this.bol) {+ this.bol = false;+ while (this.style.length &&+ ((!this.bqline &&+ this.style[this.style.length - 1][0] == RenderMarkdown.STYLE_BLOCKQUOTE) ||+ this.style[this.style.length - 1][0] == RenderMarkdown.STYLE_LI ||+ (this.style[this.style.length - 1][0] & RenderMarkdown.STYLE_LIST)) &&+ this.spaces <= this.style[this.style.length - 1][1]) {+ this.pop();+ this.style.pop();+ }+ }+ this.spaces = 0;+ }++ feed(input) {+ for (let i = 0; i < input.length; i += this.delta) {+ this.delta = 1;+ const c = input[i];+ if (c == '\r')+ continue;+ switch (this.state) {++ case RenderMarkdown.NORMAL:+ if (this.bol && this.spaces == 4 && this.newlines >= 2 &&+ isgraph(c) && c != '`' && !this.style.length) {+ this.dedent = this.spaces;+ this.spaces = 0;+ this.bol = false;+ this.push("PRE", "");+ this.epsilon(RenderMarkdown.OLDCODE);+ } else if (c == '`') {+ this.state = RenderMarkdown.TICK;+ break;+ } else if (c == '*') {+ if (this.inside(RenderMarkdown.STYLE_EMPHASIS)) {+ this.state = RenderMarkdown.EMPHASIS_STAR;+ } else if (this.inside(RenderMarkdown.STYLE_STRONG)) {+ this.state = RenderMarkdown.STRONG_STAR;+ } else {+ this.state = RenderMarkdown.STAR;+ }+ break;+ } else if (c == '>' && this.bol) {+ this.state = RenderMarkdown.BLOCKQUOTE;+ break;+ } else if (c == '-' && this.bol) {+ this.state = RenderMarkdown.HYPHEN;+ break;+ } else if (c == '#' && this.bol && !this.spaces) {+ this.state = RenderMarkdown.HASH;+ this.hashes = 1;+ } else if (isdigit(c) && this.bol) {+ this.epsilon(RenderMarkdown.OLNAME);+ } else if (c == '\n') {+ if (this.inside(RenderMarkdown.STYLE_HEADER)) {+ this.pop();+ this.style.pop();+ }+ this.bol = true;+ this.tail = false;+ this.bqline = false;+ this.state = RenderMarkdown.NEWLINE;+ this.newlines = 1;+ this.trailing_spaces = this.spaces;+ this.spaces = 0;+ break;+ } else if (c == '~') {+ this.state = RenderMarkdown.TILDE;+ this.got();+ } else if (c == '[') {+ this.state = RenderMarkdown.LSB;+ this.got();+ } else if (c == '<') {+ this.state = RenderMarkdown.LAB;+ this.got();+ } else if (c == '\\') {+ this.state = RenderMarkdown.BACKSLASH;+ this.got();+ } else if (c == ' ') {+ ++this.spaces;+ this.append(c);+ } else if (c == '\t') {+ this.spaces = (this.spaces + 3) & -4;+ this.append(c);+ } else {+ this.got();+ this.append(c);+ }+ this.tail = true;+ break;++ case RenderMarkdown.NEWLINE:+ if (c == '\n') {+ ++this.newlines;+ this.spaces = 0;+ } else if (c == ' ') {+ ++this.spaces;+ } else if (c == '\t') {+ this.spaces = (this.spaces + 3) & -4;+ } else {+ if (this.newlines >= 2) {+ if (this.style.length &&+ this.style[this.style.length - 1][0] == RenderMarkdown.STYLE_BLOCKQUOTE) {+ this.pop();+ this.style.pop();+ }+ this.push("P", "");+ this.pop();+ } else if (this.trailing_spaces >= 2) {+ this.push("BR", "");+ this.pop();+ } else {+ this.append('\n');+ }+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.BLOCKQUOTE:+ if (isblank(c)) {+ if (!this.blockquoted()) {+ this.push("BLOCKQUOTE", "");+ this.style.push([RenderMarkdown.STYLE_BLOCKQUOTE, this.spaces]);+ }+ this.state = RenderMarkdown.NORMAL;+ this.bqline = true;+ } else {+ this.got();+ this.append('>');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.HYPHEN:+ if (isblank(c)) {+ // - handle list item+ this.li(RenderMarkdown.STYLE_UL);+ this.append(c);+ this.state = RenderMarkdown.NORMAL;+ } else if (c == '-') {+ this.state = RenderMarkdown.HYPHEN_HYPHEN;+ } else {+ this.got();+ this.append('-');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.HYPHEN_HYPHEN:+ if (c == '-') {+ this.state = RenderMarkdown.HYPHEN_HYPHEN_HYPHEN;+ } else {+ this.got();+ this.append("--");+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.HYPHEN_HYPHEN_HYPHEN:+ if (c == '\n') {+ // handle --- line break+ this.got();+ this.push("HR", "");+ this.pop();+ this.epsilon(RenderMarkdown.NORMAL);+ } else {+ this.got();+ this.append("---");+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.OLNAME:+ if (isdigit(c)) {+ this.olname += c;+ } else if (c == '.') {+ this.olname += '.';+ this.state = RenderMarkdown.OLNAME_DOT;+ } else {+ this.got();+ this.append(this.olname);+ this.olname = '';+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.OLNAME_DOT:+ if (isspace(c)) {+ this.li(RenderMarkdown.STYLE_OL);+ this.olname = '';+ this.state = RenderMarkdown.NORMAL;+ } else {+ this.got();+ this.append(this.olname);+ this.olname = '';+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.BACKSLASH:+ if (ispunct(c)) {+ this.append(c);+ this.state = RenderMarkdown.NORMAL;+ } else {+ this.append('\\');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.STAR:+ if (c == '*') {+ // handle **strong** text+ // we don't call this.got() in case this is *** bar+ this.got();+ this.push("STRONG", "");+ this.state = RenderMarkdown.NORMAL;+ this.style.push([RenderMarkdown.STYLE_STRONG, 0]);+ } else if (this.bol && isblank(c)) {+ // * handle list item+ this.li(RenderMarkdown.STYLE_UL);+ this.append(c);+ this.state = RenderMarkdown.NORMAL;+ } else {+ // handle *emphasis* text+ this.got();+ this.push("EM", "");+ this.style.push([RenderMarkdown.STYLE_EMPHASIS, 0]);+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.EMPHASIS_STAR:+ if (c == '*') {+ this.state = RenderMarkdown.EMPHASIS_STAR2;+ } else {+ // leave *italic* text+ // ^+ this.style.pop();+ this.pop();+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.EMPHASIS_STAR2:+ if (c == '*') {+ this.style.pop();+ this.pop();+ if (this.inside(RenderMarkdown.STYLE_STRONG)) {+ // handle ***strong emphasis*** closing+ // ^+ this.style.pop();+ this.pop();+ this.state = RenderMarkdown.NORMAL;+ } else {+ // handle *emphasis***strong**+ // ^+ this.push("STRONG", "");+ this.state = RenderMarkdown.NORMAL;+ this.style.push([RenderMarkdown.STYLE_STRONG, 0]);+ this.state = RenderMarkdown.NORMAL;+ }+ } else {+ // handle *italic **strong** text*+ // ^+ this.push("STRONG", "");+ this.style.push([RenderMarkdown.STYLE_STRONG, 0]);+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.STRONG_STAR:+ if (c == '*') {+ this.state = RenderMarkdown.NORMAL;+ this.style.pop();+ this.pop();+ } else if (c == '\n' && !this.tail) {+ // handle *** line break+ this.got();+ this.style.pop();+ this.pop();+ this.push("HR", "");+ this.pop();+ this.epsilon(RenderMarkdown.NORMAL);+ } else {+ this.push("EM", "");+ this.style.push([RenderMarkdown.STYLE_EMPHASIS, 0]);+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.TILDE:+ if (c == '~') {+ // handle ~~strikethrough text~~+ if (this.inside(RenderMarkdown.STYLE_STRIKE)) {+ this.pop();+ this.style.pop();+ } else {+ this.push("S", "");+ this.style.push([RenderMarkdown.STYLE_STRIKE, 0]);+ }+ this.state = RenderMarkdown.NORMAL;+ } else {+ this.append('~');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.HASH:+ if (c == '#') {+ ++this.hashes;+ } else if (isblank(c)) {+ this.got();+ this.push("H" + this.hashes, "");+ this.state = RenderMarkdown.NORMAL;+ this.style.push([RenderMarkdown.STYLE_HEADER, 0]);+ } else {+ this.got();+ for (let i = 0; i < this.hashes; ++i)+ this.append('#');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.TICK:+ if (c == '`') {+ if (this.bol) {+ this.state = RenderMarkdown.TICK_TICK;+ } else {+ this.got();+ this.push("CODE", "");+ this.state = RenderMarkdown.INCODE2;+ }+ } else {+ this.got();+ this.push("CODE", "");+ this.append(c);+ this.state = RenderMarkdown.INCODE;+ }+ break;++ case RenderMarkdown.INCODE:+ // this is for `inline code` like that+ // no backslash escapes are supported here+ if (c == '`') {+ this.pop();+ this.state = RenderMarkdown.NORMAL;+ } else {+ this.append(c);+ }+ break;++ case RenderMarkdown.INCODE2:+ // this is for ``inline ` code`` like that+ // it lets you put backtick inside the code+ if (c == '`') {+ this.state = RenderMarkdown.INCODE2_TICK;+ } else {+ this.append(c);+ }+ break;++ case RenderMarkdown.INCODE2_TICK:+ if (c == '`') {+ this.state = RenderMarkdown.INCODE2_TICK2;+ } else {+ this.append('`');+ this.append(c);+ this.state = RenderMarkdown.INCODE2;+ }+ break;++ case RenderMarkdown.INCODE2_TICK2:+ if (c == '`') {+ this.append('`');+ } else {+ this.pop();+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.TICK_TICK:+ if (c == '`') {+ this.state = RenderMarkdown.TICK_TICK_TICK;+ this.tick1 = 3;+ } else {+ this.got();+ this.push("CODE", "");+ this.append(c);+ this.state = RenderMarkdown.INCODE2;+ }+ break;++ case RenderMarkdown.TICK_TICK_TICK:+ if (c == '`') {+ ++this.tick1;+ } else {+ this.dedent = this.spaces;+ this.got();+ this.epsilon(RenderMarkdown.LANG);+ this.spaces = 0;+ }+ break;++ case RenderMarkdown.LANG:+ if (!isascii(c) || !isspace(c)) {+ this.lang += c.toLowerCase();+ } else {+ this.epsilon(RenderMarkdown.LANG2);+ }+ break;++ case RenderMarkdown.LANG2:+ if (c == "\n") {+ let pre = this.push("PRE", "");+ this.setupCodeBlock(pre);+ let hdom = new HighlightDom(pre);+ if (!(this.highlighter = Highlighter.create(this.lang, hdom)))+ this.highlighter = Highlighter.create('txt', hdom);+ this.state = RenderMarkdown.CODE;+ this.bol = true;+ this.lang = '';+ }+ break;++ case RenderMarkdown.CODE:+ if (this.bol) {+ if (c == ' ') {+ ++this.spaces;+ if (this.spaces <= this.dedent)+ break;+ } else if (c == '\t') {+ this.spaces = (this.spaces + 3) & -4;+ if (this.spaces <= this.dedent)+ break;+ } else {+ this.bol = false;+ }+ }+ if (c == '\n') {+ this.bol = true;+ this.spaces = 0;+ }+ if (c == '`') {+ this.state = RenderMarkdown.CODE_TICK;+ this.tick2 = 1;+ } else {+ this.highlighter.feed(c);+ }+ break;++ case RenderMarkdown.CODE_TICK:+ if (c == '`') {+ if (++this.tick2 == this.tick1) {+ this.spaces = 0;+ this.state = HighlightMarkdown.NORMAL;+ this.highlighter.flush();+ this.highlighter = null;+ this.pop();+ }+ } else {+ for (let i = 0; i < this.tick2; ++i)+ this.highlighter.feed('`');+ this.epsilon(RenderMarkdown.CODE);+ }+ break;++ case RenderMarkdown.LAB:+ if (c == '>') {+ let a = this.push("A", "");+ a.innerText = this.href;+ a.href = this.href;+ this.pop();+ this.href = '';+ this.state = RenderMarkdown.NORMAL;+ } else if (c == '\\') {+ this.state = RenderMarkdown.LAB_BACKSLASH;+ } else {+ this.href += c;+ if (this.href.length <= 4 && !"http".startsWith(this.href)) {+ this.append('<' + this.href);+ this.href = '';+ this.state = RenderMarkdown.NORMAL;+ }+ }+ break;++ case RenderMarkdown.LAB_BACKSLASH:+ if (!ispunct(c))+ this.href += '\\';+ this.href += c;+ this.state = RenderMarkdown.LAB;+ break;++ case RenderMarkdown.LSB:+ if (c == ']') {+ this.state = RenderMarkdown.LSB_RSB;+ } else if (c == '\\') {+ this.state = RenderMarkdown.LSB_BACKSLASH;+ } else {+ this.text += c;+ }+ break;++ case RenderMarkdown.LSB_BACKSLASH:+ if (!ispunct(c))+ this.text += '\\';+ this.text += c;+ this.state = RenderMarkdown.LSB;+ break;++ case RenderMarkdown.LSB_RSB:+ if (c == '(') {+ this.state = RenderMarkdown.LSB_RSB_LPAREN;+ } else {+ this.append('[' + this.text + ']');+ this.text = '';+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;++ case RenderMarkdown.LSB_RSB_LPAREN:+ if (c == ')') {+ let a = this.push("A", "");+ a.innerText = this.text;+ a.href = this.href;+ this.pop();+ this.href = '';+ this.text = '';+ this.state = RenderMarkdown.NORMAL;+ } else if (c == '\\') {+ this.state = RenderMarkdown.LSB_RSB_LPAREN_BACKSLASH;+ } else {+ this.href += c;+ }+ break;++ case RenderMarkdown.LSB_RSB_LPAREN_BACKSLASH:+ if (!ispunct(c))+ this.href += '\\';+ this.href += c;+ this.state = RenderMarkdown.LSB_RSB_LPAREN;+ break;++ case RenderMarkdown.OLDCODE:+ if (this.bol) {+ if (c == ' ') {+ ++this.spaces;+ if (this.spaces <= this.dedent)+ break;+ } else if (c == '\t') {+ this.spaces = (this.spaces + 3) & -4;+ if (this.spaces <= this.dedent)+ break;+ } else {+ if (this.spaces < this.dedent) {+ this.pop();+ this.epsilon(RenderMarkdown.NORMAL);+ break;+ }+ this.bol = false;+ }+ }+ if (c == '\n') {+ this.bol = true;+ this.spaces = 0;+ }+ this.append(c);+ break;++ default:+ throw new Error('Invalid state: ' + this.state);+ }+ }+ }++ flush() {+ switch (this.state) {+ case RenderMarkdown.LANG:+ this.lang = '';+ break;+ case RenderMarkdown.STAR:+ case RenderMarkdown.STRONG_STAR:+ case RenderMarkdown.EMPHASIS_STAR:+ this.append('*');+ break;+ case RenderMarkdown.EMPHASIS_STAR2:+ this.append("**");+ break;+ case RenderMarkdown.TICK:+ this.append('`');+ break;+ case RenderMarkdown.TICK_TICK:+ this.append("``");+ break;+ case RenderMarkdown.TICK_TICK_TICK:+ for (let i = 0; i < this.tick1; ++i)+ this.append('`');+ break;+ case RenderMarkdown.HYPHEN:+ this.append('-');+ break;+ case RenderMarkdown.HYPHEN_HYPHEN:+ this.append("--");+ break;+ case RenderMarkdown.HYPHEN_HYPHEN_HYPHEN:+ this.append("---");+ break;+ case RenderMarkdown.INCODE:+ case RenderMarkdown.INCODE2:+ case RenderMarkdown.INCODE2_TICK:+ case RenderMarkdown.INCODE2_TICK2:+ this.pop();+ break;+ case RenderMarkdown.CODE:+ this.highlighter.flush();+ this.highlighter = null;+ this.pop();+ break;+ case RenderMarkdown.CODE_TICK:+ for (let i = 0; i < this.tick2; ++i)+ this.append('`');+ this.highlighter.flush();+ this.highlighter = null;+ this.pop();+ break;+ case RenderMarkdown.LAB:+ case RenderMarkdown.LAB_BACKSLASH:+ this.append('<' + this.href);+ this.href = '';+ break;+ case RenderMarkdown.LSB:+ case RenderMarkdown.LSB_BACKSLASH:+ this.append('[' + this.text);+ this.text = '';+ break;+ case RenderMarkdown.LSB_RSB:+ this.append('[' + this.text + ']');+ this.text = '';+ break;+ case RenderMarkdown.LSB_RSB_LPAREN:+ case RenderMarkdown.LSB_RSB_LPAREN_BACKSLASH:+ this.append('[' + this.text + '](' + this.href);+ this.text = '';+ this.href = '';+ break;+ case RenderMarkdown.TILDE:+ this.append('~');+ break;+ case RenderMarkdown.HASH:+ for (let i = 0; i < this.hashes; ++i)+ this.append('#');+ break;+ case RenderMarkdown.OLNAME:+ case RenderMarkdown.OLNAME_DOT:+ this.append(this.olname);+ this.olname = '';+ break;+ case RenderMarkdown.OLDCODE:+ this.pop();+ break;+ default:+ break;+ }+ while (this.style.length) {+ this.style.pop();+ this.pop();+ }+ this.state = RenderMarkdown.NORMAL;+ this.bol = true;+ this.spaces = 0;+ this.tail = false;+ this.newlines = 0;+ this.delegate.flush();+ this.delta = 1;+ this.dedent = 0;+ this.hashes = 0;+ this.bqline = false;+ this.tick1 = 0;+ this.tick2 = 0;+ }++ static COPY_BUTTON = null;++ static makeCopyButton() {+ if (!RenderMarkdown.COPY_BUTTON) {+ RenderMarkdown.COPY_BUTTON = document.createElement('button');+ RenderMarkdown.COPY_BUTTON.className = 'copy-button';+ RenderMarkdown.COPY_BUTTON.innerHTML =+ `<svg xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24" fill="none"+ stroke="currentColor"+ stroke-width="2"+ stroke-linecap="round"+ stroke-linejoin="round">+ <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>+ <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>+ </svg>`;+ }+ return RenderMarkdown.COPY_BUTTON.cloneNode(true);+ }++ setupCodeBlock(pre) {+ const copyButton = RenderMarkdown.makeCopyButton();+ copyButton.addEventListener('click', function() {+ try {+ copyTextToClipboard(pre.innerText);+ const originalInnerHTML = copyButton.innerHTML;+ copyButton.innerHTML =+ `<svg xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24" fill="none"+ stroke="green"+ stroke-width="2"+ stroke-linecap="round"+ stroke-linejoin="round">+ <polyline points="20 6 9 17 4 12"></polyline>+ </svg>`;+ setTimeout(() => {+ copyButton.innerHTML = originalInnerHTML;+ }, 2000);+ } catch (err) {+ console.error('Failed to copy text:', err);+ }+ });+ pre.appendChild(copyButton);+ }+}diff --git a/llamafile/server/www/test/test.css b/llamafile/server/www/test/test.cssnew file mode 100644index 00000000..136b6f74--- /dev/null+++ b/llamafile/server/www/test/test.css@@ -0,0 +1,12 @@+pre {+ border: 1px solid #aaa;+ padding: .2rem;+}++blockquote {+ font-style: italic;+}++.copy-button {+ display: none;+}diff --git a/llamafile/server/www/test/test.html b/llamafile/server/www/test/test.htmlindex 7723b8d2..4081d808 100644--- a/llamafile/server/www/test/test.html+++ b/llamafile/server/www/test/test.html@@ -1,6 +1,7 @@<!DOCTYPE html><html lang="en"><meta charset="utf-8">+<link rel="stylesheet" href="test.css"><link rel="stylesheet" href="../highlight.css"><div id="code"></div>@@ -51,5 +52,6 @@<script src="../highlight_typescript.js"></script><script src="../highlight_zig.js"></script><script src="../highlight_cmake.js"></script>+<script src="../render_markdown.js"></script><script src="test.js"></script>diff --git a/llamafile/server/www/test/test.js b/llamafile/server/www/test/test.jsindex 2a78372c..3dbc598b 100644--- a/llamafile/server/www/test/test.js+++ b/llamafile/server/www/test/test.js@@ -1,12 +1,11 @@const xhr = new XMLHttpRequest();-const lang = 'markdown';xhr.open('GET', 'test.md');xhr.onload = function() {if (xhr.status == 200) {let code = document.getElementById('code');let hdom = new HighlightDom(code);- const h = Highlighter.create(lang, hdom);+ const h = new RenderMarkdown(hdom);h.feed(xhr.responseText);h.flush();} else {diff --git a/llamafile/server/www/test/test.md b/llamafile/server/www/test/test.mdindex cd2c47ff..f2a198a2 100644--- a/llamafile/server/www/test/test.md+++ b/llamafile/server/www/test/test.md@@ -1,6 +1,3 @@-1. hi- - there-bold: **bold**italic: *italic*strike: ~~strike~~@@ -19,21 +16,20 @@ non-escapable: \a \0 \Z \~* one* two- three+ three- one-¶1-lorem ipsum-dolor sit amet--¶2-consectetur adipiscing elit-sed do eiusmod tempor incididunt-***more text <http://google.com> link-***+word+ word++ needs blank line+normal++---```cpp#include <iostream>@@ -51,13 +47,20 @@ int main() {**Explanation:**-1. **`#include <iostream>`:** This line includes the `iostream` library, which provides input and output functionalities in C++. It allows us to use objects like `std::cout` for printing to the console.+1. **`#include <iostream>`:** This line includes the `iostream` library,+ which provides input and output functionalities in C++. It allows us+ to use objects like `std::cout` for printing to the console.-2. **`int main() { ... }`:** This defines the `main` function, which is the entry point of any C++ program. The code inside the curly braces `{}` will be executed when the program runs.+2. **`int main() { ... }`:** This defines the `main` function, which is+ the entry point of any C++ program. The code inside the curly braces+ `{}` will be executed when the program runs.-3. **`int age = 25;`:** This line declares an integer variable named `age` and initializes it with the value 25.+3. **`int age = 25;`:** This line declares an integer variable named+ `age` and initializes it with the value 25.-4. **`std::cout << "Hello, world! My name is AI, and I am " << age << " years old." << std::endl;`:** This line prints a message to the console. Let's break it down:+4. **`std::cout << "Hello, world! My name is AI, and I am " << age << "+ years old." << std::endl;`:** This line prints a message to the+ console. Let's break it down:- `std::cout` is the standard output stream object.- `<<` is the insertion operator, used to send data to the output stream.- `"Hello, world! My name is AI, and I am "` is a string literal that will be printed as is.@@ -65,6 +68,96 @@ int main() {- `" years old."` is another string literal.- `std::endl` inserts a newline character, moving the cursor to the next line for any subsequent output.-5. **`return 0;`:** This line indicates that the program executed successfully. Returning 0 from the `main` function is a convention in C++ to signal successful execution.+5. **`return 0;`:** This line indicates that the program executed+ successfully. Returning 0 from the `main` function is a convention in+ C++ to signal successful execution.++**In summary:** This code snippet declares a variable, assigns it a+value, and then prints a message to the console including the variable's+value. It demonstrates basic C++ syntax for variable declaration,+output, and program structure.++---++Sure! Let's write a simple FORTRAN66 program that calculates the+factorial of a given number. I'll also provide an explanation of each+part of the code.++```fortran+ PROGRAM FACTORIAL+ INTEGER N, I, FACT+ PRINT *, 'Enter a positive integer: '+ READ *, N++ IF (N < 0) THEN+ PRINT *, 'Factorial is not defined for negative numbers.'+ ELSE+ FACT = 1+ DO 10 I = 2, N+10 FACT = FACT * I+ PRINT *, 'Factorial of ', N, ' is ', FACT+ END IF++ STOP+ END+```-**In summary:** This code snippet declares a variable, assigns it a value, and then prints a message to the console including the variable's value. It demonstrates basic C++ syntax for variable declaration, output, and program structure.+### Explanation:++1. **Program Declaration:**+ ```fortran+ PROGRAM FACTORIAL+ ```+ This line declares the name of the program, which is `FACTORIAL`.++2. **Variable Declarations:**+ ```fortran+ INTEGER N, I, FACT+ ```+ This line declares three integer variables: `N` for the input number,+ `I` for the loop counter, and `FACT` for the factorial result.++3. **User Input:**+ ```fortran+ PRINT *, 'Enter a positive integer: '+ READ *, N+ ```+ - `PRINT *, 'Enter a positive integer: '` prints a prompt asking the+ user to enter a positive integer.+ - `READ *, N` reads the integer input from the user and stores it in+ the variable `N`.++4. **Condition to Check for Negative Input:**+ ```fortran+ IF (N < 0) THEN+ PRINT *, 'Factorial is not defined for negative numbers.'+ ELSE+ FACT = 1+ DO 10 I = 2, N+10 FACT = FACT * I+ PRINT *, 'Factorial of ', N, ' is ', FACT+ END IF+ ```+ - `IF (N < 0) THEN` checks if the input number is negative.+ - If `N` is negative, it prints a message indicating that the+ factorial is not defined for negative numbers.+ - If `N` is non-negative, it initializes `FACT` to 1 and uses a `DO`+ loop to calculate the factorial by multiplying `FACT` by each+ number from 2 to `N`.+ - Finally, it prints the factorial result.++5. **Stop Statement:**+ ```fortran+ STOP+ ```+ This line stops the execution of the program.++6. **End of Program:**+ ```fortran+ END+ ```+ This line marks the end of the program.++This program is a simple example of how to use variables, input/output+operations, conditional statements, and loops in FORTRAN66. It+demonstrates basic input/output operations and error handling.