Skip to content

Distribute and run LLMs with a single file.

Add CSS syntax highlighting

Justine Tunney32c2ad8bf10a
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index 4a1cee5d..540bbacd 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -42,6 +42,8 @@ Highlight *Highlight::create(const std::string_view &lang) {
return new HighlightC(is_keyword_go, is_keyword_go_type);
} else if (lang == "sql") {
return new HighlightSql();
+ } else if (lang == "css") {
+ return new HighlightCss();
} else {
return new HighlightPlain;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 09733c80..f38914d9 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -27,8 +27,10 @@
#define HI_MACRO "\033[35m" // magenta
#define HI_ATTRIB "\033[35m" // magenta
#define HI_CONTIN "\033[33m" // yellow
+#define HI_LABEL "\033[33m" // yellow
#define HI_TYPE "\033[36m" // cyan
-#define HI_LABEL "\033[35m" // magenta
+#define HI_SELECTOR "\033[33m" // yellow
+#define HI_PROPERTY "\033[36m" // cyan
typedef const char *is_keyword_f(const char *, size_t);
@@ -166,3 +168,14 @@ class HighlightSql : public Highlight {
int t_ = 0;
std::string word_;
};
+
+class HighlightCss : public Highlight {
+ public:
+ HighlightCss();
+ ~HighlightCss() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+};
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cpp
index 1e635713..e78c3bc6 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight_c.cpp
@@ -160,6 +160,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {
}
void HighlightC::flush(std::string *r) {
+ t_ &= ~BACKSLASH;
switch (t_) {
case WORD:
if (is_keyword_(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_css.cpp b/llamafile/highlight_css.cpp
new file mode 100644
index 00000000..077fadd0
--- /dev/null
+++ b/llamafile/highlight_css.cpp
@@ -0,0 +1,208 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+#define NORMAL 0
+#define SELECTOR 1
+#define PROPERTY 2
+#define VALUE 3
+#define QUOTE 4
+#define DQUOTE 5
+#define SLASH 6
+#define SLASH_STAR 7
+#define SLASH_STAR_STAR 8
+#define BACKSLASH 0x10000
+
+HighlightCss::HighlightCss() {
+}
+
+HighlightCss::~HighlightCss() {
+}
+
+void HighlightCss::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+
+ if (t_ & BACKSLASH) {
+ t_ &= ~BACKSLASH;
+ *r += c;
+ continue;
+ } else if (c == '\\') {
+ *r += c;
+ t_ |= BACKSLASH;
+ continue;
+ }
+
+ TryAgain:
+ switch (t_ & 255) {
+
+ case NORMAL:
+ *r += HI_SELECTOR;
+ t_ = SELECTOR;
+ // fallthrough
+
+ case SELECTOR:
+ if (c == '{') {
+ t_ = PROPERTY;
+ *r += HI_RESET;
+ *r += c;
+ *r += HI_PROPERTY;
+ } else if (c == ',') {
+ *r += HI_RESET;
+ *r += c;
+ *r += HI_SELECTOR;
+ } else if (c == '/') {
+ t_ = SELECTOR << 8 | SLASH;
+ } else if (c == '\'') {
+ t_ = SELECTOR << 8 | QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = SELECTOR << 8 | DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case PROPERTY:
+ if (c == '/') {
+ t_ = PROPERTY << 8 | SLASH;
+ } else if (c == '\'') {
+ t_ = PROPERTY << 8 | QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = PROPERTY << 8 | DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == ':') {
+ t_ = VALUE;
+ *r += HI_RESET;
+ *r += c;
+ } else if (c == '}') {
+ t_ = SELECTOR;
+ *r += HI_RESET;
+ *r += c;
+ *r += HI_SELECTOR;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case VALUE:
+ if (c == '/') {
+ t_ = VALUE << 8 | SLASH;
+ } else if (c == '\'') {
+ t_ = VALUE << 8 | QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = VALUE << 8 | DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == ';') {
+ t_ = PROPERTY;
+ *r += c;
+ *r += HI_PROPERTY;
+ } else if (c == '}') {
+ t_ = SELECTOR;
+ *r += c;
+ *r += HI_SELECTOR;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case SLASH:
+ if (c == '*') {
+ *r += HI_COMMENT;
+ *r += "/*";
+ t_ &= -256;
+ t_ |= SLASH_STAR;
+ } else {
+ *r += '/';
+ t_ >>= 8;
+ goto TryAgain;
+ }
+ break;
+
+ case SLASH_STAR:
+ *r += c;
+ if (c == '*') {
+ t_ &= -256;
+ t_ |= SLASH_STAR_STAR;
+ }
+ break;
+
+ case SLASH_STAR_STAR:
+ *r += c;
+ if (c == '/') {
+ *r += HI_RESET;
+ goto Pop;
+ } else {
+ t_ &= -256;
+ t_ |= SLASH_STAR;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ goto Pop;
+ }
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ goto Pop;
+ }
+ break;
+
+ Pop:
+ t_ >>= 8;
+ if (t_ == SELECTOR)
+ *r += HI_SELECTOR;
+ if (t_ == PROPERTY)
+ *r += HI_PROPERTY;
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightCss::flush(std::string *r) {
+ switch (t_ & 255) {
+ case SLASH:
+ *r += '/';
+ break;
+ default:
+ break;
+ }
+ *r += HI_RESET;
+ t_ = 0;
+}
diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cpp
index 63f3c9e3..93448c01 100644
--- a/llamafile/highlight_markdown.cpp
+++ b/llamafile/highlight_markdown.cpp
@@ -121,38 +121,38 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
break;
Code:
- case CODE: {
- char cs[2] = {c};
- highlighter_->feed(r, cs);
- if (c == '`')
+ case CODE:
+ if (c == '`') {
t_ = CODE_TICK;
+ } else {
+ char cs[2] = {c};
+ highlighter_->feed(r, cs);
+ }
break;
- }
- case CODE_TICK: {
- char cs[2] = {c};
- highlighter_->feed(r, cs);
+ case CODE_TICK:
if (c == '`') {
t_ = CODE_TICK_TICK;
} else {
+ char cs[3] = {'`', c};
+ highlighter_->feed(r, cs);
t_ = CODE;
}
break;
- }
- case CODE_TICK_TICK: {
- char cs[2] = {c};
- highlighter_->feed(r, cs);
+ case CODE_TICK_TICK:
if (c == '`') {
t_ = NORMAL;
highlighter_->flush(r);
delete highlighter_;
highlighter_ = nullptr;
+ *r += "```";
} else {
+ char cs[4] = {'`', '`', c};
+ highlighter_->feed(r, cs);
t_ = CODE;
}
break;
- }
default:
__builtin_unreachable();
@@ -173,11 +173,21 @@ void HighlightMarkdown::flush(std::string *r) {
*r += HI_RESET;
break;
case CODE:
+ highlighter_->flush(r);
+ delete highlighter_;
+ highlighter_ = nullptr;
+ break;
case CODE_TICK:
+ highlighter_->flush(r);
+ delete highlighter_;
+ highlighter_ = nullptr;
+ *r += '`';
+ break;
case CODE_TICK_TICK:
highlighter_->flush(r);
delete highlighter_;
highlighter_ = nullptr;
+ *r += "``";
break;
default:
break;
diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cpp
index c59890d3..5b5fe7a3 100644
--- a/llamafile/highlight_python.cpp
+++ b/llamafile/highlight_python.cpp
@@ -221,6 +221,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
}
void HighlightPython::flush(std::string *r) {
+ t_ &= ~BACKSLASH;
switch (t_) {
case WORD:
if (is_keyword_python(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight_rust.cpp
index 5ccd85ca..990cd5f8 100644
--- a/llamafile/highlight_rust.cpp
+++ b/llamafile/highlight_rust.cpp
@@ -221,6 +221,7 @@ void HighlightRust::feed(std::string *r, std::string_view input) {
}
void HighlightRust::flush(std::string *r) {
+ t_ &= ~BACKSLASH;
switch (t_) {
case WORD:
if (is_keyword_rust(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_sql.cpp b/llamafile/highlight_sql.cpp
index 2eeba4c8..c86845d8 100644
--- a/llamafile/highlight_sql.cpp
+++ b/llamafile/highlight_sql.cpp
@@ -166,6 +166,7 @@ void HighlightSql::feed(std::string *r, std::string_view input) {
}
void HighlightSql::flush(std::string *r) {
+ t_ &= ~BACKSLASH;
switch (t_) {
case WORD:
if (is_keyword_sql(word_.data(), word_.size())) {