Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add CSS syntax highlighting
Justine Tunney32c2ad8bf10a
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 4a1cee5d..540bbacd 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -42,6 +42,8 @@ Highlight *Highlight::create(const std::string_view &lang) {return new HighlightC(is_keyword_go, is_keyword_go_type);} else if (lang == "sql") {return new HighlightSql();+ } else if (lang == "css") {+ return new HighlightCss();} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 09733c80..f38914d9 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -27,8 +27,10 @@#define HI_MACRO "\033[35m" // magenta#define HI_ATTRIB "\033[35m" // magenta#define HI_CONTIN "\033[33m" // yellow+#define HI_LABEL "\033[33m" // yellow#define HI_TYPE "\033[36m" // cyan-#define HI_LABEL "\033[35m" // magenta+#define HI_SELECTOR "\033[33m" // yellow+#define HI_PROPERTY "\033[36m" // cyantypedef const char *is_keyword_f(const char *, size_t);@@ -166,3 +168,14 @@ class HighlightSql : public Highlight {int t_ = 0;std::string word_;};++class HighlightCss : public Highlight {+ public:+ HighlightCss();+ ~HighlightCss() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+};diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight_c.cppindex 1e635713..e78c3bc6 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight_c.cpp@@ -160,6 +160,7 @@ void HighlightC::feed(std::string *r, std::string_view input) {}void HighlightC::flush(std::string *r) {+ t_ &= ~BACKSLASH;switch (t_) {case WORD:if (is_keyword_(word_.data(), word_.size())) {diff --git a/llamafile/highlight_css.cpp b/llamafile/highlight_css.cppnew file mode 100644index 00000000..077fadd0--- /dev/null+++ b/llamafile/highlight_css.cpp@@ -0,0 +1,208 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++#define NORMAL 0+#define SELECTOR 1+#define PROPERTY 2+#define VALUE 3+#define QUOTE 4+#define DQUOTE 5+#define SLASH 6+#define SLASH_STAR 7+#define SLASH_STAR_STAR 8+#define BACKSLASH 0x10000++HighlightCss::HighlightCss() {+}++HighlightCss::~HighlightCss() {+}++void HighlightCss::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;++ if (t_ & BACKSLASH) {+ t_ &= ~BACKSLASH;+ *r += c;+ continue;+ } else if (c == '\\') {+ *r += c;+ t_ |= BACKSLASH;+ continue;+ }++ TryAgain:+ switch (t_ & 255) {++ case NORMAL:+ *r += HI_SELECTOR;+ t_ = SELECTOR;+ // fallthrough++ case SELECTOR:+ if (c == '{') {+ t_ = PROPERTY;+ *r += HI_RESET;+ *r += c;+ *r += HI_PROPERTY;+ } else if (c == ',') {+ *r += HI_RESET;+ *r += c;+ *r += HI_SELECTOR;+ } else if (c == '/') {+ t_ = SELECTOR << 8 | SLASH;+ } else if (c == '\'') {+ t_ = SELECTOR << 8 | QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = SELECTOR << 8 | DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else {+ *r += c;+ }+ break;++ case PROPERTY:+ if (c == '/') {+ t_ = PROPERTY << 8 | SLASH;+ } else if (c == '\'') {+ t_ = PROPERTY << 8 | QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = PROPERTY << 8 | DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == ':') {+ t_ = VALUE;+ *r += HI_RESET;+ *r += c;+ } else if (c == '}') {+ t_ = SELECTOR;+ *r += HI_RESET;+ *r += c;+ *r += HI_SELECTOR;+ } else {+ *r += c;+ }+ break;++ case VALUE:+ if (c == '/') {+ t_ = VALUE << 8 | SLASH;+ } else if (c == '\'') {+ t_ = VALUE << 8 | QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = VALUE << 8 | DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == ';') {+ t_ = PROPERTY;+ *r += c;+ *r += HI_PROPERTY;+ } else if (c == '}') {+ t_ = SELECTOR;+ *r += c;+ *r += HI_SELECTOR;+ } else {+ *r += c;+ }+ break;++ case SLASH:+ if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ &= -256;+ t_ |= SLASH_STAR;+ } else {+ *r += '/';+ t_ >>= 8;+ goto TryAgain;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*') {+ t_ &= -256;+ t_ |= SLASH_STAR_STAR;+ }+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ goto Pop;+ } else {+ t_ &= -256;+ t_ |= SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ goto Pop;+ }+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ goto Pop;+ }+ break;++ Pop:+ t_ >>= 8;+ if (t_ == SELECTOR)+ *r += HI_SELECTOR;+ if (t_ == PROPERTY)+ *r += HI_PROPERTY;+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightCss::flush(std::string *r) {+ switch (t_ & 255) {+ case SLASH:+ *r += '/';+ break;+ default:+ break;+ }+ *r += HI_RESET;+ t_ = 0;+}diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cppindex 63f3c9e3..93448c01 100644--- a/llamafile/highlight_markdown.cpp+++ b/llamafile/highlight_markdown.cpp@@ -121,38 +121,38 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {break;Code:- case CODE: {- char cs[2] = {c};- highlighter_->feed(r, cs);- if (c == '`')+ case CODE:+ if (c == '`') {t_ = CODE_TICK;+ } else {+ char cs[2] = {c};+ highlighter_->feed(r, cs);+ }break;- }- case CODE_TICK: {- char cs[2] = {c};- highlighter_->feed(r, cs);+ case CODE_TICK:if (c == '`') {t_ = CODE_TICK_TICK;} else {+ char cs[3] = {'`', c};+ highlighter_->feed(r, cs);t_ = CODE;}break;- }- case CODE_TICK_TICK: {- char cs[2] = {c};- highlighter_->feed(r, cs);+ case CODE_TICK_TICK:if (c == '`') {t_ = NORMAL;highlighter_->flush(r);delete highlighter_;highlighter_ = nullptr;+ *r += "```";} else {+ char cs[4] = {'`', '`', c};+ highlighter_->feed(r, cs);t_ = CODE;}break;- }default:__builtin_unreachable();@@ -173,11 +173,21 @@ void HighlightMarkdown::flush(std::string *r) {*r += HI_RESET;break;case CODE:+ highlighter_->flush(r);+ delete highlighter_;+ highlighter_ = nullptr;+ break;case CODE_TICK:+ highlighter_->flush(r);+ delete highlighter_;+ highlighter_ = nullptr;+ *r += '`';+ break;case CODE_TICK_TICK:highlighter_->flush(r);delete highlighter_;highlighter_ = nullptr;+ *r += "``";break;default:break;diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight_python.cppindex c59890d3..5b5fe7a3 100644--- a/llamafile/highlight_python.cpp+++ b/llamafile/highlight_python.cpp@@ -221,6 +221,7 @@ void HighlightPython::feed(std::string *r, std::string_view input) {}void HighlightPython::flush(std::string *r) {+ t_ &= ~BACKSLASH;switch (t_) {case WORD:if (is_keyword_python(word_.data(), word_.size())) {diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight_rust.cppindex 5ccd85ca..990cd5f8 100644--- a/llamafile/highlight_rust.cpp+++ b/llamafile/highlight_rust.cpp@@ -221,6 +221,7 @@ void HighlightRust::feed(std::string *r, std::string_view input) {}void HighlightRust::flush(std::string *r) {+ t_ &= ~BACKSLASH;switch (t_) {case WORD:if (is_keyword_rust(word_.data(), word_.size())) {diff --git a/llamafile/highlight_sql.cpp b/llamafile/highlight_sql.cppindex 2eeba4c8..c86845d8 100644--- a/llamafile/highlight_sql.cpp+++ b/llamafile/highlight_sql.cpp@@ -166,6 +166,7 @@ void HighlightSql::feed(std::string *r, std::string_view input) {}void HighlightSql::flush(std::string *r) {+ t_ &= ~BACKSLASH;switch (t_) {case WORD:if (is_keyword_sql(word_.data(), word_.size())) {