Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add Julia syntax highlighting
Justine Tunney264e97905664
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex ffe21176..9c72900d 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -211,5 +211,9 @@ Highlight *Highlight::create(const std::string_view &lang) {if (lang == "matlab")return new HighlightMatlab;+ if (lang == "jl" || //+ lang == "julia")+ return new HighlightJulia;+return nullptr;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex 2086fe2c..59e1f018 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -137,6 +137,7 @@ is_keyword_f is_keyword_r;is_keyword_f is_keyword_r_builtin;is_keyword_f is_keyword_r_constant;is_keyword_f is_keyword_scala;+is_keyword_f is_keyword_julia;}class Highlight {@@ -713,3 +714,15 @@ class HighlightScala : public Highlight {std::string word_;unsigned char nest_[16];};++class HighlightJulia : public Highlight {+ public:+ HighlightJulia();+ ~HighlightJulia() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_julia.cpp b/llamafile/highlight_julia.cppnew file mode 100644index 00000000..05384ab6--- /dev/null+++ b/llamafile/highlight_julia.cpp@@ -0,0 +1,255 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ COMMENT,+ QUOTE,+ QUOTE_BACKSLASH,+ ANNOTATION,+ ANNOTATION2,+ DQUOTE,+ DQUOTE_VAR,+ DQUOTESTR,+ DQUOTESTR_BACKSLASH,+ DQUOTE2,+ DQUOTE3,+ DQUOTE3_BACKSLASH,+ DQUOTE31,+ DQUOTE32,+};++HighlightJulia::HighlightJulia() {+}++HighlightJulia::~HighlightJulia() {+}++void HighlightJulia::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_') {+ t_ = WORD;+ word_ += c;+ } else if (c == '#') {+ t_ = COMMENT;+ *r += HI_COMMENT;+ *r += '#';+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += '\'';+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += '"';+ } else if (c == '@') {+ t_ = ANNOTATION;+ } else {+ *r += c;+ }+ break;++ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_julia(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case COMMENT:+ *r += c;+ if (c == '\n') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ t_ = DQUOTE2;+ } else if (c == '\\') {+ t_ = DQUOTESTR_BACKSLASH;+ } else {+ t_ = DQUOTESTR;+ }+ break;++ Dquotestr:+ case DQUOTESTR:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTESTR_BACKSLASH;+ }+ break;++ case DQUOTESTR_BACKSLASH:+ *r += c;+ t_ = DQUOTESTR;+ break;++ case DQUOTE2:+ if (c == '"') {+ *r += '"';+ t_ = DQUOTE3;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ Dquote3:+ case DQUOTE3:+ *r += c;+ if (c == '"') {+ t_ = DQUOTE31;+ } else if (c == '\\') {+ t_ = DQUOTE3_BACKSLASH;+ }+ break;++ case DQUOTE31:+ *r += c;+ if (c == '"') {+ t_ = DQUOTE32;+ } else if (c == '\\') {+ t_ = DQUOTE3_BACKSLASH;+ } else {+ t_ = DQUOTE3;+ }+ break;++ case DQUOTE32:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE3_BACKSLASH;+ } else {+ t_ = DQUOTE3;+ }+ break;++ case DQUOTE3_BACKSLASH:+ *r += c;+ t_ = DQUOTE3;+ break;++ case ANNOTATION:+ if (!isascii(c) || isalpha(c) || c == '_') {+ *r += HI_ATTRIB;+ *r += '@';+ *r += c;+ t_ = ANNOTATION2;+ } else {+ *r += '@';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case ANNOTATION2:+ if (!isascii(c) || isalnum(c) || c == '_') {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightJulia::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_julia(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case ANNOTATION:+ *r += '@';+ break;+ case COMMENT:+ case QUOTE:+ case QUOTE_BACKSLASH:+ case DQUOTE:+ case DQUOTESTR:+ case DQUOTESTR_BACKSLASH:+ case DQUOTE2:+ case DQUOTE3:+ case DQUOTE3_BACKSLASH:+ case DQUOTE31:+ case DQUOTE32:+ case ANNOTATION2:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex 3439ae13..5fa7c7bf 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -45,6 +45,7 @@ const char *const kLanguages[] = {"java", //"javascript", //"json", //+ "julia", //"kotlin", //"ld", //"lisp", //diff --git a/llamafile/is_keyword_julia.gperf b/llamafile/is_keyword_julia.gperfnew file mode 100644index 00000000..2111c79b--- /dev/null+++ b/llamafile/is_keyword_julia.gperf@@ -0,0 +1,38 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_julia+%%+baremodule+begin+break+catch+const+continue+do+else+elseif+end+export+false+finally+for+function+global+if+import+let+local+macro+module+quote+return+struct+true+try+using+while