Skip to content

Distribute and run LLMs with a single file.

Add Julia syntax highlighting

Justine Tunney264e97905664
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index ffe21176..9c72900d 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -211,5 +211,9 @@ Highlight *Highlight::create(const std::string_view &lang) {
if (lang == "matlab")
return new HighlightMatlab;
+ if (lang == "jl" || //
+ lang == "julia")
+ return new HighlightJulia;
+
return nullptr;
}
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index 2086fe2c..59e1f018 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -137,6 +137,7 @@ is_keyword_f is_keyword_r;
is_keyword_f is_keyword_r_builtin;
is_keyword_f is_keyword_r_constant;
is_keyword_f is_keyword_scala;
+is_keyword_f is_keyword_julia;
}
class Highlight {
@@ -713,3 +714,15 @@ class HighlightScala : public Highlight {
std::string word_;
unsigned char nest_[16];
};
+
+class HighlightJulia : public Highlight {
+ public:
+ HighlightJulia();
+ ~HighlightJulia() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
diff --git a/llamafile/highlight_julia.cpp b/llamafile/highlight_julia.cpp
new file mode 100644
index 00000000..05384ab6
--- /dev/null
+++ b/llamafile/highlight_julia.cpp
@@ -0,0 +1,255 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ COMMENT,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ ANNOTATION,
+ ANNOTATION2,
+ DQUOTE,
+ DQUOTE_VAR,
+ DQUOTESTR,
+ DQUOTESTR_BACKSLASH,
+ DQUOTE2,
+ DQUOTE3,
+ DQUOTE3_BACKSLASH,
+ DQUOTE31,
+ DQUOTE32,
+};
+
+HighlightJulia::HighlightJulia() {
+}
+
+HighlightJulia::~HighlightJulia() {
+}
+
+void HighlightJulia::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '#') {
+ t_ = COMMENT;
+ *r += HI_COMMENT;
+ *r += '#';
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += '\'';
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += '"';
+ } else if (c == '@') {
+ t_ = ANNOTATION;
+ } else {
+ *r += c;
+ }
+ break;
+
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ word_ += c;
+ } else {
+ if (is_keyword_julia(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case COMMENT:
+ *r += c;
+ if (c == '\n') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ t_ = DQUOTE2;
+ } else if (c == '\\') {
+ t_ = DQUOTESTR_BACKSLASH;
+ } else {
+ t_ = DQUOTESTR;
+ }
+ break;
+
+ Dquotestr:
+ case DQUOTESTR:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTESTR_BACKSLASH;
+ }
+ break;
+
+ case DQUOTESTR_BACKSLASH:
+ *r += c;
+ t_ = DQUOTESTR;
+ break;
+
+ case DQUOTE2:
+ if (c == '"') {
+ *r += '"';
+ t_ = DQUOTE3;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ Dquote3:
+ case DQUOTE3:
+ *r += c;
+ if (c == '"') {
+ t_ = DQUOTE31;
+ } else if (c == '\\') {
+ t_ = DQUOTE3_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE31:
+ *r += c;
+ if (c == '"') {
+ t_ = DQUOTE32;
+ } else if (c == '\\') {
+ t_ = DQUOTE3_BACKSLASH;
+ } else {
+ t_ = DQUOTE3;
+ }
+ break;
+
+ case DQUOTE32:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE3_BACKSLASH;
+ } else {
+ t_ = DQUOTE3;
+ }
+ break;
+
+ case DQUOTE3_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE3;
+ break;
+
+ case ANNOTATION:
+ if (!isascii(c) || isalpha(c) || c == '_') {
+ *r += HI_ATTRIB;
+ *r += '@';
+ *r += c;
+ t_ = ANNOTATION2;
+ } else {
+ *r += '@';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case ANNOTATION2:
+ if (!isascii(c) || isalnum(c) || c == '_') {
+ *r += c;
+ } else {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightJulia::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_julia(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case ANNOTATION:
+ *r += '@';
+ break;
+ case COMMENT:
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case DQUOTE:
+ case DQUOTESTR:
+ case DQUOTESTR_BACKSLASH:
+ case DQUOTE2:
+ case DQUOTE3:
+ case DQUOTE3_BACKSLASH:
+ case DQUOTE31:
+ case DQUOTE32:
+ case ANNOTATION2:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index 3439ae13..5fa7c7bf 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -45,6 +45,7 @@ const char *const kLanguages[] = {
"java", //
"javascript", //
"json", //
+ "julia", //
"kotlin", //
"ld", //
"lisp", //
diff --git a/llamafile/is_keyword_julia.gperf b/llamafile/is_keyword_julia.gperf
new file mode 100644
index 00000000..2111c79b
--- /dev/null
+++ b/llamafile/is_keyword_julia.gperf
@@ -0,0 +1,38 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_julia
+%%
+baremodule
+begin
+break
+catch
+const
+continue
+do
+else
+elseif
+end
+export
+false
+finally
+for
+function
+global
+if
+import
+let
+local
+macro
+module
+quote
+return
+struct
+true
+try
+using
+while