Skip to content

Distribute and run LLMs with a single file.

Add R syntax highlighting

Justine Tunney38398fa7b03b
diff --git a/build/rules.mk b/build/rules.mk
index d539d67c..ae066bb2 100644
--- a/build/rules.mk
+++ b/build/rules.mk
@@ -9,6 +9,10 @@ o/$(MODE)/%.o: %.c $(COSMOCC)
@mkdir -p $(@D)
$(COMPILE.c) -o $@ $<
+o/$(MODE)/%.o: o/$(MODE)/%.c $(COSMOCC)
+ @mkdir -p $(@D)
+ $(COMPILE.c) -o $@ $<
+
o/$(MODE)/%.o: %.cc $(COSMOCC)
@mkdir -p $(@D)
$(COMPILE.cc) -o $@ $<
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 8ee011cc..d21fbaf0 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -6,17 +6,19 @@ PKGS += LLAMAFILE
LLAMAFILE_FILES := $(wildcard llamafile/*.*)
LLAMAFILE_HDRS = $(filter %.h,$(LLAMAFILE_FILES))
LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES)) $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)
+LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)
+LLAMAFILE_SRCS_GPERF_C = $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)
+LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU) $(LLAMAFILE_SRCS_GPERF)
LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))
LLAMAFILE_OBJS := \
$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o) \
$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
- $(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)
+ $(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o) \
+ $(LLAMAFILE_SRCS_GPERF_C:%.c=%.o) \
$(LLAMAFILE_OBJS): private CCFLAGS += -g
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cpp
index eedc9697..ef5e191d 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight.cpp
@@ -158,6 +158,9 @@ Highlight *Highlight::create(const std::string_view &lang) {
if (lang == "d")
return new HighlightC(is_keyword_d);
+ if (lang == "r")
+ return new HighlightR;
+
if (lang == "zig")
return new HighlightZig;
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
index e7df0a5e..7ac97bfc 100644
--- a/llamafile/highlight.h
+++ b/llamafile/highlight.h
@@ -131,6 +131,9 @@ is_keyword_f is_keyword_ld_warning;
is_keyword_f is_keyword_matlab;
is_keyword_f is_keyword_matlab_builtin;
is_keyword_f is_keyword_matlab_constant;
+is_keyword_f is_keyword_r;
+is_keyword_f is_keyword_r_builtin;
+is_keyword_f is_keyword_r_constant;
}
class Highlight {
@@ -641,3 +644,15 @@ class HighlightMatlab : public Highlight {
int t_ = 0;
std::string word_;
};
+
+class HighlightR : public Highlight {
+ public:
+ HighlightR();
+ ~HighlightR() override;
+ void feed(std::string *result, std::string_view input) override;
+ void flush(std::string *result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
diff --git a/llamafile/highlight_r.cpp b/llamafile/highlight_r.cpp
new file mode 100644
index 00000000..0b7c5a09
--- /dev/null
+++ b/llamafile/highlight_r.cpp
@@ -0,0 +1,263 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "highlight.h"
+
+#include <ctype.h>
+
+enum {
+ NORMAL,
+ WORD,
+ COMMENT,
+ QUOTE,
+ QUOTE_BACKSLASH,
+ DQUOTE,
+ DQUOTE_BACKSLASH,
+ HYPHEN,
+ HYPHEN_GT,
+ LT,
+ LT_LT,
+ COLON,
+};
+
+HighlightR::HighlightR() {
+}
+
+HighlightR::~HighlightR() {
+}
+
+void HighlightR::feed(std::string *r, std::string_view input) {
+ int c;
+ for (size_t i = 0; i < input.size(); ++i) {
+ c = input[i] & 255;
+ switch (t_) {
+
+ Normal:
+ case NORMAL:
+ if (!isascii(c) || isalpha(c)) {
+ t_ = WORD;
+ word_ += c;
+ } else if (c == '#') {
+ *r += HI_COMMENT;
+ *r += '#';
+ t_ = COMMENT;
+ } else if (c == '\'') {
+ t_ = QUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '"') {
+ t_ = DQUOTE;
+ *r += HI_STRING;
+ *r += c;
+ } else if (c == '-') {
+ t_ = HYPHEN;
+ } else if (c == '<') {
+ t_ = LT;
+ } else if (c == ':') {
+ t_ = COLON;
+ } else if (c == '$' || c == '@') {
+ *r += HI_OPERATOR;
+ *r += c;
+ *r += HI_RESET;
+ } else {
+ *r += c;
+ }
+ break;
+
+ Word:
+ case WORD:
+ if (!isascii(c) || isalnum(c) || c == '_' || c == '.') {
+ word_ += c;
+ } else {
+ if (is_keyword_r(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_r_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_r_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case COMMENT:
+ *r += c;
+ if (c == '\n') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ }
+ break;
+
+ case QUOTE:
+ *r += c;
+ if (c == '\'') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = QUOTE_BACKSLASH;
+ }
+ break;
+
+ case QUOTE_BACKSLASH:
+ *r += c;
+ t_ = QUOTE;
+ break;
+
+ case DQUOTE:
+ *r += c;
+ if (c == '"') {
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else if (c == '\\') {
+ t_ = DQUOTE_BACKSLASH;
+ }
+ break;
+
+ case DQUOTE_BACKSLASH:
+ *r += c;
+ t_ = DQUOTE;
+ break;
+
+ case COLON:
+ if (c == ':') {
+ *r += HI_OPERATOR;
+ *r += "::";
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += ':';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case LT:
+ if (c == '<') {
+ t_ = LT_LT;
+ } else if (c == '-') {
+ *r += HI_OPERATOR;
+ *r += "<-";
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += '<';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case HYPHEN:
+ if (c == '>') {
+ t_ = HYPHEN_GT;
+ } else {
+ *r += '-';
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case LT_LT:
+ if (c == '-') {
+ *r += HI_OPERATOR;
+ *r += "<<-";
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += "<<";
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ case HYPHEN_GT:
+ if (c == '>') {
+ *r += HI_OPERATOR;
+ *r += "->>";
+ *r += HI_RESET;
+ t_ = NORMAL;
+ } else {
+ *r += HI_OPERATOR;
+ *r += "->";
+ *r += HI_RESET;
+ t_ = NORMAL;
+ goto Normal;
+ }
+ break;
+
+ default:
+ __builtin_unreachable();
+ }
+ }
+}
+
+void HighlightR::flush(std::string *r) {
+ switch (t_) {
+ case WORD:
+ if (is_keyword_r(word_.data(), word_.size())) {
+ *r += HI_KEYWORD;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_r_builtin(word_.data(), word_.size())) {
+ *r += HI_BUILTIN;
+ *r += word_;
+ *r += HI_RESET;
+ } else if (is_keyword_r_constant(word_.data(), word_.size())) {
+ *r += HI_CONSTANT;
+ *r += word_;
+ *r += HI_RESET;
+ } else {
+ *r += word_;
+ }
+ word_.clear();
+ break;
+ case HYPHEN:
+ *r += '-';
+ break;
+ case HYPHEN_GT:
+ *r += "->";
+ break;
+ case LT:
+ *r += '<';
+ break;
+ case LT_LT:
+ *r += "<<";
+ break;
+ case COLON:
+ *r += ':';
+ break;
+ case QUOTE:
+ case QUOTE_BACKSLASH:
+ case DQUOTE:
+ case DQUOTE_BACKSLASH:
+ case COMMENT:
+ *r += HI_RESET;
+ break;
+ default:
+ break;
+ }
+ t_ = NORMAL;
+}
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cpp
index c0197f16..602ad203 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight_test.cpp
@@ -57,6 +57,7 @@ const char *const kLanguages[] = {
"perl", //
"php!", //
"python", //
+ "r", //
"ruby", //
"rust", //
"shell", //
diff --git a/llamafile/is_keyword_r.gperf b/llamafile/is_keyword_r.gperf
new file mode 100644
index 00000000..9083903b
--- /dev/null
+++ b/llamafile/is_keyword_r.gperf
@@ -0,0 +1,23 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_r
+%%
+NA
+NA_character_
+NA_complex_
+NA_integer_
+NA_real_
+break
+else
+for
+function
+if
+in
+next
+repeat
+while
diff --git a/llamafile/is_keyword_r_builtin.gperf b/llamafile/is_keyword_r_builtin.gperf
new file mode 100644
index 00000000..1614322b
--- /dev/null
+++ b/llamafile/is_keyword_r_builtin.gperf
@@ -0,0 +1,162 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_r_builtin
+%%
+Arg
+Conj
+Im
+Mod
+Re
+acos
+aggregate
+anova
+aov
+apply
+as.array
+as.character
+as.complex
+as.data.frame
+as.logical
+as.numeric
+asin
+atan
+atan2
+attr
+attributes
+barplot
+binom.test
+boxplot
+by
+cbind
+choose
+class
+colMeans
+colSums
+colsum
+convolve
+coplot
+cor
+cos
+cov
+cummax
+cummin
+cumprod
+cumsum
+cut
+density
+diag
+diff
+dim
+dimnames
+do.call
+dotchart
+exp
+fft
+filter
+grep
+gsub
+hist
+ifelse
+interaction.plot
+is.array
+is.character
+is.complex
+is.data.frame
+is.element
+is.na
+is.null
+is.numeric
+lapply
+length
+list
+log
+log10
+match
+matrix
+max
+mean
+median
+merge
+min
+mvfft
+na.fail
+na.omit
+ncol
+nrow
+optim
+pairwise.t.test
+paste
+pie
+plot
+pmatch
+pmax
+pmin
+power.t.test
+print
+prod
+prop.table
+prop.test
+quantile
+range
+rank
+rbeta
+rbind
+rbinom
+rcauchy
+rchisq
+reshape
+return
+rev
+rexp
+rf
+rgamma
+rgeom
+rhyper
+rlnorm
+rlogis
+rnbinom
+rnorm
+round
+rowMeans
+rowsum
+rpois
+rt
+runif
+rweibull
+rwilcox
+sample
+scale
+sd
+seq
+sin
+solve
+sort
+stack
+stripplot
+strsplit
+subset
+substr
+sum
+sunflowerplot
+t
+table
+tan
+tapply
+tolower
+toupper
+typeof
+unclass
+union
+unique
+unstack
+var
+weighted.mean
+which
+which.max
+which.min
+xtabs
diff --git a/llamafile/is_keyword_r_constant.gperf b/llamafile/is_keyword_r_constant.gperf
new file mode 100644
index 00000000..355b12e0
--- /dev/null
+++ b/llamafile/is_keyword_r_constant.gperf
@@ -0,0 +1,14 @@
+%{
+#include <string.h>
+%}
+%pic
+%compare-strncmp
+%language=ANSI-C
+%readonly-tables
+%define lookup-function-name is_keyword_r_constant
+%%
+FALSE
+Inf
+NULL
+NaN
+TRUE