Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add R syntax highlighting
Justine Tunney38398fa7b03b
diff --git a/build/rules.mk b/build/rules.mkindex d539d67c..ae066bb2 100644--- a/build/rules.mk+++ b/build/rules.mk@@ -9,6 +9,10 @@ o/$(MODE)/%.o: %.c $(COSMOCC)@mkdir -p $(@D)$(COMPILE.c) -o $@ $<+o/$(MODE)/%.o: o/$(MODE)/%.c $(COSMOCC)+ @mkdir -p $(@D)+ $(COMPILE.c) -o $@ $<+o/$(MODE)/%.o: %.cc $(COSMOCC)@mkdir -p $(@D)$(COMPILE.cc) -o $@ $<diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 8ee011cc..d21fbaf0 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -6,17 +6,19 @@ PKGS += LLAMAFILELLAMAFILE_FILES := $(wildcard llamafile/*.*)LLAMAFILE_HDRS = $(filter %.h,$(LLAMAFILE_FILES))LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))-LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES)) $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)+LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_FILES))-LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)+LLAMAFILE_SRCS_GPERF_C = $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)+LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU) $(LLAMAFILE_SRCS_GPERF)LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))LLAMAFILE_OBJS := \$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o) \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \- $(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)+ $(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o) \+ $(LLAMAFILE_SRCS_GPERF_C:%.c=%.o) \$(LLAMAFILE_OBJS): private CCFLAGS += -gdiff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex eedc9697..ef5e191d 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -158,6 +158,9 @@ Highlight *Highlight::create(const std::string_view &lang) {if (lang == "d")return new HighlightC(is_keyword_d);+ if (lang == "r")+ return new HighlightR;+if (lang == "zig")return new HighlightZig;diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex e7df0a5e..7ac97bfc 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -131,6 +131,9 @@ is_keyword_f is_keyword_ld_warning;is_keyword_f is_keyword_matlab;is_keyword_f is_keyword_matlab_builtin;is_keyword_f is_keyword_matlab_constant;+is_keyword_f is_keyword_r;+is_keyword_f is_keyword_r_builtin;+is_keyword_f is_keyword_r_constant;}class Highlight {@@ -641,3 +644,15 @@ class HighlightMatlab : public Highlight {int t_ = 0;std::string word_;};++class HighlightR : public Highlight {+ public:+ HighlightR();+ ~HighlightR() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_r.cpp b/llamafile/highlight_r.cppnew file mode 100644index 00000000..0b7c5a09--- /dev/null+++ b/llamafile/highlight_r.cpp@@ -0,0 +1,263 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++enum {+ NORMAL,+ WORD,+ COMMENT,+ QUOTE,+ QUOTE_BACKSLASH,+ DQUOTE,+ DQUOTE_BACKSLASH,+ HYPHEN,+ HYPHEN_GT,+ LT,+ LT_LT,+ COLON,+};++HighlightR::HighlightR() {+}++HighlightR::~HighlightR() {+}++void HighlightR::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;+ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c)) {+ t_ = WORD;+ word_ += c;+ } else if (c == '#') {+ *r += HI_COMMENT;+ *r += '#';+ t_ = COMMENT;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '-') {+ t_ = HYPHEN;+ } else if (c == '<') {+ t_ = LT;+ } else if (c == ':') {+ t_ = COLON;+ } else if (c == '$' || c == '@') {+ *r += HI_OPERATOR;+ *r += c;+ *r += HI_RESET;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalnum(c) || c == '_' || c == '.') {+ word_ += c;+ } else {+ if (is_keyword_r(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_r_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_r_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case COMMENT:+ *r += c;+ if (c == '\n') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = QUOTE_BACKSLASH;+ }+ break;++ case QUOTE_BACKSLASH:+ *r += c;+ t_ = QUOTE;+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else if (c == '\\') {+ t_ = DQUOTE_BACKSLASH;+ }+ break;++ case DQUOTE_BACKSLASH:+ *r += c;+ t_ = DQUOTE;+ break;++ case COLON:+ if (c == ':') {+ *r += HI_OPERATOR;+ *r += "::";+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += ':';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case LT:+ if (c == '<') {+ t_ = LT_LT;+ } else if (c == '-') {+ *r += HI_OPERATOR;+ *r += "<-";+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += '<';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case HYPHEN:+ if (c == '>') {+ t_ = HYPHEN_GT;+ } else {+ *r += '-';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case LT_LT:+ if (c == '-') {+ *r += HI_OPERATOR;+ *r += "<<-";+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += "<<";+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case HYPHEN_GT:+ if (c == '>') {+ *r += HI_OPERATOR;+ *r += "->>";+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ *r += HI_OPERATOR;+ *r += "->";+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightR::flush(std::string *r) {+ switch (t_) {+ case WORD:+ if (is_keyword_r(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_r_builtin(word_.data(), word_.size())) {+ *r += HI_BUILTIN;+ *r += word_;+ *r += HI_RESET;+ } else if (is_keyword_r_constant(word_.data(), word_.size())) {+ *r += HI_CONSTANT;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case HYPHEN:+ *r += '-';+ break;+ case HYPHEN_GT:+ *r += "->";+ break;+ case LT:+ *r += '<';+ break;+ case LT_LT:+ *r += "<<";+ break;+ case COLON:+ *r += ':';+ break;+ case QUOTE:+ case QUOTE_BACKSLASH:+ case DQUOTE:+ case DQUOTE_BACKSLASH:+ case COMMENT:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight_test.cppindex c0197f16..602ad203 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight_test.cpp@@ -57,6 +57,7 @@ const char *const kLanguages[] = {"perl", //"php!", //"python", //+ "r", //"ruby", //"rust", //"shell", //diff --git a/llamafile/is_keyword_r.gperf b/llamafile/is_keyword_r.gperfnew file mode 100644index 00000000..9083903b--- /dev/null+++ b/llamafile/is_keyword_r.gperf@@ -0,0 +1,23 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_r+%%+NA+NA_character_+NA_complex_+NA_integer_+NA_real_+break+else+for+function+if+in+next+repeat+whilediff --git a/llamafile/is_keyword_r_builtin.gperf b/llamafile/is_keyword_r_builtin.gperfnew file mode 100644index 00000000..1614322b--- /dev/null+++ b/llamafile/is_keyword_r_builtin.gperf@@ -0,0 +1,162 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_r_builtin+%%+Arg+Conj+Im+Mod+Re+acos+aggregate+anova+aov+apply+as.array+as.character+as.complex+as.data.frame+as.logical+as.numeric+asin+atan+atan2+attr+attributes+barplot+binom.test+boxplot+by+cbind+choose+class+colMeans+colSums+colsum+convolve+coplot+cor+cos+cov+cummax+cummin+cumprod+cumsum+cut+density+diag+diff+dim+dimnames+do.call+dotchart+exp+fft+filter+grep+gsub+hist+ifelse+interaction.plot+is.array+is.character+is.complex+is.data.frame+is.element+is.na+is.null+is.numeric+lapply+length+list+log+log10+match+matrix+max+mean+median+merge+min+mvfft+na.fail+na.omit+ncol+nrow+optim+pairwise.t.test+paste+pie+plot+pmatch+pmax+pmin+power.t.test+prod+prop.table+prop.test+quantile+range+rank+rbeta+rbind+rbinom+rcauchy+rchisq+reshape+return+rev+rexp+rf+rgamma+rgeom+rhyper+rlnorm+rlogis+rnbinom+rnorm+round+rowMeans+rowsum+rpois+rt+runif+rweibull+rwilcox+sample+scale+sd+seq+sin+solve+sort+stack+stripplot+strsplit+subset+substr+sum+sunflowerplot+t+table+tan+tapply+tolower+toupper+typeof+unclass+union+unique+unstack+var+weighted.mean+which+which.max+which.min+xtabsdiff --git a/llamafile/is_keyword_r_constant.gperf b/llamafile/is_keyword_r_constant.gperfnew file mode 100644index 00000000..355b12e0--- /dev/null+++ b/llamafile/is_keyword_r_constant.gperf@@ -0,0 +1,14 @@+%{+#include <string.h>+%}+%pic+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_r_constant+%%+FALSE+Inf+NULL+NaN+TRUE