Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Refactor package structure
Justine Tunney05fa4f35c0fc
diff --git a/Makefile b/Makefileindex e3f6f6ef..c5cf66f4 100644--- a/Makefile+++ b/Makefile@@ -11,12 +11,11 @@ MAKEFLAGS += --no-builtin-rulesinclude build/config.mkinclude build/rules.mk+include third_party/BUILD.mkinclude llamafile/BUILD.mkinclude llama.cpp/BUILD.mkinclude stable-diffusion.cpp/BUILD.mkinclude whisper.cpp/BUILD.mk-include double-conversion/BUILD.mk-include stb/BUILD.mk# the root package is `o//` by default# building a package also builds its sub-packages@@ -25,7 +24,7 @@ o/$(MODE)/: o/$(MODE)/llamafile \o/$(MODE)/llama.cpp \o/$(MODE)/stable-diffusion.cpp \o/$(MODE)/whisper.cpp \- o/$(MODE)/stb \+ o/$(MODE)/third_party \o/$(MODE)/depend.test# for installing to `make PREFIX=/usr/local`diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mkindex ed9e0e17..83183b00 100644--- a/llama.cpp/llava/BUILD.mk+++ b/llama.cpp/llava/BUILD.mk@@ -16,7 +16,7 @@ o/$(MODE)/llama.cpp/llava/llava-quantize: \o/$(MODE)/llama.cpp/llava/llava-quantize.1.asc.zip.o \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a+ o/$(MODE)/third_party/stb/stb.a$(LLAMA_CPP_LLAVA_OBJS): llama.cpp/llava/BUILD.mkdiff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex 85a8db0e..0c0f36af 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -13,7 +13,7 @@#include "llama.cpp/ggml-metal.h"#include "llama.cpp/string.h"-#include "stb/stb_image.h"+#include "third_party/stb/stb_image.h"#include <cassert>#include <cmath>diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mkindex 5985629e..fb432e16 100644--- a/llama.cpp/main/BUILD.mk+++ b/llama.cpp/main/BUILD.mk@@ -14,7 +14,8 @@ o/$(MODE)/llama.cpp/main/main: \o/$(MODE)/llama.cpp/server/server.a \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a \+ o/$(MODE)/llamafile/highlight/highlight.a \+ o/$(MODE)/third_party/stb/stb.a \o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 76978305..555b0e10 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -5,7 +5,7 @@#include "llama.cpp/llama.h"#include "llama.cpp/grammar-parser.h"#include "llama.cpp/llava/llava.h"-#include "stb/stb_image.h"+#include "third_party/stb/stb_image.h"#include "utils.h"#include "oai.h"#include "llamafile/micros.h"diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex a374cfd3..f5734415 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -9,16 +9,13 @@ LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))-LLAMAFILE_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_FILES))-LLAMAFILE_SRCS_GPERF_C = $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)-LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU) $(LLAMAFILE_SRCS_GPERF)+LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))LLAMAFILE_OBJS := \$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o) \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o) \- $(LLAMAFILE_SRCS_GPERF_C:%.c=%.o) \$(LLAMAFILE_OBJS): private CCFLAGS += -g@@ -26,6 +23,7 @@ $(LLAMAFILE_OBJS): private CCFLAGS += -g# therefore we want to avoid it going inside the .a fileLLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))+include llamafile/highlight/BUILD.mkinclude llamafile/server/BUILD.mko/$(MODE)/llamafile/zipalign: \@@ -57,9 +55,6 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/addnl \o/$(MODE)/llamafile/high \o/$(MODE)/llamafile/datauri_test.runs \- o/$(MODE)/llamafile/highlight_c_test.runs \- o/$(MODE)/llamafile/highlight_python_test.runs \- o/$(MODE)/llamafile/highlight_test.runs \o/$(MODE)/llamafile/parse_cidr_test.runs \o/$(MODE)/llamafile/pool_cancel_test.runs \o/$(MODE)/llamafile/pool_test.runs \@@ -170,34 +165,15 @@ o/$(MODE)/llamafile/pool_test: \o/$(MODE)/llamafile/crash.o \o/$(MODE)/llamafile/pool.o \-o/$(MODE)/llamafile/highlight_test: \- o/$(MODE)/llamafile/highlight_test.o \- o/$(MODE)/llama.cpp/llama.cpp.a \--o/$(MODE)/llamafile/highlight_c_test: \- o/$(MODE)/llamafile/highlight_c_test.o \- o/$(MODE)/llamafile/highlight_c.o \- o/$(MODE)/llamafile/is_keyword_c.o \- o/$(MODE)/llamafile/is_keyword_c_constant.o \- o/$(MODE)/llamafile/is_keyword_c_type.o \- o/$(MODE)/llamafile/is_keyword_c_pod.o \- o/$(MODE)/llamafile/is_keyword_cpp.o \--o/$(MODE)/llamafile/highlight_python_test: \- o/$(MODE)/llamafile/highlight_python_test.o \- o/$(MODE)/llamafile/highlight_python.o \- o/$(MODE)/llamafile/is_keyword_python.o \- o/$(MODE)/llamafile/is_keyword_python_builtin.o \- o/$(MODE)/llamafile/is_keyword_python_constant.o \-o/$(MODE)/llamafile/datauri_test: \o/$(MODE)/llamafile/datauri_test.o \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a \+ o/$(MODE)/third_party/stb/stb.a \-o/$(MODE)/llamafile/high: \- o/$(MODE)/llamafile/high.o \- o/$(MODE)/llama.cpp/llama.cpp.a \+o/$(MODE)/llamafile/high: \+ o/$(MODE)/llamafile/high.o \+ o/$(MODE)/llamafile/highlight/highlight.a \+ o/$(MODE)/llama.cpp/llama.cpp.a \o/$(MODE)/llamafile/hex2xterm: \o/$(MODE)/llamafile/hex2xterm.o \@@ -257,5 +233,4 @@ o/$(MODE)/llamafile/pick_a_warp_kernel.o: private CFLAGS += -fopenmp.PHONY: o/$(MODE)/llamafile/checko/$(MODE)/llamafile/check: \- $(LLAMAFILE_SRCS_GPERF_C) \o/$(MODE)/llamafile/tinyblas_test.runsdiff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cppindex 41daea8c..ad29dbd3 100644--- a/llamafile/chatbot_repl.cpp+++ b/llamafile/chatbot_repl.cpp@@ -26,7 +26,7 @@#include "llama.cpp/llama.h"#include "llamafile/bestline.h"#include "llamafile/color.h"-#include "llamafile/highlight.h"+#include "llamafile/highlight/highlight.h"#include "llamafile/llama.h"namespace lf {diff --git a/llamafile/high.cpp b/llamafile/high.cppindex c19b9d46..24c0deae 100644--- a/llamafile/high.cpp+++ b/llamafile/high.cpp@@ -15,7 +15,7 @@// See the License for the specific language governing permissions and// limitations under the License.-#include "highlight.h"+#include "llamafile/highlight/highlight.h"#include "string.h"#include <fcntl.h>#include <stdio.h>diff --git a/llamafile/highlight.h b/llamafile/highlight.hdeleted file mode 100644index 99cc6ce4..00000000--- a/llamafile/highlight.h+++ /dev/null@@ -1,813 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#pragma once-#include <string>-#include <string_view>--#define HI_RESET "\033[0m"-#define HI_BOLD "\033[1m"-#define HI_UNBOLD "\033[22m"-#define HI_ITALIC HI_INVERT-#define HI_INVERT "\033[7m"-#define HI_KEYWORD "\033[1;34m" // bold blue-#define HI_STRING "\033[32m" // green-#define HI_COMMENT "\033[31m" // red-#define HI_VAR "\033[1;35m" // magenta-#define HI_MACRO "\033[35m" // magenta-#define HI_ATTRIB "\033[35m" // magenta-#define HI_LINENO "\033[2m" // fade-#define HI_CONTIN "\033[33m" // yellow-#define HI_LABEL "\033[33m" // yellow-#define HI_DEF "\033[2;33m" // dim yellow-#define HI_TYPE "\033[36m" // cyan-#define HI_CLASS "\033[1;36m" // bold cyan-#define HI_SELECTOR "\033[33m" // yellow-#define HI_PROPERTY "\033[36m" // cyan-#define HI_TAG "\033[33m" // yellow-#define HI_INCODE "\033[1;35m" // magenta-#define HI_BUILTIN "\033[35m" // magenta-#define HI_CONSTANT "\033[1;35m" // bold magenta-#define HI_LISPKW "\033[35m" // magenta-#define HI_ENTITY "\033[36m" // cyan-#define HI_OPERATOR "\033[36m" // cyan-#define HI_ESCAPE "\033[33m" // yellow-#define HI_QUALIFIER "\033[35m" // magenta-#define HI_IMMEDIATE "\033[36m" // cyan-#define HI_REGISTER "\033[1;35m" // bold magenta-#define HI_DIRECTIVE "\033[1;31m" // bold red-#define HI_WARNING "\033[1;31m" // bold red-#define HI_MATH "\033[1;35m" // bold magenta--typedef const char *is_keyword_f(const char *, size_t);--extern "C" {-is_keyword_f is_keyword_c;-is_keyword_f is_keyword_c_pod;-is_keyword_f is_keyword_c_type;-is_keyword_f is_keyword_c_builtin;-is_keyword_f is_keyword_c_constant;-is_keyword_f is_keyword_cpp;-is_keyword_f is_keyword_cxx;-is_keyword_f is_keyword_js;-is_keyword_f is_keyword_js_builtin;-is_keyword_f is_keyword_js_constant;-is_keyword_f is_keyword_java;-is_keyword_f is_keyword_java_constant;-is_keyword_f is_keyword_python;-is_keyword_f is_keyword_python_builtin;-is_keyword_f is_keyword_python_constant;-is_keyword_f is_keyword_rust;-is_keyword_f is_keyword_rust_type;-is_keyword_f is_keyword_rust_constant;-is_keyword_f is_keyword_fortran;-is_keyword_f is_keyword_fortran_type;-is_keyword_f is_keyword_fortran_builtin;-is_keyword_f is_keyword_cobol;-is_keyword_f is_keyword_pascal;-is_keyword_f is_keyword_pascal_type;-is_keyword_f is_keyword_pascal_builtin;-is_keyword_f is_keyword_go;-is_keyword_f is_keyword_go_type;-is_keyword_f is_keyword_sql;-is_keyword_f is_keyword_sql_type;-is_keyword_f is_keyword_php;-is_keyword_f is_keyword_php_constant;-is_keyword_f is_keyword_csharp;-is_keyword_f is_keyword_csharp_constant;-is_keyword_f is_keyword_kotlin;-is_keyword_f is_keyword_kotlin_type;-is_keyword_f is_keyword_lua;-is_keyword_f is_keyword_lua_builtin;-is_keyword_f is_keyword_lua_constant;-is_keyword_f is_keyword_lisp;-is_keyword_f is_keyword_ada;-is_keyword_f is_keyword_ada_constant;-is_keyword_f is_keyword_haskell;-is_keyword_f is_keyword_perl;-is_keyword_f is_keyword_shell;-is_keyword_f is_keyword_shell_builtin;-is_keyword_f is_keyword_swift;-is_keyword_f is_keyword_swift_type;-is_keyword_f is_keyword_swift_builtin;-is_keyword_f is_keyword_swift_constant;-is_keyword_f is_keyword_d;-is_keyword_f is_keyword_d_constant;-is_keyword_f is_keyword_zig;-is_keyword_f is_keyword_zig_type;-is_keyword_f is_keyword_zig_builtin;-is_keyword_f is_keyword_zig_constant;-is_keyword_f is_keyword_tcl;-is_keyword_f is_keyword_tcl_type;-is_keyword_f is_keyword_tcl_builtin;-is_keyword_f is_keyword_ruby;-is_keyword_f is_keyword_ruby_builtin;-is_keyword_f is_keyword_ruby_constant;-is_keyword_f is_keyword_typescript;-is_keyword_f is_keyword_typescript_type;-is_keyword_f is_keyword_typescript_constant;-is_keyword_f is_keyword_forth;-is_keyword_f is_keyword_forth_def;-is_keyword_f is_keyword_m4;-is_keyword_f is_keyword_make;-is_keyword_f is_keyword_make_builtin;-is_keyword_f is_keyword_asm_prefix;-is_keyword_f is_keyword_asm_qualifier;-is_keyword_f is_keyword_basic;-is_keyword_f is_keyword_basic_type;-is_keyword_f is_keyword_basic_builtin;-is_keyword_f is_keyword_basic_constant;-is_keyword_f is_keyword_ld;-is_keyword_f is_keyword_ld_builtin;-is_keyword_f is_keyword_ld_warning;-is_keyword_f is_keyword_matlab;-is_keyword_f is_keyword_matlab_builtin;-is_keyword_f is_keyword_matlab_constant;-is_keyword_f is_keyword_r;-is_keyword_f is_keyword_r_builtin;-is_keyword_f is_keyword_r_constant;-is_keyword_f is_keyword_scala;-is_keyword_f is_keyword_julia;-is_keyword_f is_keyword_ocaml;-is_keyword_f is_keyword_ocaml_builtin;-is_keyword_f is_keyword_ocaml_constant;-is_keyword_f is_keyword_cmake;-is_keyword_f is_keyword_css_at;-is_keyword_f is_keyword_css_bang;-}--class Highlight {- public:- static Highlight *create(const std::string_view &lang);- virtual ~Highlight() = default;- virtual void feed(std::string *result, std::string_view input) = 0;- virtual void flush(std::string *result) = 0;-};--class HighlightTxt : public Highlight {- public:- HighlightTxt();- ~HighlightTxt() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-};--class ColorBleeder : public Highlight {- public:- ColorBleeder(Highlight *h);- ~ColorBleeder() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- void relay(std::string *r, const std::string &s);- void restore(std::string *r);- unsigned char t_ = 0;- unsigned char x_ = 0;- unsigned char n_ = 0;- unsigned char inverted_ = 0;- unsigned char intensity_ = 0;- unsigned char foreground_ = 0;- unsigned char background_ = 0;- unsigned char sgr_codes_[8];- Highlight *h_;-};--class HighlightC : public Highlight {- public:- HighlightC(is_keyword_f *is_keyword = is_keyword_c, is_keyword_f *is_type = nullptr,- is_keyword_f *is_builtin = nullptr, is_keyword_f *is_constant = nullptr);- ~HighlightC() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int i_ = 0;- bool is_pod_ = 0;- bool is_bol_ = true;- bool is_cpp_ = false;- bool is_define_ = false;- bool is_include_ = false;- std::string word_;- std::string heredoc_;- is_keyword_f *is_type_;- is_keyword_f *is_keyword_;- is_keyword_f *is_builtin_;- is_keyword_f *is_constant_;-};--class HighlightD : public Highlight {- public:- HighlightD();- ~HighlightD() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int depth_ = 0;- unsigned char opener_ = 0;- unsigned char closer_ = 0;- std::string heredoc_;- std::string heredoc2_;- std::string word_;-};--class HighlightJava : public Highlight {- public:- HighlightJava();- ~HighlightJava() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightGo : public Highlight {- public:- HighlightGo();- ~HighlightGo() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightJs : public Highlight {- public:- HighlightJs();- ~HighlightJs() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int expect_;- int nesti_ = 0;- std::string word_;- unsigned char nest_[16];-};--class HighlightTypescript : public Highlight {- public:- HighlightTypescript();- ~HighlightTypescript() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int expect_;- int nesti_ = 0;- std::string word_;- unsigned char nest_[16];-};--class HighlightPython : public Highlight {- public:- HighlightPython();- ~HighlightPython() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightMarkdown : public Highlight {- public:- HighlightMarkdown();- ~HighlightMarkdown() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- bool bol_ = true;- bool tail_ = false;- std::string lang_;- Highlight *highlighter_ = nullptr;-};--class HighlightRust : public Highlight {- public:- HighlightRust();- ~HighlightRust() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int nest_ = 0;- std::string word_;-};--class HighlightFortran : public Highlight {- public:- HighlightFortran();- ~HighlightFortran() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int col_ = -1;- std::string word_;-};--class HighlightCobol : public Highlight {- public:- HighlightCobol();- ~HighlightCobol() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int col_ = -1;- std::string word_;-};--class HighlightPascal : public Highlight {- public:- HighlightPascal();- ~HighlightPascal() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightSql : public Highlight {- public:- HighlightSql();- ~HighlightSql() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightCss : public Highlight {- public:- HighlightCss();- ~HighlightCss() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightHtml : public Highlight {- public:- HighlightHtml();- ~HighlightHtml() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int i_ = 0;- std::string name_;- std::string closer_;- std::string pending_;- Highlight *highlighter_ = nullptr;-};--class HighlightPhp : public Highlight {- public:- HighlightPhp();- ~HighlightPhp() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightLua : public Highlight {- public:- HighlightLua();- ~HighlightLua() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int level1_;- int level2_;- std::string word_;-};--class HighlightLisp : public Highlight {- public:- HighlightLisp();- ~HighlightLisp() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- bool is_first_ = false;- std::string symbol_;-};--class HighlightAda : public Highlight {- public:- HighlightAda();- ~HighlightAda() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int t_ = 0;- int last_ = 0;- std::string symbol_;-};--class HighlightHaskell : public Highlight {- public:- HighlightHaskell();- ~HighlightHaskell() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int level_ = 0;- std::string symbol_;-};--class HighlightPerl : public Highlight {- public:- HighlightPerl();- ~HighlightPerl() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int i_ = 0;- int last_ = 0;- int expect_ = 0;- unsigned char opener_ = 0;- unsigned char closer_ = 0;- bool pending_heredoc_ = false;- bool indented_heredoc_ = false;- std::string word_;- std::string heredoc_;-};--class HighlightShell : public Highlight {- public:- HighlightShell();- ~HighlightShell() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int i_ = 0;- int curl_ = 0;- int last_ = 0;- bool pending_heredoc_ = false;- bool indented_heredoc_ = false;- bool no_interpolation_ = false;- std::string word_;- std::string heredoc_;-};--class HighlightZig : public Highlight {- public:- HighlightZig();- ~HighlightZig() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- std::string word_;-};--class HighlightTcl : public Highlight {- public:- HighlightTcl();- ~HighlightTcl() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- std::string word_;-};--class HighlightCsharp : public Highlight {- public:- HighlightCsharp();- ~HighlightCsharp() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int trips1_;- int trips2_;- std::string word_;-};--class HighlightRuby : public Highlight {- public:- HighlightRuby();- ~HighlightRuby() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int i_ = 0;- int level_ = 0;- int nesti_ = 0;- int expect_ = 0;- unsigned char q_ = 0;- unsigned char opener_ = 0;- unsigned char closer_ = 0;- bool is_definition_ = 0;- bool pending_heredoc_ = false;- bool indented_heredoc_ = false;- unsigned char nest_[16];- unsigned char levels_[16];- unsigned char openers_[16];- unsigned char closers_[16];- std::string word_;- std::string heredoc_;-};--class HighlightForth : public Highlight {- public:- HighlightForth();- ~HighlightForth() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- char closer_ = 0;- bool is_label_ = false;- std::string word_;-};--class HighlightM4 : public Highlight {- public:- HighlightM4();- ~HighlightM4() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightMake : public Highlight {- public:- HighlightMake();- ~HighlightMake() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- std::string word_;-};--class HighlightAsm : public Highlight {- public:- HighlightAsm();- ~HighlightAsm() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int col_ = 0;- int last_ = 0;- bool is_preprocessor_ = false;- bool is_first_thing_on_line_ = true;- std::string word_;-};--class HighlightBasic : public Highlight {- public:- HighlightBasic();- ~HighlightBasic() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- bool is_bol_ = true;- std::string word_;-};--class HighlightLd : public Highlight {- public:- HighlightLd();- ~HighlightLd() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- bool is_bol_ = true;- bool is_cpp_ = false;- std::string word_;-};--class HighlightTex : public Highlight {- public:- HighlightTex();- ~HighlightTex() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- std::string word_;-};--class HighlightKotlin : public Highlight {- public:- HighlightKotlin();- ~HighlightKotlin() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int nesti_ = 0;- std::string word_;- unsigned char nest_[16];-};--class HighlightMatlab : public Highlight {- public:- HighlightMatlab();- ~HighlightMatlab() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightR : public Highlight {- public:- HighlightR();- ~HighlightR() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightSwift : public Highlight {- public:- HighlightSwift();- ~HighlightSwift() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int hash1_ = 0;- int hash2_ = 0;- int nesti_ = 0;- int expect_ = 0;- std::string word_;- unsigned char nest_[16];- unsigned char hash_[16];-};--class HighlightScala : public Highlight {- public:- HighlightScala();- ~HighlightScala() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int nesti_ = 0;- std::string word_;- unsigned char nest_[16];-};--class HighlightJulia : public Highlight {- public:- HighlightJulia();- ~HighlightJulia() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string word_;-};--class HighlightOcaml : public Highlight {- public:- HighlightOcaml();- ~HighlightOcaml() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- int nest_ = 0;- std::string word_;- std::string word2_;-};--class HighlightCmake : public Highlight {- public:- HighlightCmake();- ~HighlightCmake() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int c_ = 0;- int u_ = 0;- int t_ = 0;- int spaces_ = 0;- std::string word_;-};--class HighlightBnf : public Highlight {- public:- HighlightBnf();- ~HighlightBnf() override;- void feed(std::string *result, std::string_view input) override;- void flush(std::string *result) override;-- private:- int t_ = 0;- std::string operator_;-};diff --git a/llamafile/highlight/.clang-format b/llamafile/highlight/.clang-formatnew file mode 100644index 00000000..d1ee89c9--- /dev/null+++ b/llamafile/highlight/.clang-format@@ -0,0 +1,14 @@+---+BasedOnStyle: Mozilla+IndentWidth: 4+ColumnLimit: 80+---+Language: Cpp+AllowShortFunctionsOnASingleLine: false+AlignTrailingComments: false+AlignEscapedNewlines: DontAlign+AlwaysBreakTemplateDeclarations: true+ConstructorInitializerAllOnOneLineOrOnePerLine: true+FixNamespaceComments: true+IndentCaseLabels: false+---diff --git a/llamafile/highlight/BUILD.mk b/llamafile/highlight/BUILD.mknew file mode 100644index 00000000..e37b604c--- /dev/null+++ b/llamafile/highlight/BUILD.mk@@ -0,0 +1,54 @@+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘++PKGS += LLAMAFILE_HIGHLIGHT++LLAMAFILE_HIGHLIGHT_FILES := $(wildcard llamafile/highlight/*)+LLAMAFILE_HIGHLIGHT_HDRS = $(filter %.h,$(LLAMAFILE_HIGHLIGHT_FILES))+LLAMAFILE_HIGHLIGHT_INCS = $(filter %.inc,$(LLAMAFILE_HIGHLIGHT_FILES))+LLAMAFILE_HIGHLIGHT_SRCS_C = $(filter %.c,$(LLAMAFILE_HIGHLIGHT_FILES))+LLAMAFILE_HIGHLIGHT_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_HIGHLIGHT_FILES))+LLAMAFILE_HIGHLIGHT_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_HIGHLIGHT_FILES))+LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C = $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)++LLAMAFILE_HIGHLIGHT_SRCS = \+ $(LLAMAFILE_HIGHLIGHT_SRCS_C) \+ $(LLAMAFILE_HIGHLIGHT_SRCS_CPP) \+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF) \++LLAMAFILE_HIGHLIGHT_OBJS = \+ $(LLAMAFILE_HIGHLIGHT_SRCS_C:%.c=o/$(MODE)/%.o) \+ $(LLAMAFILE_HIGHLIGHT_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C:%.c=%.o) \++o/$(MODE)/llamafile/highlight/highlight.a: $(LLAMAFILE_HIGHLIGHT_OBJS)++$(LLAMAFILE_HIGHLIGHT_OBJS): llamafile/highlight/BUILD.mk++o/$(MODE)/llamafile/highlight/highlight_test: \+ o/$(MODE)/llamafile/highlight/highlight_test.o \+ o/$(MODE)/llamafile/highlight/highlight.a \++o/$(MODE)/llamafile/highlight/highlight_c_test: \+ o/$(MODE)/llamafile/highlight/highlight_c_test.o \+ o/$(MODE)/llamafile/highlight/highlight_c.o \+ o/$(MODE)/llamafile/highlight/is_keyword_c.o \+ o/$(MODE)/llamafile/highlight/is_keyword_c_constant.o \+ o/$(MODE)/llamafile/highlight/is_keyword_c_type.o \+ o/$(MODE)/llamafile/highlight/is_keyword_c_pod.o \+ o/$(MODE)/llamafile/highlight/is_keyword_cpp.o \++o/$(MODE)/llamafile/highlight/highlight_python_test: \+ o/$(MODE)/llamafile/highlight/highlight_python_test.o \+ o/$(MODE)/llamafile/highlight/highlight_python.o \+ o/$(MODE)/llamafile/highlight/is_keyword_python.o \+ o/$(MODE)/llamafile/highlight/is_keyword_python_builtin.o \+ o/$(MODE)/llamafile/highlight/is_keyword_python_constant.o \++.PHONY: o/$(MODE)/llamafile/highlight+o/$(MODE)/llamafile/highlight: \+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C) \+ o/$(MODE)/llamafile/highlight/highlight.a \+ o/$(MODE)/llamafile/highlight/highlight_c_test.runs \+ o/$(MODE)/llamafile/highlight/highlight_python_test.runs \+ o/$(MODE)/llamafile/highlight/highlight_test.runs \diff --git a/llamafile/color_bleeder.cpp b/llamafile/highlight/color_bleeder.cppsimilarity index 94%rename from llamafile/color_bleeder.cpprename to llamafile/highlight/color_bleeder.cppindex 7cc33b6c..e338fcc8 100644--- a/llamafile/color_bleeder.cpp+++ b/llamafile/highlight/color_bleeder.cpp@@ -37,19 +37,24 @@//// only the subset of ansi escape codes that we use are supported.-enum {+enum+{NORMAL,ESC,CSI,};-ColorBleeder::ColorBleeder(Highlight *h) : h_(h) {+ColorBleeder::ColorBleeder(Highlight* h) : h_(h)+{}-ColorBleeder::~ColorBleeder() {+ColorBleeder::~ColorBleeder()+{}-void ColorBleeder::restore(std::string *r) {+void+ColorBleeder::restore(std::string* r)+{bool got_some = false;if (!intensity_ && !inverted_ && !foreground_ && !background_)return;@@ -78,7 +83,9 @@ void ColorBleeder::restore(std::string *r) {*r += 'm';}-void ColorBleeder::relay(std::string *r, const std::string &s) {+void+ColorBleeder::relay(std::string* r, const std::string& s)+{for (char c : s) {*r += c;switch (t_) {@@ -171,13 +178,17 @@ void ColorBleeder::relay(std::string *r, const std::string &s) {}}-void ColorBleeder::feed(std::string *r, std::string_view input) {+void+ColorBleeder::feed(std::string* r, std::string_view input)+{std::string s;h_->feed(&s, input);relay(r, s);}-void ColorBleeder::flush(std::string *r) {+void+ColorBleeder::flush(std::string* r)+{std::string s;h_->flush(&s);relay(r, s);diff --git a/llamafile/highlight.cpp b/llamafile/highlight/highlight.cppsimilarity index 50%rename from llamafile/highlight.cpprename to llamafile/highlight/highlight.cppindex c4d3b85e..c0e3de60 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight/highlight.cpp@@ -17,197 +17,289 @@#include "highlight.h"-struct HighlightEntry {- char *name;- Highlight *(*ctor)(void);+struct HighlightEntry+{+ char* name;+ Highlight* (*ctor)(void);};-extern "C" Highlight *highlight_create_ada_(void) {+extern "C" Highlight*+highlight_create_ada_(void)+{return new HighlightAda;}-extern "C" Highlight *highlight_create_asm_(void) {+extern "C" Highlight*+highlight_create_asm_(void)+{return new HighlightAsm;}-extern "C" Highlight *highlight_create_basic_(void) {+extern "C" Highlight*+highlight_create_basic_(void)+{return new HighlightBasic;}-extern "C" Highlight *highlight_create_bnf_(void) {+extern "C" Highlight*+highlight_create_bnf_(void)+{return new HighlightBnf;}-extern "C" Highlight *highlight_create_c_(void) {+extern "C" Highlight*+highlight_create_c_(void)+{return new HighlightC(is_keyword_c, //is_keyword_c_type, //is_keyword_c_builtin, //is_keyword_c_constant);}-extern "C" Highlight *highlight_create_cxx_(void) {+extern "C" Highlight*+highlight_create_cxx_(void)+{return new HighlightC(is_keyword_cxx, //is_keyword_c_type, //is_keyword_c_builtin, //is_keyword_c_constant);}-extern "C" Highlight *highlight_create_cmake_(void) {+extern "C" Highlight*+highlight_create_cmake_(void)+{return new HighlightCmake;}-extern "C" Highlight *highlight_create_cobol_(void) {+extern "C" Highlight*+highlight_create_cobol_(void)+{return new HighlightCobol;}-extern "C" Highlight *highlight_create_csharp_(void) {+extern "C" Highlight*+highlight_create_csharp_(void)+{return new HighlightCsharp;}-extern "C" Highlight *highlight_create_css_(void) {+extern "C" Highlight*+highlight_create_css_(void)+{return new HighlightCss;}-extern "C" Highlight *highlight_create_d_(void) {+extern "C" Highlight*+highlight_create_d_(void)+{return new HighlightD;}-extern "C" Highlight *highlight_create_forth_(void) {+extern "C" Highlight*+highlight_create_forth_(void)+{return new HighlightForth;}-extern "C" Highlight *highlight_create_fortran_(void) {+extern "C" Highlight*+highlight_create_fortran_(void)+{return new HighlightFortran;}-extern "C" Highlight *highlight_create_go_(void) {+extern "C" Highlight*+highlight_create_go_(void)+{return new HighlightGo;}-extern "C" Highlight *highlight_create_haskell_(void) {+extern "C" Highlight*+highlight_create_haskell_(void)+{return new HighlightHaskell;}-extern "C" Highlight *highlight_create_html_(void) {+extern "C" Highlight*+highlight_create_html_(void)+{return new HighlightHtml;}-extern "C" Highlight *highlight_create_java_(void) {+extern "C" Highlight*+highlight_create_java_(void)+{return new HighlightJava;}-extern "C" Highlight *highlight_create_js_(void) {+extern "C" Highlight*+highlight_create_js_(void)+{return new HighlightJs;}-extern "C" Highlight *highlight_create_julia_(void) {+extern "C" Highlight*+highlight_create_julia_(void)+{return new HighlightJulia;}-extern "C" Highlight *highlight_create_kotlin_(void) {+extern "C" Highlight*+highlight_create_kotlin_(void)+{return new HighlightKotlin;}-extern "C" Highlight *highlight_create_ld_(void) {+extern "C" Highlight*+highlight_create_ld_(void)+{return new HighlightLd;}-extern "C" Highlight *highlight_create_lisp_(void) {+extern "C" Highlight*+highlight_create_lisp_(void)+{return new HighlightLisp;}-extern "C" Highlight *highlight_create_lua_(void) {+extern "C" Highlight*+highlight_create_lua_(void)+{return new HighlightLua;}-extern "C" Highlight *highlight_create_m4_(void) {+extern "C" Highlight*+highlight_create_m4_(void)+{return new HighlightM4;}-extern "C" Highlight *highlight_create_make_(void) {+extern "C" Highlight*+highlight_create_make_(void)+{return new HighlightMake;}-extern "C" Highlight *highlight_create_markdown_(void) {+extern "C" Highlight*+highlight_create_markdown_(void)+{return new HighlightMarkdown;}-extern "C" Highlight *highlight_create_matlab_(void) {+extern "C" Highlight*+highlight_create_matlab_(void)+{return new HighlightMatlab;}-extern "C" Highlight *highlight_create_ocaml_(void) {+extern "C" Highlight*+highlight_create_ocaml_(void)+{return new HighlightOcaml;}-extern "C" Highlight *highlight_create_pascal_(void) {+extern "C" Highlight*+highlight_create_pascal_(void)+{return new HighlightPascal;}-extern "C" Highlight *highlight_create_perl_(void) {+extern "C" Highlight*+highlight_create_perl_(void)+{return new HighlightPerl;}-extern "C" Highlight *highlight_create_php_(void) {+extern "C" Highlight*+highlight_create_php_(void)+{return new HighlightPhp;}-extern "C" Highlight *highlight_create_python_(void) {+extern "C" Highlight*+highlight_create_python_(void)+{return new HighlightPython;}-extern "C" Highlight *highlight_create_r_(void) {+extern "C" Highlight*+highlight_create_r_(void)+{return new HighlightR;}-extern "C" Highlight *highlight_create_ruby_(void) {+extern "C" Highlight*+highlight_create_ruby_(void)+{return new HighlightRuby;}-extern "C" Highlight *highlight_create_rust_(void) {+extern "C" Highlight*+highlight_create_rust_(void)+{return new HighlightRust;}-extern "C" Highlight *highlight_create_scala_(void) {+extern "C" Highlight*+highlight_create_scala_(void)+{return new HighlightScala;}-extern "C" Highlight *highlight_create_shell_(void) {+extern "C" Highlight*+highlight_create_shell_(void)+{return new HighlightShell;}-extern "C" Highlight *highlight_create_sql_(void) {+extern "C" Highlight*+highlight_create_sql_(void)+{return new HighlightSql;}-extern "C" Highlight *highlight_create_swift_(void) {+extern "C" Highlight*+highlight_create_swift_(void)+{return new HighlightSwift;}-extern "C" Highlight *highlight_create_tcl_(void) {+extern "C" Highlight*+highlight_create_tcl_(void)+{return new HighlightTcl;}-extern "C" Highlight *highlight_create_tex_(void) {+extern "C" Highlight*+highlight_create_tex_(void)+{return new HighlightTex;}-extern "C" Highlight *highlight_create_txt_(void) {+extern "C" Highlight*+highlight_create_txt_(void)+{return new HighlightTxt;}-extern "C" Highlight *highlight_create_typescript_(void) {+extern "C" Highlight*+highlight_create_typescript_(void)+{return new HighlightTypescript;}-extern "C" Highlight *highlight_create_zig_(void) {+extern "C" Highlight*+highlight_create_zig_(void)+{return new HighlightZig;}-extern "C" const HighlightEntry *highlight_lookup_(const char *str, size_t len);+extern "C" const HighlightEntry*+highlight_lookup_(const char* str, size_t len);-Highlight *Highlight::create(const std::string_view &lang) {- const HighlightEntry *slot;+Highlight*+Highlight::create(const std::string_view& lang)+{+ const HighlightEntry* slot;if ((slot = highlight_lookup_(lang.data(), lang.size())))return slot->ctor();return nullptr;diff --git a/llamafile/highlight/highlight.h b/llamafile/highlight/highlight.hnew file mode 100644index 00000000..1cc34cda--- /dev/null+++ b/llamafile/highlight/highlight.h@@ -0,0 +1,862 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <string>+#include <string_view>++#define HI_RESET "\033[0m"+#define HI_BOLD "\033[1m"+#define HI_UNBOLD "\033[22m"+#define HI_ITALIC HI_INVERT+#define HI_INVERT "\033[7m"+#define HI_KEYWORD "\033[1;34m" // bold blue+#define HI_STRING "\033[32m" // green+#define HI_COMMENT "\033[31m" // red+#define HI_VAR "\033[1;35m" // magenta+#define HI_MACRO "\033[35m" // magenta+#define HI_ATTRIB "\033[35m" // magenta+#define HI_LINENO "\033[2m" // fade+#define HI_CONTIN "\033[33m" // yellow+#define HI_LABEL "\033[33m" // yellow+#define HI_DEF "\033[2;33m" // dim yellow+#define HI_TYPE "\033[36m" // cyan+#define HI_CLASS "\033[1;36m" // bold cyan+#define HI_SELECTOR "\033[33m" // yellow+#define HI_PROPERTY "\033[36m" // cyan+#define HI_TAG "\033[33m" // yellow+#define HI_INCODE "\033[1;35m" // magenta+#define HI_BUILTIN "\033[35m" // magenta+#define HI_CONSTANT "\033[1;35m" // bold magenta+#define HI_LISPKW "\033[35m" // magenta+#define HI_ENTITY "\033[36m" // cyan+#define HI_OPERATOR "\033[36m" // cyan+#define HI_ESCAPE "\033[33m" // yellow+#define HI_QUALIFIER "\033[35m" // magenta+#define HI_IMMEDIATE "\033[36m" // cyan+#define HI_REGISTER "\033[1;35m" // bold magenta+#define HI_DIRECTIVE "\033[1;31m" // bold red+#define HI_WARNING "\033[1;31m" // bold red+#define HI_MATH "\033[1;35m" // bold magenta++typedef const char*+is_keyword_f(const char*, size_t);++extern "C"+{+ is_keyword_f is_keyword_c;+ is_keyword_f is_keyword_c_pod;+ is_keyword_f is_keyword_c_type;+ is_keyword_f is_keyword_c_builtin;+ is_keyword_f is_keyword_c_constant;+ is_keyword_f is_keyword_cpp;+ is_keyword_f is_keyword_cxx;+ is_keyword_f is_keyword_js;+ is_keyword_f is_keyword_js_builtin;+ is_keyword_f is_keyword_js_constant;+ is_keyword_f is_keyword_java;+ is_keyword_f is_keyword_java_constant;+ is_keyword_f is_keyword_python;+ is_keyword_f is_keyword_python_builtin;+ is_keyword_f is_keyword_python_constant;+ is_keyword_f is_keyword_rust;+ is_keyword_f is_keyword_rust_type;+ is_keyword_f is_keyword_rust_constant;+ is_keyword_f is_keyword_fortran;+ is_keyword_f is_keyword_fortran_type;+ is_keyword_f is_keyword_fortran_builtin;+ is_keyword_f is_keyword_cobol;+ is_keyword_f is_keyword_pascal;+ is_keyword_f is_keyword_pascal_type;+ is_keyword_f is_keyword_pascal_builtin;+ is_keyword_f is_keyword_go;+ is_keyword_f is_keyword_go_type;+ is_keyword_f is_keyword_sql;+ is_keyword_f is_keyword_sql_type;+ is_keyword_f is_keyword_php;+ is_keyword_f is_keyword_php_constant;+ is_keyword_f is_keyword_csharp;+ is_keyword_f is_keyword_csharp_constant;+ is_keyword_f is_keyword_kotlin;+ is_keyword_f is_keyword_kotlin_type;+ is_keyword_f is_keyword_lua;+ is_keyword_f is_keyword_lua_builtin;+ is_keyword_f is_keyword_lua_constant;+ is_keyword_f is_keyword_lisp;+ is_keyword_f is_keyword_ada;+ is_keyword_f is_keyword_ada_constant;+ is_keyword_f is_keyword_haskell;+ is_keyword_f is_keyword_perl;+ is_keyword_f is_keyword_shell;+ is_keyword_f is_keyword_shell_builtin;+ is_keyword_f is_keyword_swift;+ is_keyword_f is_keyword_swift_type;+ is_keyword_f is_keyword_swift_builtin;+ is_keyword_f is_keyword_swift_constant;+ is_keyword_f is_keyword_d;+ is_keyword_f is_keyword_d_constant;+ is_keyword_f is_keyword_zig;+ is_keyword_f is_keyword_zig_type;+ is_keyword_f is_keyword_zig_builtin;+ is_keyword_f is_keyword_zig_constant;+ is_keyword_f is_keyword_tcl;+ is_keyword_f is_keyword_tcl_type;+ is_keyword_f is_keyword_tcl_builtin;+ is_keyword_f is_keyword_ruby;+ is_keyword_f is_keyword_ruby_builtin;+ is_keyword_f is_keyword_ruby_constant;+ is_keyword_f is_keyword_typescript;+ is_keyword_f is_keyword_typescript_type;+ is_keyword_f is_keyword_typescript_constant;+ is_keyword_f is_keyword_forth;+ is_keyword_f is_keyword_forth_def;+ is_keyword_f is_keyword_m4;+ is_keyword_f is_keyword_make;+ is_keyword_f is_keyword_make_builtin;+ is_keyword_f is_keyword_asm_prefix;+ is_keyword_f is_keyword_asm_qualifier;+ is_keyword_f is_keyword_basic;+ is_keyword_f is_keyword_basic_type;+ is_keyword_f is_keyword_basic_builtin;+ is_keyword_f is_keyword_basic_constant;+ is_keyword_f is_keyword_ld;+ is_keyword_f is_keyword_ld_builtin;+ is_keyword_f is_keyword_ld_warning;+ is_keyword_f is_keyword_matlab;+ is_keyword_f is_keyword_matlab_builtin;+ is_keyword_f is_keyword_matlab_constant;+ is_keyword_f is_keyword_r;+ is_keyword_f is_keyword_r_builtin;+ is_keyword_f is_keyword_r_constant;+ is_keyword_f is_keyword_scala;+ is_keyword_f is_keyword_julia;+ is_keyword_f is_keyword_ocaml;+ is_keyword_f is_keyword_ocaml_builtin;+ is_keyword_f is_keyword_ocaml_constant;+ is_keyword_f is_keyword_cmake;+ is_keyword_f is_keyword_css_at;+ is_keyword_f is_keyword_css_bang;+}++class Highlight+{+ public:+ static Highlight* create(const std::string_view& lang);+ virtual ~Highlight() = default;+ virtual void feed(std::string* result, std::string_view input) = 0;+ virtual void flush(std::string* result) = 0;+};++class HighlightTxt : public Highlight+{+ public:+ HighlightTxt();+ ~HighlightTxt() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;+};++class ColorBleeder : public Highlight+{+ public:+ ColorBleeder(Highlight* h);+ ~ColorBleeder() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ void relay(std::string* r, const std::string& s);+ void restore(std::string* r);+ unsigned char t_ = 0;+ unsigned char x_ = 0;+ unsigned char n_ = 0;+ unsigned char inverted_ = 0;+ unsigned char intensity_ = 0;+ unsigned char foreground_ = 0;+ unsigned char background_ = 0;+ unsigned char sgr_codes_[8];+ Highlight* h_;+};++class HighlightC : public Highlight+{+ public:+ HighlightC(is_keyword_f* is_keyword = is_keyword_c,+ is_keyword_f* is_type = nullptr,+ is_keyword_f* is_builtin = nullptr,+ is_keyword_f* is_constant = nullptr);+ ~HighlightC() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int i_ = 0;+ bool is_pod_ = 0;+ bool is_bol_ = true;+ bool is_cpp_ = false;+ bool is_define_ = false;+ bool is_include_ = false;+ std::string word_;+ std::string heredoc_;+ is_keyword_f* is_type_;+ is_keyword_f* is_keyword_;+ is_keyword_f* is_builtin_;+ is_keyword_f* is_constant_;+};++class HighlightD : public Highlight+{+ public:+ HighlightD();+ ~HighlightD() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int depth_ = 0;+ unsigned char opener_ = 0;+ unsigned char closer_ = 0;+ std::string heredoc_;+ std::string heredoc2_;+ std::string word_;+};++class HighlightJava : public Highlight+{+ public:+ HighlightJava();+ ~HighlightJava() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightGo : public Highlight+{+ public:+ HighlightGo();+ ~HighlightGo() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightJs : public Highlight+{+ public:+ HighlightJs();+ ~HighlightJs() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int expect_;+ int nesti_ = 0;+ std::string word_;+ unsigned char nest_[16];+};++class HighlightTypescript : public Highlight+{+ public:+ HighlightTypescript();+ ~HighlightTypescript() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int expect_;+ int nesti_ = 0;+ std::string word_;+ unsigned char nest_[16];+};++class HighlightPython : public Highlight+{+ public:+ HighlightPython();+ ~HighlightPython() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightMarkdown : public Highlight+{+ public:+ HighlightMarkdown();+ ~HighlightMarkdown() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ bool bol_ = true;+ bool tail_ = false;+ std::string lang_;+ Highlight* highlighter_ = nullptr;+};++class HighlightRust : public Highlight+{+ public:+ HighlightRust();+ ~HighlightRust() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int nest_ = 0;+ std::string word_;+};++class HighlightFortran : public Highlight+{+ public:+ HighlightFortran();+ ~HighlightFortran() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int col_ = -1;+ std::string word_;+};++class HighlightCobol : public Highlight+{+ public:+ HighlightCobol();+ ~HighlightCobol() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int col_ = -1;+ std::string word_;+};++class HighlightPascal : public Highlight+{+ public:+ HighlightPascal();+ ~HighlightPascal() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightSql : public Highlight+{+ public:+ HighlightSql();+ ~HighlightSql() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightCss : public Highlight+{+ public:+ HighlightCss();+ ~HighlightCss() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightHtml : public Highlight+{+ public:+ HighlightHtml();+ ~HighlightHtml() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int i_ = 0;+ std::string name_;+ std::string closer_;+ std::string pending_;+ Highlight* highlighter_ = nullptr;+};++class HighlightPhp : public Highlight+{+ public:+ HighlightPhp();+ ~HighlightPhp() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightLua : public Highlight+{+ public:+ HighlightLua();+ ~HighlightLua() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int level1_;+ int level2_;+ std::string word_;+};++class HighlightLisp : public Highlight+{+ public:+ HighlightLisp();+ ~HighlightLisp() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ bool is_first_ = false;+ std::string symbol_;+};++class HighlightAda : public Highlight+{+ public:+ HighlightAda();+ ~HighlightAda() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int t_ = 0;+ int last_ = 0;+ std::string symbol_;+};++class HighlightHaskell : public Highlight+{+ public:+ HighlightHaskell();+ ~HighlightHaskell() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int level_ = 0;+ std::string symbol_;+};++class HighlightPerl : public Highlight+{+ public:+ HighlightPerl();+ ~HighlightPerl() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int i_ = 0;+ int last_ = 0;+ int expect_ = 0;+ unsigned char opener_ = 0;+ unsigned char closer_ = 0;+ bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;+ std::string word_;+ std::string heredoc_;+};++class HighlightShell : public Highlight+{+ public:+ HighlightShell();+ ~HighlightShell() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int i_ = 0;+ int curl_ = 0;+ int last_ = 0;+ bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;+ bool no_interpolation_ = false;+ std::string word_;+ std::string heredoc_;+};++class HighlightZig : public Highlight+{+ public:+ HighlightZig();+ ~HighlightZig() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ std::string word_;+};++class HighlightTcl : public Highlight+{+ public:+ HighlightTcl();+ ~HighlightTcl() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ std::string word_;+};++class HighlightCsharp : public Highlight+{+ public:+ HighlightCsharp();+ ~HighlightCsharp() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int trips1_;+ int trips2_;+ std::string word_;+};++class HighlightRuby : public Highlight+{+ public:+ HighlightRuby();+ ~HighlightRuby() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int i_ = 0;+ int level_ = 0;+ int nesti_ = 0;+ int expect_ = 0;+ unsigned char q_ = 0;+ unsigned char opener_ = 0;+ unsigned char closer_ = 0;+ bool is_definition_ = 0;+ bool pending_heredoc_ = false;+ bool indented_heredoc_ = false;+ unsigned char nest_[16];+ unsigned char levels_[16];+ unsigned char openers_[16];+ unsigned char closers_[16];+ std::string word_;+ std::string heredoc_;+};++class HighlightForth : public Highlight+{+ public:+ HighlightForth();+ ~HighlightForth() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ char closer_ = 0;+ bool is_label_ = false;+ std::string word_;+};++class HighlightM4 : public Highlight+{+ public:+ HighlightM4();+ ~HighlightM4() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightMake : public Highlight+{+ public:+ HighlightMake();+ ~HighlightMake() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ std::string word_;+};++class HighlightAsm : public Highlight+{+ public:+ HighlightAsm();+ ~HighlightAsm() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int col_ = 0;+ int last_ = 0;+ bool is_preprocessor_ = false;+ bool is_first_thing_on_line_ = true;+ std::string word_;+};++class HighlightBasic : public Highlight+{+ public:+ HighlightBasic();+ ~HighlightBasic() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ bool is_bol_ = true;+ std::string word_;+};++class HighlightLd : public Highlight+{+ public:+ HighlightLd();+ ~HighlightLd() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ bool is_bol_ = true;+ bool is_cpp_ = false;+ std::string word_;+};++class HighlightTex : public Highlight+{+ public:+ HighlightTex();+ ~HighlightTex() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ std::string word_;+};++class HighlightKotlin : public Highlight+{+ public:+ HighlightKotlin();+ ~HighlightKotlin() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int nesti_ = 0;+ std::string word_;+ unsigned char nest_[16];+};++class HighlightMatlab : public Highlight+{+ public:+ HighlightMatlab();+ ~HighlightMatlab() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightR : public Highlight+{+ public:+ HighlightR();+ ~HighlightR() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightSwift : public Highlight+{+ public:+ HighlightSwift();+ ~HighlightSwift() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int hash1_ = 0;+ int hash2_ = 0;+ int nesti_ = 0;+ int expect_ = 0;+ std::string word_;+ unsigned char nest_[16];+ unsigned char hash_[16];+};++class HighlightScala : public Highlight+{+ public:+ HighlightScala();+ ~HighlightScala() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int nesti_ = 0;+ std::string word_;+ unsigned char nest_[16];+};++class HighlightJulia : public Highlight+{+ public:+ HighlightJulia();+ ~HighlightJulia() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string word_;+};++class HighlightOcaml : public Highlight+{+ public:+ HighlightOcaml();+ ~HighlightOcaml() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ int nest_ = 0;+ std::string word_;+ std::string word2_;+};++class HighlightCmake : public Highlight+{+ public:+ HighlightCmake();+ ~HighlightCmake() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int c_ = 0;+ int u_ = 0;+ int t_ = 0;+ int spaces_ = 0;+ std::string word_;+};++class HighlightBnf : public Highlight+{+ public:+ HighlightBnf();+ ~HighlightBnf() override;+ void feed(std::string* result, std::string_view input) override;+ void flush(std::string* result) override;++ private:+ int t_ = 0;+ std::string operator_;+};diff --git a/llamafile/highlight_ada.cpp b/llamafile/highlight/highlight_ada.cppsimilarity index 93%rename from llamafile/highlight_ada.cpprename to llamafile/highlight/highlight_ada.cppindex 581ec7ca..928fcf50 100644--- a/llamafile/highlight_ada.cpp+++ b/llamafile/highlight/highlight_ada.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,SYMBOL,QUOTE,@@ -28,13 +29,17 @@ enum {COMMENT,};-HighlightAda::HighlightAda() {+HighlightAda::HighlightAda()+{}-HighlightAda::~HighlightAda() {+HighlightAda::~HighlightAda()+{}-void HighlightAda::feed(std::string *r, std::string_view input) {+void+HighlightAda::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {last_ = c_;@@ -71,7 +76,8 @@ void HighlightAda::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += symbol_;*r += HI_RESET;- } else if (is_keyword_ada_constant(symbol_.data(), symbol_.size())) {+ } else if (is_keyword_ada_constant(symbol_.data(),+ symbol_.size())) {*r += HI_CONSTANT;*r += symbol_;*r += HI_RESET;@@ -126,7 +132,9 @@ void HighlightAda::feed(std::string *r, std::string_view input) {}}-void HighlightAda::flush(std::string *r) {+void+HighlightAda::flush(std::string* r)+{switch (t_) {case SYMBOL:if (is_keyword_ada(symbol_.data(), symbol_.size())) {diff --git a/llamafile/highlight_asm.cpp b/llamafile/highlight/highlight_asm.cppsimilarity index 95%rename from llamafile/highlight_asm.cpprename to llamafile/highlight/highlight_asm.cppindex ee520700..286e2684 100644--- a/llamafile/highlight_asm.cpp+++ b/llamafile/highlight/highlight_asm.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>@@ -32,7 +32,8 @@//// doing that requires special care-enum {+enum+{NORMAL,WORD,COMMENT,@@ -57,7 +58,9 @@ enum {HASH,};-static bool is_immediate(int c) {+static bool+is_immediate(int c)+{switch (c) {case '&':case '*':@@ -78,13 +81,17 @@ static bool is_immediate(int c) {}}-HighlightAsm::HighlightAsm() {+HighlightAsm::HighlightAsm()+{}-HighlightAsm::~HighlightAsm() {+HighlightAsm::~HighlightAsm()+{}-void HighlightAsm::feed(std::string *r, std::string_view input) {+void+HighlightAsm::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -240,7 +247,8 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {// fallthroughcase WORD:- if (!isascii(c) || isalnum(c) || c == '$' || c == '_' || c == '-' || c == '.') {+ if (!isascii(c) || isalnum(c) || c == '$' || c == '_' || c == '-' ||+ c == '.') {lf::append_wchar(&word_, c);} else {if (is_first_thing_on_line_) {@@ -255,7 +263,8 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {*r += HI_LABEL;*r += word_;*r += HI_RESET;- } else if (word_ == "C" || word_ == "dnl" || word_ == "m4_dnl") {+ } else if (word_ == "C" || word_ == "dnl" ||+ word_ == "m4_dnl") {*r += HI_COMMENT;*r += word_;word_.clear();@@ -268,11 +277,13 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {if (!is_keyword_asm_prefix(word_.data(), word_.size()))is_first_thing_on_line_ = false;}- } else if (is_preprocessor_ && is_keyword_c_builtin(word_.data(), word_.size())) {+ } else if (is_preprocessor_ &&+ is_keyword_c_builtin(word_.data(), word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_asm_qualifier(word_.data(), word_.size())) {+ } else if (is_keyword_asm_qualifier(word_.data(),+ word_.size())) {*r += HI_QUALIFIER;*r += word_;*r += HI_RESET;@@ -406,7 +417,9 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {}}-void HighlightAsm::flush(std::string *r) {+void+HighlightAsm::flush(std::string* r)+{switch (t_) {case WORD:*r += word_;diff --git a/llamafile/highlight_basic.cpp b/llamafile/highlight/highlight_basic.cppsimilarity index 89%rename from llamafile/highlight_basic.cpprename to llamafile/highlight/highlight_basic.cppindex 830ac824..acd1b6d4 100644--- a/llamafile/highlight_basic.cpp+++ b/llamafile/highlight/highlight_basic.cpp@@ -16,8 +16,7 @@// limitations under the License.#include "highlight.h"-#include "string.h"-#include <ctype.h>+#include <cctype>// beginner's all-purpose symbolic instruction code//@@ -29,7 +28,8 @@//// doing that requires extra keywords-enum {+enum+{NORMAL,WORD,LINENO,@@ -38,13 +38,26 @@ enum {DIRECTIVE,};-HighlightBasic::HighlightBasic() {+static std::string+ToLower(const std::string_view& s)+{+ std::string b;+ for (char c : s)+ b += std::tolower(c);+ return b;}-HighlightBasic::~HighlightBasic() {+HighlightBasic::HighlightBasic()+{}-void HighlightBasic::feed(std::string *r, std::string_view input) {+HighlightBasic::~HighlightBasic()+{+}++void+HighlightBasic::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -81,7 +94,7 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {word_ += c;} else {if (is_keyword_basic(word_.data(), word_.size())) {- if (lf::tolower(word_) == "rem") {+ if (ToLower(word_) == "rem") {*r += HI_COMMENT;*r += word_;t_ = COMMENT;@@ -96,11 +109,13 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {*r += HI_TYPE;*r += word_;*r += HI_RESET;- } else if (is_keyword_basic_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_basic_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_basic_constant(word_.data(), word_.size())) {+ } else if (is_keyword_basic_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -155,11 +170,13 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {}}-void HighlightBasic::flush(std::string *r) {+void+HighlightBasic::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_basic(word_.data(), word_.size())) {- if (lf::tolower(word_) == "rem") {+ if (ToLower(word_) == "rem") {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;diff --git a/llamafile/highlight_bnf.cpp b/llamafile/highlight/highlight_bnf.cppsimilarity index 94%rename from llamafile/highlight_bnf.cpprename to llamafile/highlight/highlight_bnf.cppindex ab7f0678..79d3ab3c 100644--- a/llamafile/highlight_bnf.cpp+++ b/llamafile/highlight/highlight_bnf.cpp@@ -17,7 +17,8 @@#include "highlight.h"-enum {+enum+{NORMAL,COMMENT,DQUOTE,@@ -28,7 +29,9 @@ enum {OPERATOR,};-static bool is_operator(const std::string &op) {+static bool+is_operator(const std::string& op)+{return op == "::=" || //op == "|" || //op == "?" || //@@ -36,7 +39,9 @@ static bool is_operator(const std::string &op) {op == "+";}-static bool is_operator_char(int c) {+static bool+is_operator_char(int c)+{switch (c) {case '!':case '$':@@ -64,13 +69,17 @@ static bool is_operator_char(int c) {}}-HighlightBnf::HighlightBnf() {+HighlightBnf::HighlightBnf()+{}-HighlightBnf::~HighlightBnf() {+HighlightBnf::~HighlightBnf()+{}-void HighlightBnf::feed(std::string *r, std::string_view input) {+void+HighlightBnf::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -178,7 +187,9 @@ void HighlightBnf::feed(std::string *r, std::string_view input) {}}-void HighlightBnf::flush(std::string *r) {+void+HighlightBnf::flush(std::string* r)+{switch (t_) {case OPERATOR:if (is_operator(operator_)) {diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight/highlight_c.cppsimilarity index 90%rename from llamafile/highlight_c.cpprename to llamafile/highlight/highlight_c.cppindex 2cc9b2f6..563a8d69 100644--- a/llamafile/highlight_c.cpp+++ b/llamafile/highlight/highlight_c.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -41,20 +42,24 @@ enum {UNIVERSAL,};-HighlightC::HighlightC(is_keyword_f *is_keyword, //- is_keyword_f *is_type, //- is_keyword_f *is_builtin, //- is_keyword_f *is_constant)- : is_keyword_(is_keyword),- is_type_(is_type),- is_builtin_(is_builtin),- is_constant_(is_constant) {+HighlightC::HighlightC(is_keyword_f* is_keyword, //+ is_keyword_f* is_type, //+ is_keyword_f* is_builtin, //+ is_keyword_f* is_constant)+ : is_keyword_(is_keyword)+ , is_type_(is_type)+ , is_builtin_(is_builtin)+ , is_constant_(is_constant)+{}-HighlightC::~HighlightC() {+HighlightC::~HighlightC()+{}-void HighlightC::feed(std::string *r, std::string_view input) {+void+HighlightC::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -121,7 +126,8 @@ void HighlightC::feed(std::string *r, std::string_view input) {is_define_ = true;}*r += HI_RESET;- } else if (is_keyword_c_constant(word_.data(), word_.size())) {+ } else if (is_keyword_c_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -135,17 +141,21 @@ void HighlightC::feed(std::string *r, std::string_view input) {*r += HI_RESET;if (is_keyword_c_pod(word_.data(), word_.size()))is_pod_ = true;- } else if (is_pod_ || (is_type_ && is_type_(word_.data(), word_.size()))) {+ } else if (is_pod_ ||+ (is_type_ &&+ is_type_(word_.data(), word_.size()))) {*r += HI_TYPE;*r += word_;*r += HI_RESET;is_pod_ = false;- } else if (is_builtin_ && is_builtin_(word_.data(), word_.size())) {+ } else if (is_builtin_ &&+ is_builtin_(word_.data(), word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;is_pod_ = false;- } else if (is_constant_ && is_constant_(word_.data(), word_.size())) {+ } else if (is_constant_ &&+ is_constant_(word_.data(), word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -360,7 +370,9 @@ void HighlightC::feed(std::string *r, std::string_view input) {}}-void HighlightC::flush(std::string *r) {+void+HighlightC::flush(std::string* r)+{switch (t_) {case WORD:if (is_cpp_) {@@ -381,7 +393,8 @@ void HighlightC::flush(std::string *r) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_pod_ || (is_type_ && is_type_(word_.data(), word_.size()))) {+ } else if (is_pod_ ||+ (is_type_ && is_type_(word_.data(), word_.size()))) {*r += HI_TYPE;*r += word_;*r += HI_RESET;@@ -389,7 +402,8 @@ void HighlightC::flush(std::string *r) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_constant_ && is_constant_(word_.data(), word_.size())) {+ } else if (is_constant_ &&+ is_constant_(word_.data(), word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight/highlight_c_test.cppsimilarity index 74%rename from llamafile/highlight_c_test.cpprename to llamafile/highlight/highlight_c_test.cppindex 227baf46..bd1ad9f6 100644--- a/llamafile/highlight_c_test.cpp+++ b/llamafile/highlight/highlight_c_test.cpp@@ -24,7 +24,9 @@ using namespace std::string_literals;HighlightC g_highlight;-void test(std::string input, std::string expect) {+void+test(std::string input, std::string expect)+{std::string got;g_highlight.feed(&got, input);if (got != expect) {@@ -33,12 +35,16 @@ void test(std::string input, std::string expect) {"\t input: %`'s\n""\texpected: %`'s\n""\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());+ input.c_str(),+ expect.c_str(),+ got.c_str());exit(1);}}-void testf(std::string input, std::string expect) {+void+testf(std::string input, std::string expect)+{std::string got;g_highlight.feed(&got, input);g_highlight.flush(&got);@@ -48,20 +54,28 @@ void testf(std::string input, std::string expect) {"\t input: %`'s\n""\texpected: %`'s\n""\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());+ input.c_str(),+ expect.c_str(),+ got.c_str());exit(1);}}-int main(int argc, char *argv[]) {+int+main(int argc, char* argv[])+{test("", "");test("vola", "");test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");- test("yo // hi\nthere ", "yo "s + HI_COMMENT + "// hi\n" + HI_RESET + "there ");+ test("yo // hi\nthere ",+ "yo "s + HI_COMMENT + "// hi\n" + HI_RESET + "there ");testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);- test("yo /* hi */ there ", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");- testf("yo /* hi */ there", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");- testf("yo /* h*i */ there", "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");+ test("yo /* hi */ there ",+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");+ testf("yo /* hi */ there",+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");+ testf("yo /* h*i */ there",+ "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");testf("vola", "vola");testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);testf("/hi/", "/hi/");diff --git a/llamafile/highlight_cmake.cpp b/llamafile/highlight/highlight_cmake.cppsimilarity index 96%rename from llamafile/highlight_cmake.cpprename to llamafile/highlight/highlight_cmake.cppindex ce34c5ad..1c2c5ff6 100644--- a/llamafile/highlight_cmake.cpp+++ b/llamafile/highlight/highlight_cmake.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,BACKSLASH,WORD,@@ -34,13 +35,17 @@ enum {DQUOTE_VAR,};-HighlightCmake::HighlightCmake() {+HighlightCmake::HighlightCmake()+{}-HighlightCmake::~HighlightCmake() {+HighlightCmake::~HighlightCmake()+{}-void HighlightCmake::feed(std::string *r, std::string_view input) {+void+HighlightCmake::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {wchar_t c;@@ -234,7 +239,9 @@ void HighlightCmake::feed(std::string *r, std::string_view input) {}}-void HighlightCmake::flush(std::string *r) {+void+HighlightCmake::flush(std::string* r)+{switch (t_) {case WORD:*r += word_;diff --git a/llamafile/highlight_cobol.cpp b/llamafile/highlight/highlight_cobol.cppsimilarity index 95%rename from llamafile/highlight_cobol.cpprename to llamafile/highlight/highlight_cobol.cppindex e57f84e2..df2f9727 100644--- a/llamafile/highlight_cobol.cpp+++ b/llamafile/highlight/highlight_cobol.cpp@@ -21,7 +21,8 @@// COmmon Business-Oriented Language-enum {+enum+{NORMAL,WORD,QUOTE,@@ -31,13 +32,17 @@ enum {COMMENT,};-HighlightCobol::HighlightCobol() {+HighlightCobol::HighlightCobol()+{}-HighlightCobol::~HighlightCobol() {+HighlightCobol::~HighlightCobol()+{}-void HighlightCobol::feed(std::string *r, std::string_view input) {+void+HighlightCobol::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -160,7 +165,9 @@ void HighlightCobol::feed(std::string *r, std::string_view input) {}}-void HighlightCobol::flush(std::string *r) {+void+HighlightCobol::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_cobol(word_.data(), word_.size())) {diff --git a/llamafile/highlight_create.gperf b/llamafile/highlight/highlight_create.gperfsimilarity index 100%rename from llamafile/highlight_create.gperfrename to llamafile/highlight/highlight_create.gperfdiff --git a/llamafile/highlight_csharp.cpp b/llamafile/highlight/highlight_csharp.cppsimilarity index 96%rename from llamafile/highlight_csharp.cpprename to llamafile/highlight/highlight_csharp.cppindex 2e84ca6c..53e15033 100644--- a/llamafile/highlight_csharp.cpp+++ b/llamafile/highlight/highlight_csharp.cpp@@ -36,7 +36,8 @@// As we can see above, you can use four, five, or more dquotes so you// can embed triple quoted strings inside.-enum {+enum+{NORMAL,WORD,QUOTE,@@ -54,13 +55,17 @@ enum {TRIPS_DQUOTE,};-HighlightCsharp::HighlightCsharp() {+HighlightCsharp::HighlightCsharp()+{}-HighlightCsharp::~HighlightCsharp() {+HighlightCsharp::~HighlightCsharp()+{}-void HighlightCsharp::feed(std::string *r, std::string_view input) {+void+HighlightCsharp::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -94,7 +99,8 @@ void HighlightCsharp::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_csharp_constant(word_.data(), word_.size())) {+ } else if (is_keyword_csharp_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -242,7 +248,9 @@ void HighlightCsharp::feed(std::string *r, std::string_view input) {}}-void HighlightCsharp::flush(std::string *r) {+void+HighlightCsharp::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_csharp(word_.data(), word_.size())) {diff --git a/llamafile/highlight_css.cpp b/llamafile/highlight/highlight_css.cppsimilarity index 97%rename from llamafile/highlight_css.cpprename to llamafile/highlight/highlight_css.cppindex a0f72813..5b45d08a 100644--- a/llamafile/highlight_css.cpp+++ b/llamafile/highlight/highlight_css.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,SELECTOR,PROPERTY,@@ -35,13 +36,17 @@ enum {BANG,};-HighlightCss::HighlightCss() {+HighlightCss::HighlightCss()+{}-HighlightCss::~HighlightCss() {+HighlightCss::~HighlightCss()+{}-void HighlightCss::feed(std::string *r, std::string_view input) {+void+HighlightCss::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -273,7 +278,9 @@ void HighlightCss::feed(std::string *r, std::string_view input) {}}-void HighlightCss::flush(std::string *r) {+void+HighlightCss::flush(std::string* r)+{while (t_) {switch (t_ & 255) {case AT:diff --git a/llamafile/highlight_d.cpp b/llamafile/highlight/highlight_d.cppsimilarity index 98%rename from llamafile/highlight_d.cpprename to llamafile/highlight/highlight_d.cppindex 0e038529..c9f89269 100644--- a/llamafile/highlight_d.cpp+++ b/llamafile/highlight/highlight_d.cpp@@ -16,14 +16,15 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>// D probably has the best lexical syntax documentation of any language.// https://dlang.org/spec/lex.html-enum {+enum+{NORMAL,WORD,QUOTE,@@ -52,7 +53,9 @@ enum {X_DQUOTE,};-static bool is_line_terminator(wchar_t c) {+static bool+is_line_terminator(wchar_t c)+{switch (c) {case '\r':case '\n':@@ -64,7 +67,9 @@ static bool is_line_terminator(wchar_t c) {}}-static int mirror(int c) {+static int+mirror(int c)+{switch (c) {case '(':return ')';@@ -79,13 +84,17 @@ static int mirror(int c) {}}-HighlightD::HighlightD() {+HighlightD::HighlightD()+{}-HighlightD::~HighlightD() {+HighlightD::~HighlightD()+{}-void HighlightD::feed(std::string *r, std::string_view input) {+void+HighlightD::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {wchar_t c;@@ -452,7 +461,9 @@ void HighlightD::feed(std::string *r, std::string_view input) {}}-void HighlightD::flush(std::string *r) {+void+HighlightD::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_d(word_.data(), word_.size())) {diff --git a/llamafile/highlight_forth.cpp b/llamafile/highlight/highlight_forth.cppsimilarity index 91%rename from llamafile/highlight_forth.cpprename to llamafile/highlight/highlight_forth.cppindex 83148298..f6be6a77 100644--- a/llamafile/highlight_forth.cpp+++ b/llamafile/highlight/highlight_forth.cpp@@ -19,18 +19,23 @@#include <ctype.h>-enum {+enum+{NORMAL,SYNTAX,};-HighlightForth::HighlightForth() {+HighlightForth::HighlightForth()+{}-HighlightForth::~HighlightForth() {+HighlightForth::~HighlightForth()+{}-void HighlightForth::feed(std::string *r, std::string_view input) {+void+HighlightForth::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -49,8 +54,10 @@ void HighlightForth::feed(std::string *r, std::string_view input) {*r += word_;t_ = SYNTAX;closer_ = '\n';- } else if (word_ == "(" || // inline comment, e.g. ( arg1 arg2 -- res1 )- word_ == ".(") { // printed comment, e.g. .( compiling... )+ } else if (word_ == "(" || // inline comment, e.g. ( arg1+ // arg2 -- res1 )+ word_ == ".(") { // printed comment, e.g. .(+ // compiling... )*r += HI_COMMENT;*r += word_;t_ = SYNTAX;@@ -98,7 +105,9 @@ void HighlightForth::feed(std::string *r, std::string_view input) {}}-void HighlightForth::flush(std::string *r) {+void+HighlightForth::flush(std::string* r)+{switch (t_) {case NORMAL:if (is_label_) {diff --git a/llamafile/highlight_fortran.cpp b/llamafile/highlight/highlight_fortran.cppsimilarity index 93%rename from llamafile/highlight_fortran.cpprename to llamafile/highlight/highlight_fortran.cppindex d7284b4f..e598c4c5 100644--- a/llamafile/highlight_fortran.cpp+++ b/llamafile/highlight/highlight_fortran.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -29,13 +30,17 @@ enum {COMMENT,};-HighlightFortran::HighlightFortran() {+HighlightFortran::HighlightFortran()+{}-HighlightFortran::~HighlightFortran() {+HighlightFortran::~HighlightFortran()+{}-void HighlightFortran::feed(std::string *r, std::string_view input) {+void+HighlightFortran::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -105,11 +110,13 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_fortran_type(word_.data(), word_.size())) {+ } else if (is_keyword_fortran_type(word_.data(),+ word_.size())) {*r += HI_TYPE;*r += word_;*r += HI_RESET;- } else if (is_keyword_fortran_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_fortran_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;@@ -166,7 +173,9 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {}}-void HighlightFortran::flush(std::string *r) {+void+HighlightFortran::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_fortran(word_.data(), word_.size())) {diff --git a/llamafile/highlight_go.cpp b/llamafile/highlight/highlight_go.cppsimilarity index 96%rename from llamafile/highlight_go.cpprename to llamafile/highlight/highlight_go.cppindex 0686edf2..66970de1 100644--- a/llamafile/highlight_go.cpp+++ b/llamafile/highlight/highlight_go.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -34,13 +35,17 @@ enum {TICK_BACKSLASH,};-HighlightGo::HighlightGo() {+HighlightGo::HighlightGo()+{}-HighlightGo::~HighlightGo() {+HighlightGo::~HighlightGo()+{}-void HighlightGo::feed(std::string *r, std::string_view input) {+void+HighlightGo::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -183,7 +188,9 @@ void HighlightGo::feed(std::string *r, std::string_view input) {}}-void HighlightGo::flush(std::string *r) {+void+HighlightGo::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_go(word_.data(), word_.size())) {diff --git a/llamafile/highlight_haskell.cpp b/llamafile/highlight/highlight_haskell.cppsimilarity index 97%rename from llamafile/highlight_haskell.cpprename to llamafile/highlight/highlight_haskell.cppindex e60b9edc..4a74be49 100644--- a/llamafile/highlight_haskell.cpp+++ b/llamafile/highlight/highlight_haskell.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,DQUOTE,@@ -38,13 +39,17 @@ enum {LT,};-HighlightHaskell::HighlightHaskell() {+HighlightHaskell::HighlightHaskell()+{}-HighlightHaskell::~HighlightHaskell() {+HighlightHaskell::~HighlightHaskell()+{}-void HighlightHaskell::feed(std::string *r, std::string_view input) {+void+HighlightHaskell::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -272,7 +277,9 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {}}-void HighlightHaskell::flush(std::string *r) {+void+HighlightHaskell::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_haskell(symbol_.data(), symbol_.size())) {diff --git a/llamafile/highlight_html.cpp b/llamafile/highlight/highlight_html.cppsimilarity index 96%rename from llamafile/highlight_html.cpprename to llamafile/highlight/highlight_html.cppindex 9bc979d0..b2b0ef38 100644--- a/llamafile/highlight_html.cpp+++ b/llamafile/highlight/highlight_html.cpp@@ -17,7 +17,8 @@#include "highlight.h"-enum {+enum+{NORMAL,TAG,TAG2,@@ -37,13 +38,17 @@ enum {ENTITY,};-HighlightHtml::HighlightHtml() {+HighlightHtml::HighlightHtml()+{}-HighlightHtml::~HighlightHtml() {+HighlightHtml::~HighlightHtml()+{}-void HighlightHtml::feed(std::string *r, std::string_view input) {+void+HighlightHtml::feed(std::string* r, std::string_view input)+{char c;for (size_t i = 0; i < input.size(); ++i) {c = input[i];@@ -83,7 +88,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {*r += '<';*r += HI_TAG;*r += c;- name_ += tolower(c);+ name_ += std::tolower(c);t_ = TAG2;}break;@@ -99,7 +104,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {*r += HI_ATTRIB;} else {*r += c;- name_ += tolower(c);+ name_ += std::tolower(c);}break;@@ -219,7 +224,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {break;case RELAY:- if (closer_[i_] == tolower(c)) {+ if (closer_[i_] == std::tolower(c)) {pending_ += c;if (++i_ == closer_.size()) {highlighter_->flush(r);@@ -294,7 +299,9 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {}}-void HighlightHtml::flush(std::string *r) {+void+HighlightHtml::flush(std::string* r)+{switch (t_) {case TAG:*r += '<';diff --git a/llamafile/highlight_java.cpp b/llamafile/highlight/highlight_java.cppsimilarity index 96%rename from llamafile/highlight_java.cpprename to llamafile/highlight/highlight_java.cppindex 7e436ae4..df537d30 100644--- a/llamafile/highlight_java.cpp+++ b/llamafile/highlight/highlight_java.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -40,13 +41,17 @@ enum {DQUOTE32, // """...""};-HighlightJava::HighlightJava() {+HighlightJava::HighlightJava()+{}-HighlightJava::~HighlightJava() {+HighlightJava::~HighlightJava()+{}-void HighlightJava::feed(std::string *r, std::string_view input) {+void+HighlightJava::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -82,7 +87,8 @@ void HighlightJava::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_java_constant(word_.data(), word_.size())) {+ } else if (is_keyword_java_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -255,7 +261,9 @@ void HighlightJava::feed(std::string *r, std::string_view input) {}}-void HighlightJava::flush(std::string *r) {+void+HighlightJava::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_java(word_.data(), word_.size())) {diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight/highlight_js.cppsimilarity index 97%rename from llamafile/highlight_js.cpprename to llamafile/highlight/highlight_js.cppindex fe2a0a41..ee45f32d 100644--- a/llamafile/highlight_js.cpp+++ b/llamafile/highlight/highlight_js.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -40,19 +41,24 @@ enum {REGEX_SQUARE_BACKSLASH,};-enum {+enum+{EXPECT_VALUE,EXPECT_OPERATOR,};-HighlightJs::HighlightJs() {+HighlightJs::HighlightJs()+{}-HighlightJs::~HighlightJs() {+HighlightJs::~HighlightJs()+{}// https://262.ecma-international.org/12.0/#sec-line-terminators-static bool is_line_terminator(wchar_t c) {+static bool+is_line_terminator(wchar_t c)+{switch (c) {case '\r':case '\n':@@ -64,7 +70,9 @@ static bool is_line_terminator(wchar_t c) {}}-void HighlightJs::feed(std::string *r, std::string_view input) {+void+HighlightJs::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -329,7 +337,9 @@ void HighlightJs::feed(std::string *r, std::string_view input) {}}-void HighlightJs::flush(std::string *r) {+void+HighlightJs::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_js(word_.data(), word_.size())) {diff --git a/llamafile/highlight_julia.cpp b/llamafile/highlight/highlight_julia.cppsimilarity index 96%rename from llamafile/highlight_julia.cpprename to llamafile/highlight/highlight_julia.cppindex 05384ab6..2a8263c8 100644--- a/llamafile/highlight_julia.cpp+++ b/llamafile/highlight/highlight_julia.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,COMMENT,@@ -38,13 +39,17 @@ enum {DQUOTE32,};-HighlightJulia::HighlightJulia() {+HighlightJulia::HighlightJulia()+{}-HighlightJulia::~HighlightJulia() {+HighlightJulia::~HighlightJulia()+{}-void HighlightJulia::feed(std::string *r, std::string_view input) {+void+HighlightJulia::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -219,7 +224,9 @@ void HighlightJulia::feed(std::string *r, std::string_view input) {}}-void HighlightJulia::flush(std::string *r) {+void+HighlightJulia::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_julia(word_.data(), word_.size())) {diff --git a/llamafile/highlight_kotlin.cpp b/llamafile/highlight/highlight_kotlin.cppsimilarity index 97%rename from llamafile/highlight_kotlin.cpprename to llamafile/highlight/highlight_kotlin.cppindex 9824befd..c55ac3d4 100644--- a/llamafile/highlight_kotlin.cpp+++ b/llamafile/highlight/highlight_kotlin.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -46,13 +47,17 @@ enum {DQUOTE32,};-HighlightKotlin::HighlightKotlin() {+HighlightKotlin::HighlightKotlin()+{}-HighlightKotlin::~HighlightKotlin() {+HighlightKotlin::~HighlightKotlin()+{}-void HighlightKotlin::feed(std::string *r, std::string_view input) {+void+HighlightKotlin::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -95,7 +100,8 @@ void HighlightKotlin::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_java_constant(word_.data(), word_.size())) {+ } else if (is_keyword_java_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -334,7 +340,9 @@ void HighlightKotlin::feed(std::string *r, std::string_view input) {}}-void HighlightKotlin::flush(std::string *r) {+void+HighlightKotlin::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_kotlin(word_.data(), word_.size())) {diff --git a/llamafile/highlight_ld.cpp b/llamafile/highlight/highlight_ld.cppsimilarity index 93%rename from llamafile/highlight_ld.cpprename to llamafile/highlight/highlight_ld.cppindex 93443b12..823a1283 100644--- a/llamafile/highlight_ld.cpp+++ b/llamafile/highlight/highlight_ld.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,DQUOTE,@@ -30,13 +31,17 @@ enum {SLASH_STAR_STAR,};-HighlightLd::HighlightLd() {+HighlightLd::HighlightLd()+{}-HighlightLd::~HighlightLd() {+HighlightLd::~HighlightLd()+{}-void HighlightLd::feed(std::string *r, std::string_view input) {+void+HighlightLd::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -70,7 +75,8 @@ void HighlightLd::feed(std::string *r, std::string_view input) {Word:case WORD:- if (!isascii(c) || isalnum(c) || c == '_' || c == '.' || c == '/' || c == ':') {+ if (!isascii(c) || isalnum(c) || c == '_' || c == '.' || c == '/' ||+ c == ':') {word_ += c;} else {if (is_cpp_) {@@ -78,7 +84,8 @@ void HighlightLd::feed(std::string *r, std::string_view input) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_c_constant(word_.data(), word_.size())) {+ } else if (is_keyword_c_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -90,11 +97,13 @@ void HighlightLd::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_ld_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_ld_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_ld_warning(word_.data(), word_.size())) {+ } else if (is_keyword_ld_warning(word_.data(),+ word_.size())) {*r += HI_WARNING;*r += word_;*r += HI_RESET;@@ -181,7 +190,9 @@ void HighlightLd::feed(std::string *r, std::string_view input) {}}-void HighlightLd::flush(std::string *r) {+void+HighlightLd::flush(std::string* r)+{switch (t_) {case WORD:if (is_cpp_) {diff --git a/llamafile/highlight_lisp.cpp b/llamafile/highlight/highlight_lisp.cppsimilarity index 93%rename from llamafile/highlight_lisp.cpprename to llamafile/highlight/highlight_lisp.cppindex c9ed8eea..db937069 100644--- a/llamafile/highlight_lisp.cpp+++ b/llamafile/highlight/highlight_lisp.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,SYMBOL,DQUOTE,@@ -27,13 +28,17 @@ enum {COMMENT,};-HighlightLisp::HighlightLisp() {+HighlightLisp::HighlightLisp()+{}-HighlightLisp::~HighlightLisp() {+HighlightLisp::~HighlightLisp()+{}-void HighlightLisp::feed(std::string *r, std::string_view input) {+void+HighlightLisp::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -82,7 +87,8 @@ void HighlightLisp::feed(std::string *r, std::string_view input) {c == '`' || //c == '"' || //c == '\'') {- if (is_first_ && is_keyword_lisp(symbol_.data(), symbol_.size())) {+ if (is_first_ &&+ is_keyword_lisp(symbol_.data(), symbol_.size())) {*r += HI_KEYWORD;*r += symbol_;*r += HI_RESET;@@ -131,7 +137,9 @@ void HighlightLisp::feed(std::string *r, std::string_view input) {}}-void HighlightLisp::flush(std::string *r) {+void+HighlightLisp::flush(std::string* r)+{switch (t_) {case SYMBOL:if (is_first_ && is_keyword_lisp(symbol_.data(), symbol_.size())) {diff --git a/llamafile/highlight_lua.cpp b/llamafile/highlight/highlight_lua.cppsimilarity index 96%rename from llamafile/highlight_lua.cpprename to llamafile/highlight/highlight_lua.cppindex 9e62e8d2..2759c94b 100644--- a/llamafile/highlight_lua.cpp+++ b/llamafile/highlight/highlight_lua.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -36,13 +37,17 @@ enum {LITERAL_RSB,};-HighlightLua::HighlightLua() {+HighlightLua::HighlightLua()+{}-HighlightLua::~HighlightLua() {+HighlightLua::~HighlightLua()+{}-void HighlightLua::feed(std::string *r, std::string_view input) {+void+HighlightLua::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -79,7 +84,8 @@ void HighlightLua::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_lua_constant(word_.data(), word_.size())) {+ } else if (is_keyword_lua_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -220,7 +226,9 @@ void HighlightLua::feed(std::string *r, std::string_view input) {}}-void HighlightLua::flush(std::string *r) {+void+HighlightLua::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_lua(word_.data(), word_.size())) {diff --git a/llamafile/highlight_m4.cpp b/llamafile/highlight/highlight_m4.cppsimilarity index 94%rename from llamafile/highlight_m4.cpprename to llamafile/highlight/highlight_m4.cppindex 7dbc1942..f4a4eb38 100644--- a/llamafile/highlight_m4.cpp+++ b/llamafile/highlight/highlight_m4.cpp@@ -19,20 +19,25 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,COMMENT,DOLLAR,};-HighlightM4::HighlightM4() {+HighlightM4::HighlightM4()+{}-HighlightM4::~HighlightM4() {+HighlightM4::~HighlightM4()+{}-void HighlightM4::feed(std::string *r, std::string_view input) {+void+HighlightM4::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -107,7 +112,9 @@ void HighlightM4::feed(std::string *r, std::string_view input) {}}-void HighlightM4::flush(std::string *r) {+void+HighlightM4::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_m4(word_.data(), word_.size())) {diff --git a/llamafile/highlight_make.cpp b/llamafile/highlight/highlight_make.cppsimilarity index 96%rename from llamafile/highlight_make.cpprename to llamafile/highlight/highlight_make.cppindex 0696d75d..b82db978 100644--- a/llamafile/highlight_make.cpp+++ b/llamafile/highlight/highlight_make.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,COMMENT,@@ -31,7 +32,9 @@ enum {BACKSLASH,};-static bool is_automatic_variable(int c) {+static bool+is_automatic_variable(int c)+{switch (c) {case '@':case '%':@@ -47,13 +50,17 @@ static bool is_automatic_variable(int c) {}}-HighlightMake::HighlightMake() {+HighlightMake::HighlightMake()+{}-HighlightMake::~HighlightMake() {+HighlightMake::~HighlightMake()+{}-void HighlightMake::feed(std::string *r, std::string_view input) {+void+HighlightMake::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -208,7 +215,9 @@ void HighlightMake::feed(std::string *r, std::string_view input) {}}-void HighlightMake::flush(std::string *r) {+void+HighlightMake::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_make(word_.data(), word_.size())) {diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight/highlight_markdown.cppsimilarity index 96%rename from llamafile/highlight_markdown.cpprename to llamafile/highlight/highlight_markdown.cppindex 33c819f4..1ea9aaec 100644--- a/llamafile/highlight_markdown.cpp+++ b/llamafile/highlight/highlight_markdown.cpp@@ -16,10 +16,11 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>-enum {+enum+{NORMAL,TICK,TICK_TICK,@@ -40,13 +41,17 @@ enum {EMPHASIS_BACKSLASH,};-HighlightMarkdown::HighlightMarkdown() {+HighlightMarkdown::HighlightMarkdown()+{}-HighlightMarkdown::~HighlightMarkdown() {+HighlightMarkdown::~HighlightMarkdown()+{}-void HighlightMarkdown::feed(std::string *r, std::string_view input) {+void+HighlightMarkdown::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -115,7 +120,8 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {t_ = NORMAL;} else {// handle *emphasized* text- // inverted because \e[3m has a poorly supported western bias+ // inverted because \e[3m has a poorly supported western+ // bias*r += '*';*r += HI_ITALIC;lf::append_wchar(r, c);@@ -245,7 +251,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case LANG:if (!isascii(c) || !isspace(c)) {lf::append_wchar(r, c);- lang_ += tolower(c);+ lang_ += std::tolower(c);} else {if (!(highlighter_ = Highlight::create(lang_)))highlighter_ = new HighlightTxt;@@ -304,7 +310,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {}}-void HighlightMarkdown::flush(std::string *r) {+void+HighlightMarkdown::flush(std::string* r)+{switch (t_) {case LANG:lang_.clear();diff --git a/llamafile/highlight_matlab.cpp b/llamafile/highlight/highlight_matlab.cppsimilarity index 92%rename from llamafile/highlight_matlab.cpprename to llamafile/highlight/highlight_matlab.cppindex e0ee4ad4..678fc4f4 100644--- a/llamafile/highlight_matlab.cpp+++ b/llamafile/highlight/highlight_matlab.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,COMMENT,@@ -29,13 +30,17 @@ enum {DQUOTE_BACKSLASH,};-HighlightMatlab::HighlightMatlab() {+HighlightMatlab::HighlightMatlab()+{}-HighlightMatlab::~HighlightMatlab() {+HighlightMatlab::~HighlightMatlab()+{}-void HighlightMatlab::feed(std::string *r, std::string_view input) {+void+HighlightMatlab::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -71,11 +76,13 @@ void HighlightMatlab::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_matlab_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_matlab_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_matlab_constant(word_.data(), word_.size())) {+ } else if (is_keyword_matlab_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -132,7 +139,9 @@ void HighlightMatlab::feed(std::string *r, std::string_view input) {}}-void HighlightMatlab::flush(std::string *r) {+void+HighlightMatlab::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_matlab(word_.data(), word_.size())) {diff --git a/llamafile/highlight_ocaml.cpp b/llamafile/highlight/highlight_ocaml.cppsimilarity index 94%rename from llamafile/highlight_ocaml.cpprename to llamafile/highlight/highlight_ocaml.cppindex 9fd9c194..a4831918 100644--- a/llamafile/highlight_ocaml.cpp+++ b/llamafile/highlight/highlight_ocaml.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -35,13 +36,17 @@ enum {RAWSTR_PIPE,};-HighlightOcaml::HighlightOcaml() {+HighlightOcaml::HighlightOcaml()+{}-HighlightOcaml::~HighlightOcaml() {+HighlightOcaml::~HighlightOcaml()+{}-void HighlightOcaml::feed(std::string *r, std::string_view input) {+void+HighlightOcaml::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -72,18 +77,21 @@ void HighlightOcaml::feed(std::string *r, std::string_view input) {Word:case WORD:- if (!isascii(c) || isalnum(c) || c == '_' || c == '\'' || c == '~') {+ if (!isascii(c) || isalnum(c) || c == '_' || c == '\'' ||+ c == '~') {word_ += c;} else {if (is_keyword_ocaml(word_.data(), word_.size())) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_ocaml_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_ocaml_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_ocaml_constant(word_.data(), word_.size())) {+ } else if (is_keyword_ocaml_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -227,7 +235,9 @@ void HighlightOcaml::feed(std::string *r, std::string_view input) {}}-void HighlightOcaml::flush(std::string *r) {+void+HighlightOcaml::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_ocaml(word_.data(), word_.size())) {diff --git a/llamafile/highlight_pascal.cpp b/llamafile/highlight/highlight_pascal.cppsimilarity index 95%rename from llamafile/highlight_pascal.cpprename to llamafile/highlight/highlight_pascal.cppindex eceb3535..e304aa5a 100644--- a/llamafile/highlight_pascal.cpp+++ b/llamafile/highlight/highlight_pascal.cpp@@ -28,7 +28,8 @@//// doing that requires extra keywords-enum {+enum+{NORMAL,WORD,QUOTE,@@ -41,13 +42,17 @@ enum {PAREN_STAR_STAR,};-HighlightPascal::HighlightPascal() {+HighlightPascal::HighlightPascal()+{}-HighlightPascal::~HighlightPascal() {+HighlightPascal::~HighlightPascal()+{}-void HighlightPascal::feed(std::string *r, std::string_view input) {+void+HighlightPascal::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -93,7 +98,8 @@ void HighlightPascal::feed(std::string *r, std::string_view input) {*r += HI_TYPE;*r += word_;*r += HI_RESET;- } else if (is_keyword_pascal_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_pascal_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;@@ -184,7 +190,9 @@ void HighlightPascal::feed(std::string *r, std::string_view input) {}}-void HighlightPascal::flush(std::string *r) {+void+HighlightPascal::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_pascal(word_.data(), word_.size())) {diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight/highlight_perl.cppsimilarity index 97%rename from llamafile/highlight_perl.cpprename to llamafile/highlight/highlight_perl.cppindex 4380b25c..53bf9771 100644--- a/llamafile/highlight_perl.cpp+++ b/llamafile/highlight/highlight_perl.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -48,12 +49,15 @@ enum {BACKSLASH,};-enum {+enum+{EXPECT_VALUE,EXPECT_OPERATOR,};-static int mirror(int c) {+static int+mirror(int c)+{switch (c) {case '(':return ')';@@ -68,7 +72,9 @@ static int mirror(int c) {}}-static bool is_magic_var(int c) {+static bool+is_magic_var(int c)+{switch (c) {case '!':case '"':@@ -92,7 +98,9 @@ static bool is_magic_var(int c) {}}-static bool is_regex_punct(int c) {+static bool+is_regex_punct(int c)+{switch (c) {case '!':case '"':@@ -123,7 +131,9 @@ static bool is_regex_punct(int c) {}}-static bool is_regex_prefix(const std::string_view &s) {+static bool+is_regex_prefix(const std::string_view& s)+{return s == "m" || //s == "s" || //s == "y" || //@@ -135,19 +145,25 @@ static bool is_regex_prefix(const std::string_view &s) {s == "qr";}-static bool is_double_regex(const std::string_view &s) {+static bool+is_double_regex(const std::string_view& s)+{return s == "s" || //s == "y" || //s == "tr";}-HighlightPerl::HighlightPerl() {+HighlightPerl::HighlightPerl()+{}-HighlightPerl::~HighlightPerl() {+HighlightPerl::~HighlightPerl()+{}-void HighlightPerl::feed(std::string *r, std::string_view input) {+void+HighlightPerl::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -521,7 +537,9 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {}}-void HighlightPerl::flush(std::string *r) {+void+HighlightPerl::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_perl(word_.data(), word_.size())) {diff --git a/llamafile/highlight_php.cpp b/llamafile/highlight/highlight_php.cppsimilarity index 95%rename from llamafile/highlight_php.cpprename to llamafile/highlight/highlight_php.cppindex d097786e..334856b7 100644--- a/llamafile/highlight_php.cpp+++ b/llamafile/highlight/highlight_php.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -35,13 +36,17 @@ enum {VAR,};-HighlightPhp::HighlightPhp() {+HighlightPhp::HighlightPhp()+{}-HighlightPhp::~HighlightPhp() {+HighlightPhp::~HighlightPhp()+{}-void HighlightPhp::feed(std::string *r, std::string_view input) {+void+HighlightPhp::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -83,7 +88,8 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_php_constant(word_.data(), word_.size())) {+ } else if (is_keyword_php_constant(word_.data(),+ word_.size())) {*r += HI_TYPE;*r += word_;*r += HI_RESET;@@ -197,7 +203,9 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {}}-void HighlightPhp::flush(std::string *r) {+void+HighlightPhp::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_php(word_.data(), word_.size())) {diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight/highlight_python.cppsimilarity index 95%rename from llamafile/highlight_python.cpprename to llamafile/highlight/highlight_python.cppindex 06f0f950..103d9692 100644--- a/llamafile/highlight_python.cpp+++ b/llamafile/highlight/highlight_python.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,COM,@@ -41,13 +42,17 @@ enum {DQUOTE32,};-HighlightPython::HighlightPython() {+HighlightPython::HighlightPython()+{}-HighlightPython::~HighlightPython() {+HighlightPython::~HighlightPython()+{}-void HighlightPython::feed(std::string *r, std::string_view input) {+void+HighlightPython::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -84,11 +89,13 @@ void HighlightPython::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_python_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_python_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_python_constant(word_.data(), word_.size())) {+ } else if (is_keyword_python_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -265,7 +272,9 @@ void HighlightPython::feed(std::string *r, std::string_view input) {}}-void HighlightPython::flush(std::string *r) {+void+HighlightPython::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_python(word_.data(), word_.size())) {diff --git a/llamafile/highlight_python_test.cpp b/llamafile/highlight/highlight_python_test.cppsimilarity index 77%rename from llamafile/highlight_python_test.cpprename to llamafile/highlight/highlight_python_test.cppindex 0c5f0787..657a5374 100644--- a/llamafile/highlight_python_test.cpp+++ b/llamafile/highlight/highlight_python_test.cpp@@ -24,7 +24,9 @@ using namespace std::string_literals;HighlightPython g_highlight;-void test(std::string input, std::string expect) {+void+test(std::string input, std::string expect)+{std::string got;g_highlight.feed(&got, input);if (got != expect) {@@ -33,12 +35,16 @@ void test(std::string input, std::string expect) {"\t input: %`'s\n""\texpected: %`'s\n""\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());+ input.c_str(),+ expect.c_str(),+ got.c_str());exit(1);}}-void testf(std::string input, std::string expect) {+void+testf(std::string input, std::string expect)+{std::string got;g_highlight.feed(&got, input);g_highlight.flush(&got);@@ -48,16 +54,21 @@ void testf(std::string input, std::string expect) {"\t input: %`'s\n""\texpected: %`'s\n""\t got: %`'s\n",- input.c_str(), expect.c_str(), got.c_str());+ input.c_str(),+ expect.c_str(),+ got.c_str());exit(1);}}-int main(int argc, char *argv[]) {+int+main(int argc, char* argv[])+{test("", "");test("d", "");test("ef ", ""s + HI_KEYWORD + "def" + HI_RESET + " ");- test("yo # hi\nthere ", "yo "s + HI_COMMENT + "# hi\n" + HI_RESET + "there ");+ test("yo # hi\nthere ",+ "yo "s + HI_COMMENT + "# hi\n" + HI_RESET + "there ");testf("# sup", ""s + HI_COMMENT + "# sup" + HI_RESET);testf("de", "de");testf("def", ""s + HI_KEYWORD + "def" + HI_RESET);@@ -66,10 +77,13 @@ int main(int argc, char *argv[]) {testf("x '''''' y", "x "s + HI_STRING + "''''''" + HI_RESET + " y");testf("x '''xxx''' y", "x "s + HI_STRING + "'''xxx'''" + HI_RESET + " y");testf("x '''xx'x''' y", "x "s + HI_STRING + "'''xx'x'''" + HI_RESET + " y");- testf("x '''xx''x''' y", "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");+ testf("x '''xx''x''' y",+ "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");testf("x \"\" y", "x "s + HI_STRING + "\"\"" + HI_RESET + " y");- testf("x \"\"\"\"\"\" y", "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");- testf("x \"\"\"xxx\"\"\" y", "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");+ testf("x \"\"\"\"\"\" y",+ "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");+ testf("x \"\"\"xxx\"\"\" y",+ "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");diff --git a/llamafile/highlight_r.cpp b/llamafile/highlight/highlight_r.cppsimilarity index 97%rename from llamafile/highlight_r.cpprename to llamafile/highlight/highlight_r.cppindex 0b7c5a09..bf75dc81 100644--- a/llamafile/highlight_r.cpp+++ b/llamafile/highlight/highlight_r.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,COMMENT,@@ -34,13 +35,17 @@ enum {COLON,};-HighlightR::HighlightR() {+HighlightR::HighlightR()+{}-HighlightR::~HighlightR() {+HighlightR::~HighlightR()+{}-void HighlightR::feed(std::string *r, std::string_view input) {+void+HighlightR::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -214,7 +219,9 @@ void HighlightR::feed(std::string *r, std::string_view input) {}}-void HighlightR::flush(std::string *r) {+void+HighlightR::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_r(word_.data(), word_.size())) {diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight/highlight_ruby.cppsimilarity index 97%rename from llamafile/highlight_ruby.cpprename to llamafile/highlight/highlight_ruby.cppindex 928c1206..7500ebed 100644--- a/llamafile/highlight_ruby.cpp+++ b/llamafile/highlight/highlight_ruby.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,NUMBER,WORD,@@ -65,13 +66,16 @@ enum {QUESTION_BACKSLASH,};-enum {+enum+{EXPECT_EXPR,EXPECT_VALUE,EXPECT_OPERATOR,};-static int mirror(int c) {+static int+mirror(int c)+{switch (c) {case '(':return ')';@@ -86,7 +90,9 @@ static int mirror(int c) {}}-static bool ispunct_overridable(int c) {+static bool+ispunct_overridable(int c)+{switch (c) {case '%':case '&':@@ -107,7 +113,9 @@ static bool ispunct_overridable(int c) {}}-static bool is_dollar_one(int c) {+static bool+is_dollar_one(int c)+{switch (c) {case '!':case '"':@@ -141,7 +149,9 @@ static bool is_dollar_one(int c) {}}-static bool is_percent_literal(int c) {+static bool+is_percent_literal(int c)+{switch (c) {case 'q':case 'Q':@@ -158,7 +168,9 @@ static bool is_percent_literal(int c) {}}-static bool isident(int c) {+static bool+isident(int c)+{return !isascii(c) || //isalnum(c) || //c == '!' || //@@ -174,13 +186,17 @@ static bool isident(int c) {c == '_';}-HighlightRuby::HighlightRuby() {+HighlightRuby::HighlightRuby()+{}-HighlightRuby::~HighlightRuby() {+HighlightRuby::~HighlightRuby()+{}-void HighlightRuby::feed(std::string *r, std::string_view input) {+void+HighlightRuby::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {wchar_t c;@@ -258,10 +274,12 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {*r += HI_COMMENT;lf::append_wchar(r, c);t_ = COMMENT;- } else if (c == '<' && (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {+ } else if (c == '<' &&+ (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {lf::append_wchar(r, c);t_ = LT;- } else if (c == '/' && (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {+ } else if (c == '/' &&+ (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {t_ = REGEX;*r += HI_STRING;lf::append_wchar(r, c);@@ -365,12 +383,14 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {expect_ = EXPECT_OPERATOR;}} else if (expect_ == EXPECT_EXPR &&- is_keyword_ruby_builtin(word_.data(), word_.size())) {+ is_keyword_ruby_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;expect_ = EXPECT_VALUE;- } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {+ } else if (is_keyword_ruby_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -909,7 +929,9 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {}}-void HighlightRuby::flush(std::string *r) {+void+HighlightRuby::flush(std::string* r)+{switch (t_) {case EQUAL_WORD:*r += '=';diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight/highlight_rust.cppsimilarity index 94%rename from llamafile/highlight_rust.cpprename to llamafile/highlight/highlight_rust.cppindex 20bd8076..2763e111 100644--- a/llamafile/highlight_rust.cpp+++ b/llamafile/highlight/highlight_rust.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -36,13 +37,17 @@ enum {ATTRIB,};-HighlightRust::HighlightRust() {+HighlightRust::HighlightRust()+{}-HighlightRust::~HighlightRust() {+HighlightRust::~HighlightRust()+{}-void HighlightRust::feed(std::string *r, std::string_view input) {+void+HighlightRust::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -79,7 +84,8 @@ void HighlightRust::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (word_.size() >= 2 && word_[word_.size() - 1] == '!') {+ } else if (word_.size() >= 2 &&+ word_[word_.size() - 1] == '!') {*r += HI_MACRO;*r += word_;*r += HI_RESET;@@ -87,7 +93,8 @@ void HighlightRust::feed(std::string *r, std::string_view input) {*r += HI_TYPE;*r += word_;*r += HI_RESET;- } else if (is_keyword_rust_constant(word_.data(), word_.size())) {+ } else if (is_keyword_rust_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -230,7 +237,9 @@ void HighlightRust::feed(std::string *r, std::string_view input) {}}-void HighlightRust::flush(std::string *r) {+void+HighlightRust::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_rust(word_.data(), word_.size())) {diff --git a/llamafile/highlight_scala.cpp b/llamafile/highlight/highlight_scala.cppsimilarity index 97%rename from llamafile/highlight_scala.cpprename to llamafile/highlight/highlight_scala.cppindex 226de140..142e56ba 100644--- a/llamafile/highlight_scala.cpp+++ b/llamafile/highlight/highlight_scala.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -46,13 +47,17 @@ enum {DQUOTE32, // """...""};-HighlightScala::HighlightScala() {+HighlightScala::HighlightScala()+{}-HighlightScala::~HighlightScala() {+HighlightScala::~HighlightScala()+{}-void HighlightScala::feed(std::string *r, std::string_view input) {+void+HighlightScala::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -95,7 +100,8 @@ void HighlightScala::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_java_constant(word_.data(), word_.size())) {+ } else if (is_keyword_java_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -334,7 +340,9 @@ void HighlightScala::feed(std::string *r, std::string_view input) {}}-void HighlightScala::flush(std::string *r) {+void+HighlightScala::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_scala(word_.data(), word_.size())) {diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight/highlight_shell.cppsimilarity index 98%rename from llamafile/highlight_shell.cpprename to llamafile/highlight/highlight_shell.cppindex 31bff95f..20669e8c 100644--- a/llamafile/highlight_shell.cpp+++ b/llamafile/highlight/highlight_shell.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -50,13 +51,17 @@ enum {BACKSLASH,};-HighlightShell::HighlightShell() {+HighlightShell::HighlightShell()+{}-HighlightShell::~HighlightShell() {+HighlightShell::~HighlightShell()+{}-void HighlightShell::feed(std::string *r, std::string_view input) {+void+HighlightShell::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -139,7 +144,8 @@ void HighlightShell::feed(std::string *r, std::string_view input) {*r += HI_KEYWORD;*r += word_;*r += HI_RESET;- } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_shell_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;@@ -507,7 +513,9 @@ void HighlightShell::feed(std::string *r, std::string_view input) {}}-void HighlightShell::flush(std::string *r) {+void+HighlightShell::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_shell(word_.data(), word_.size())) {diff --git a/llamafile/highlight_sql.cpp b/llamafile/highlight/highlight_sql.cppsimilarity index 96%rename from llamafile/highlight_sql.cpprename to llamafile/highlight/highlight_sql.cppindex 8621b1cd..01ca9ca6 100644--- a/llamafile/highlight_sql.cpp+++ b/llamafile/highlight/highlight_sql.cpp@@ -19,7 +19,8 @@#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -33,13 +34,17 @@ enum {SLASH_STAR_STAR,};-HighlightSql::HighlightSql() {+HighlightSql::HighlightSql()+{}-HighlightSql::~HighlightSql() {+HighlightSql::~HighlightSql()+{}-void HighlightSql::feed(std::string *r, std::string_view input) {+void+HighlightSql::feed(std::string* r, std::string_view input)+{int c;for (size_t i = 0; i < input.size(); ++i) {c = input[i] & 255;@@ -173,7 +178,9 @@ void HighlightSql::feed(std::string *r, std::string_view input) {}}-void HighlightSql::flush(std::string *r) {+void+HighlightSql::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_sql(word_.data(), word_.size())) {diff --git a/llamafile/highlight_swift.cpp b/llamafile/highlight/highlight_swift.cppsimilarity index 97%rename from llamafile/highlight_swift.cpprename to llamafile/highlight/highlight_swift.cppindex 8af3d2d1..73b31a69 100644--- a/llamafile/highlight_swift.cpp+++ b/llamafile/highlight/highlight_swift.cpp@@ -16,7 +16,7 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>@@ -34,7 +34,8 @@// ##"""hi \##(2 * (2 + 2)) there"""##//-enum {+enum+{NORMAL,WORD,SLASH,@@ -58,18 +59,23 @@ enum {REGEX_BACKSLASH,};-enum {+enum+{EXPECT_VALUE,EXPECT_OPERATOR,};-HighlightSwift::HighlightSwift() {+HighlightSwift::HighlightSwift()+{}-HighlightSwift::~HighlightSwift() {+HighlightSwift::~HighlightSwift()+{}-void HighlightSwift::feed(std::string *r, std::string_view input) {+void+HighlightSwift::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -148,11 +154,13 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {*r += HI_TYPE;*r += word_;*r += HI_RESET;- } else if (is_keyword_swift_builtin(word_.data(), word_.size())) {+ } else if (is_keyword_swift_builtin(word_.data(),+ word_.size())) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_swift_constant(word_.data(), word_.size())) {+ } else if (is_keyword_swift_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -415,7 +423,9 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {}}-void HighlightSwift::flush(std::string *r) {+void+HighlightSwift::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_swift(word_.data(), word_.size())) {diff --git a/llamafile/highlight_tcl.cpp b/llamafile/highlight/highlight_tcl.cppsimilarity index 96%rename from llamafile/highlight_tcl.cpprename to llamafile/highlight/highlight_tcl.cppindex 1da1f9e0..beecff7f 100644--- a/llamafile/highlight_tcl.cpp+++ b/llamafile/highlight/highlight_tcl.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,DQUOTE,@@ -33,13 +34,17 @@ enum {BACKSLASH,};-HighlightTcl::HighlightTcl() {+HighlightTcl::HighlightTcl()+{}-HighlightTcl::~HighlightTcl() {+HighlightTcl::~HighlightTcl()+{}-void HighlightTcl::feed(std::string *r, std::string_view input) {+void+HighlightTcl::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -187,7 +192,9 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {}}-void HighlightTcl::flush(std::string *r) {+void+HighlightTcl::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_tcl(word_.data(), word_.size())) {diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight/highlight_test.cppsimilarity index 80%rename from llamafile/highlight_test.cpprename to llamafile/highlight/highlight_test.cppindex 7100585b..078e4ed7 100644--- a/llamafile/highlight_test.cpp+++ b/llamafile/highlight/highlight_test.cpp@@ -16,7 +16,6 @@// limitations under the License.#include "highlight.h"-#include "macros.h"#include <ctype.h>#include <stdio.h>@@ -27,7 +26,7 @@#define ITERATIONS 200000#define CHARSET " raQq123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"-const char *const kLanguages[] = {+const char* const kLanguages[] = {"ada", //"asm", //"basic", //@@ -75,7 +74,9 @@ const char *const kLanguages[] = {"zig", //};-std::string generate_random_string(int n) {+std::string+generate_random_string(int n)+{std::string s;s.reserve(n);for (int i = 0; i < n; ++i)@@ -83,7 +84,9 @@ std::string generate_random_string(int n) {return s;}-std::string remove_ansi_sgr_codes(const std::string &input) {+std::string+remove_ansi_sgr_codes(const std::string& input)+{std::string result;result.reserve(input.length());bool in_escape_sequence = false;@@ -100,7 +103,9 @@ std::string remove_ansi_sgr_codes(const std::string &input) {return result;}-bool is_color_reset(const std::string &input) {+bool+is_color_reset(const std::string& input)+{int t = 0;int number = 0;bool has_color = false;@@ -141,9 +146,11 @@ bool is_color_reset(const std::string &input) {return !has_color;}-int main(int argc, char *argv[]) {- for (int l = 0; l < ARRAYLEN(kLanguages); ++l) {- Highlight *h = Highlight::create(kLanguages[l]);+int+main(int argc, char* argv[])+{+ for (int l = 0; l < sizeof(kLanguages) / sizeof(*kLanguages); ++l) {+ Highlight* h = Highlight::create(kLanguages[l]);if (!h) {fprintf(stderr, "Highlight::create(%`'s) failed\n", kLanguages[l]);exit(1);@@ -154,14 +161,22 @@ int main(int argc, char *argv[]) {h->feed(&colorized, sauce);h->flush(&colorized);if (!is_color_reset(colorized)) {- fprintf(stderr, "%s highlight failed to reset color: %`'s -> %`'s\n", kLanguages[l],- sauce.c_str(), colorized.c_str());+ fprintf(stderr,+ "%s highlight failed to reset color: %`'s -> %`'s\n",+ kLanguages[l],+ sauce.c_str(),+ colorized.c_str());exit(1);}std::string plain = remove_ansi_sgr_codes(colorized);if (sauce != plain) {- fprintf(stderr, "%s highlight failed to preserve code: %`'s -> %`'s -> %`'s\n",- kLanguages[l], sauce.c_str(), colorized.c_str(), plain.c_str());+ fprintf(stderr,+ "%s highlight failed to preserve code: %`'s -> %`'s -> "+ "%`'s\n",+ kLanguages[l],+ sauce.c_str(),+ colorized.c_str(),+ plain.c_str());exit(1);}}diff --git a/llamafile/highlight_tex.cpp b/llamafile/highlight/highlight_tex.cppsimilarity index 96%rename from llamafile/highlight_tex.cpprename to llamafile/highlight/highlight_tex.cppindex 55e6ae38..c41b2c22 100644--- a/llamafile/highlight_tex.cpp+++ b/llamafile/highlight/highlight_tex.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,BACKSLASH,COMMAND,@@ -33,13 +34,17 @@ enum {STRING_QUOTE,};-HighlightTex::HighlightTex() {+HighlightTex::HighlightTex()+{}-HighlightTex::~HighlightTex() {+HighlightTex::~HighlightTex()+{}-void HighlightTex::feed(std::string *r, std::string_view input) {+void+HighlightTex::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -190,7 +195,9 @@ void HighlightTex::feed(std::string *r, std::string_view input) {}}-void HighlightTex::flush(std::string *r) {+void+HighlightTex::flush(std::string* r)+{switch (t_) {case BACKTICK:*r += '`';diff --git a/llamafile/highlight_txt.cpp b/llamafile/highlight/highlight_txt.cppsimilarity index 83%rename from llamafile/highlight_txt.cpprename to llamafile/highlight/highlight_txt.cppindex 18d5e390..10e85a7f 100644--- a/llamafile/highlight_txt.cpp+++ b/llamafile/highlight/highlight_txt.cpp@@ -17,16 +17,22 @@#include "highlight.h"-HighlightTxt::HighlightTxt() {+HighlightTxt::HighlightTxt()+{}-HighlightTxt::~HighlightTxt() {+HighlightTxt::~HighlightTxt()+{}-void HighlightTxt::feed(std::string *r, std::string_view input) {+void+HighlightTxt::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i)*r += input[i] & 255;}-void HighlightTxt::flush(std::string *r) {+void+HighlightTxt::flush(std::string* r)+{}diff --git a/llamafile/highlight_typescript.cpp b/llamafile/highlight/highlight_typescript.cppsimilarity index 96%rename from llamafile/highlight_typescript.cpprename to llamafile/highlight/highlight_typescript.cppindex a6214306..9db3b35c 100644--- a/llamafile/highlight_typescript.cpp+++ b/llamafile/highlight/highlight_typescript.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -40,18 +41,23 @@ enum {REGEX_SQUARE_BACKSLASH,};-enum {+enum+{EXPECT_VALUE,EXPECT_OPERATOR,};-HighlightTypescript::HighlightTypescript() {+HighlightTypescript::HighlightTypescript()+{}-HighlightTypescript::~HighlightTypescript() {+HighlightTypescript::~HighlightTypescript()+{}-void HighlightTypescript::feed(std::string *r, std::string_view input) {+void+HighlightTypescript::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -129,7 +135,8 @@ void HighlightTypescript::feed(std::string *r, std::string_view input) {*r += word_;*r += HI_RESET;expect_ = EXPECT_VALUE;- } else if (is_keyword_typescript_type(word_.data(), word_.size())) {+ } else if (is_keyword_typescript_type(word_.data(),+ word_.size())) {*r += HI_TYPE;*r += word_;*r += HI_RESET;@@ -316,7 +323,9 @@ void HighlightTypescript::feed(std::string *r, std::string_view input) {}}-void HighlightTypescript::flush(std::string *r) {+void+HighlightTypescript::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_typescript(word_.data(), word_.size())) {diff --git a/llamafile/highlight_zig.cpp b/llamafile/highlight/highlight_zig.cppsimilarity index 95%rename from llamafile/highlight_zig.cpprename to llamafile/highlight/highlight_zig.cppindex c251870a..4f2ef9ca 100644--- a/llamafile/highlight_zig.cpp+++ b/llamafile/highlight/highlight_zig.cpp@@ -16,11 +16,12 @@// limitations under the License.#include "highlight.h"-#include "string.h"+#include "util.h"#include <cosmo.h>#include <ctype.h>-enum {+enum+{NORMAL,WORD,QUOTE,@@ -33,13 +34,17 @@ enum {BACKSLASH_BACKSLASH,};-HighlightZig::HighlightZig() {+HighlightZig::HighlightZig()+{}-HighlightZig::~HighlightZig() {+HighlightZig::~HighlightZig()+{}-void HighlightZig::feed(std::string *r, std::string_view input) {+void+HighlightZig::feed(std::string* r, std::string_view input)+{for (size_t i = 0; i < input.size(); ++i) {wchar_t c;int b = input[i] & 255;@@ -101,7 +106,8 @@ void HighlightZig::feed(std::string *r, std::string_view input) {*r += HI_BUILTIN;*r += word_;*r += HI_RESET;- } else if (is_keyword_zig_constant(word_.data(), word_.size())) {+ } else if (is_keyword_zig_constant(word_.data(),+ word_.size())) {*r += HI_CONSTANT;*r += word_;*r += HI_RESET;@@ -183,7 +189,9 @@ void HighlightZig::feed(std::string *r, std::string_view input) {}}-void HighlightZig::flush(std::string *r) {+void+HighlightZig::flush(std::string* r)+{switch (t_) {case WORD:if (is_keyword_zig(word_.data(), word_.size())) {diff --git a/llamafile/is_keyword_ada.gperf b/llamafile/highlight/is_keyword_ada.gperfsimilarity index 100%rename from llamafile/is_keyword_ada.gperfrename to llamafile/highlight/is_keyword_ada.gperfdiff --git a/llamafile/is_keyword_ada_constant.gperf b/llamafile/highlight/is_keyword_ada_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_ada_constant.gperfrename to llamafile/highlight/is_keyword_ada_constant.gperfdiff --git a/llamafile/is_keyword_asm_prefix.gperf b/llamafile/highlight/is_keyword_asm_prefix.gperfsimilarity index 100%rename from llamafile/is_keyword_asm_prefix.gperfrename to llamafile/highlight/is_keyword_asm_prefix.gperfdiff --git a/llamafile/is_keyword_asm_qualifier.gperf b/llamafile/highlight/is_keyword_asm_qualifier.gperfsimilarity index 100%rename from llamafile/is_keyword_asm_qualifier.gperfrename to llamafile/highlight/is_keyword_asm_qualifier.gperfdiff --git a/llamafile/is_keyword_basic.gperf b/llamafile/highlight/is_keyword_basic.gperfsimilarity index 100%rename from llamafile/is_keyword_basic.gperfrename to llamafile/highlight/is_keyword_basic.gperfdiff --git a/llamafile/is_keyword_basic_builtin.gperf b/llamafile/highlight/is_keyword_basic_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_basic_builtin.gperfrename to llamafile/highlight/is_keyword_basic_builtin.gperfdiff --git a/llamafile/is_keyword_basic_constant.gperf b/llamafile/highlight/is_keyword_basic_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_basic_constant.gperfrename to llamafile/highlight/is_keyword_basic_constant.gperfdiff --git a/llamafile/is_keyword_basic_type.gperf b/llamafile/highlight/is_keyword_basic_type.gperfsimilarity index 100%rename from llamafile/is_keyword_basic_type.gperfrename to llamafile/highlight/is_keyword_basic_type.gperfdiff --git a/llamafile/is_keyword_c.gperf b/llamafile/highlight/is_keyword_c.gperfsimilarity index 100%rename from llamafile/is_keyword_c.gperfrename to llamafile/highlight/is_keyword_c.gperfdiff --git a/llamafile/is_keyword_c_builtin.gperf b/llamafile/highlight/is_keyword_c_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_c_builtin.gperfrename to llamafile/highlight/is_keyword_c_builtin.gperfdiff --git a/llamafile/is_keyword_c_constant.gperf b/llamafile/highlight/is_keyword_c_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_c_constant.gperfrename to llamafile/highlight/is_keyword_c_constant.gperfdiff --git a/llamafile/is_keyword_c_pod.gperf b/llamafile/highlight/is_keyword_c_pod.gperfsimilarity index 100%rename from llamafile/is_keyword_c_pod.gperfrename to llamafile/highlight/is_keyword_c_pod.gperfdiff --git a/llamafile/is_keyword_c_type.gperf b/llamafile/highlight/is_keyword_c_type.gperfsimilarity index 100%rename from llamafile/is_keyword_c_type.gperfrename to llamafile/highlight/is_keyword_c_type.gperfdiff --git a/llamafile/is_keyword_cmake.gperf b/llamafile/highlight/is_keyword_cmake.gperfsimilarity index 100%rename from llamafile/is_keyword_cmake.gperfrename to llamafile/highlight/is_keyword_cmake.gperfdiff --git a/llamafile/is_keyword_cobol.gperf b/llamafile/highlight/is_keyword_cobol.gperfsimilarity index 100%rename from llamafile/is_keyword_cobol.gperfrename to llamafile/highlight/is_keyword_cobol.gperfdiff --git a/llamafile/is_keyword_cpp.gperf b/llamafile/highlight/is_keyword_cpp.gperfsimilarity index 100%rename from llamafile/is_keyword_cpp.gperfrename to llamafile/highlight/is_keyword_cpp.gperfdiff --git a/llamafile/is_keyword_csharp.gperf b/llamafile/highlight/is_keyword_csharp.gperfsimilarity index 100%rename from llamafile/is_keyword_csharp.gperfrename to llamafile/highlight/is_keyword_csharp.gperfdiff --git a/llamafile/is_keyword_csharp_constant.gperf b/llamafile/highlight/is_keyword_csharp_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_csharp_constant.gperfrename to llamafile/highlight/is_keyword_csharp_constant.gperfdiff --git a/llamafile/is_keyword_css_at.gperf b/llamafile/highlight/is_keyword_css_at.gperfsimilarity index 100%rename from llamafile/is_keyword_css_at.gperfrename to llamafile/highlight/is_keyword_css_at.gperfdiff --git a/llamafile/is_keyword_css_bang.gperf b/llamafile/highlight/is_keyword_css_bang.gperfsimilarity index 100%rename from llamafile/is_keyword_css_bang.gperfrename to llamafile/highlight/is_keyword_css_bang.gperfdiff --git a/llamafile/is_keyword_cxx.gperf b/llamafile/highlight/is_keyword_cxx.gperfsimilarity index 100%rename from llamafile/is_keyword_cxx.gperfrename to llamafile/highlight/is_keyword_cxx.gperfdiff --git a/llamafile/is_keyword_d.gperf b/llamafile/highlight/is_keyword_d.gperfsimilarity index 100%rename from llamafile/is_keyword_d.gperfrename to llamafile/highlight/is_keyword_d.gperfdiff --git a/llamafile/is_keyword_d_constant.gperf b/llamafile/highlight/is_keyword_d_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_d_constant.gperfrename to llamafile/highlight/is_keyword_d_constant.gperfdiff --git a/llamafile/is_keyword_forth.gperf b/llamafile/highlight/is_keyword_forth.gperfsimilarity index 100%rename from llamafile/is_keyword_forth.gperfrename to llamafile/highlight/is_keyword_forth.gperfdiff --git a/llamafile/is_keyword_forth_def.gperf b/llamafile/highlight/is_keyword_forth_def.gperfsimilarity index 100%rename from llamafile/is_keyword_forth_def.gperfrename to llamafile/highlight/is_keyword_forth_def.gperfdiff --git a/llamafile/is_keyword_fortran.gperf b/llamafile/highlight/is_keyword_fortran.gperfsimilarity index 100%rename from llamafile/is_keyword_fortran.gperfrename to llamafile/highlight/is_keyword_fortran.gperfdiff --git a/llamafile/is_keyword_fortran_builtin.gperf b/llamafile/highlight/is_keyword_fortran_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_fortran_builtin.gperfrename to llamafile/highlight/is_keyword_fortran_builtin.gperfdiff --git a/llamafile/is_keyword_fortran_type.gperf b/llamafile/highlight/is_keyword_fortran_type.gperfsimilarity index 100%rename from llamafile/is_keyword_fortran_type.gperfrename to llamafile/highlight/is_keyword_fortran_type.gperfdiff --git a/llamafile/is_keyword_go.gperf b/llamafile/highlight/is_keyword_go.gperfsimilarity index 100%rename from llamafile/is_keyword_go.gperfrename to llamafile/highlight/is_keyword_go.gperfdiff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/highlight/is_keyword_go_type.gperfsimilarity index 100%rename from llamafile/is_keyword_go_type.gperfrename to llamafile/highlight/is_keyword_go_type.gperfdiff --git a/llamafile/is_keyword_haskell.gperf b/llamafile/highlight/is_keyword_haskell.gperfsimilarity index 100%rename from llamafile/is_keyword_haskell.gperfrename to llamafile/highlight/is_keyword_haskell.gperfdiff --git a/llamafile/is_keyword_java.gperf b/llamafile/highlight/is_keyword_java.gperfsimilarity index 100%rename from llamafile/is_keyword_java.gperfrename to llamafile/highlight/is_keyword_java.gperfdiff --git a/llamafile/is_keyword_java_constant.gperf b/llamafile/highlight/is_keyword_java_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_java_constant.gperfrename to llamafile/highlight/is_keyword_java_constant.gperfdiff --git a/llamafile/is_keyword_js.gperf b/llamafile/highlight/is_keyword_js.gperfsimilarity index 100%rename from llamafile/is_keyword_js.gperfrename to llamafile/highlight/is_keyword_js.gperfdiff --git a/llamafile/is_keyword_js_builtin.gperf b/llamafile/highlight/is_keyword_js_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_js_builtin.gperfrename to llamafile/highlight/is_keyword_js_builtin.gperfdiff --git a/llamafile/is_keyword_js_constant.gperf b/llamafile/highlight/is_keyword_js_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_js_constant.gperfrename to llamafile/highlight/is_keyword_js_constant.gperfdiff --git a/llamafile/is_keyword_julia.gperf b/llamafile/highlight/is_keyword_julia.gperfsimilarity index 100%rename from llamafile/is_keyword_julia.gperfrename to llamafile/highlight/is_keyword_julia.gperfdiff --git a/llamafile/is_keyword_kotlin.gperf b/llamafile/highlight/is_keyword_kotlin.gperfsimilarity index 100%rename from llamafile/is_keyword_kotlin.gperfrename to llamafile/highlight/is_keyword_kotlin.gperfdiff --git a/llamafile/is_keyword_ld.gperf b/llamafile/highlight/is_keyword_ld.gperfsimilarity index 100%rename from llamafile/is_keyword_ld.gperfrename to llamafile/highlight/is_keyword_ld.gperfdiff --git a/llamafile/is_keyword_ld_builtin.gperf b/llamafile/highlight/is_keyword_ld_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_ld_builtin.gperfrename to llamafile/highlight/is_keyword_ld_builtin.gperfdiff --git a/llamafile/is_keyword_ld_warning.gperf b/llamafile/highlight/is_keyword_ld_warning.gperfsimilarity index 100%rename from llamafile/is_keyword_ld_warning.gperfrename to llamafile/highlight/is_keyword_ld_warning.gperfdiff --git a/llamafile/is_keyword_lisp.gperf b/llamafile/highlight/is_keyword_lisp.gperfsimilarity index 100%rename from llamafile/is_keyword_lisp.gperfrename to llamafile/highlight/is_keyword_lisp.gperfdiff --git a/llamafile/is_keyword_lua.gperf b/llamafile/highlight/is_keyword_lua.gperfsimilarity index 100%rename from llamafile/is_keyword_lua.gperfrename to llamafile/highlight/is_keyword_lua.gperfdiff --git a/llamafile/is_keyword_lua_builtin.gperf b/llamafile/highlight/is_keyword_lua_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_lua_builtin.gperfrename to llamafile/highlight/is_keyword_lua_builtin.gperfdiff --git a/llamafile/is_keyword_lua_constant.gperf b/llamafile/highlight/is_keyword_lua_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_lua_constant.gperfrename to llamafile/highlight/is_keyword_lua_constant.gperfdiff --git a/llamafile/is_keyword_m4.gperf b/llamafile/highlight/is_keyword_m4.gperfsimilarity index 100%rename from llamafile/is_keyword_m4.gperfrename to llamafile/highlight/is_keyword_m4.gperfdiff --git a/llamafile/is_keyword_make.gperf b/llamafile/highlight/is_keyword_make.gperfsimilarity index 100%rename from llamafile/is_keyword_make.gperfrename to llamafile/highlight/is_keyword_make.gperfdiff --git a/llamafile/is_keyword_make_builtin.gperf b/llamafile/highlight/is_keyword_make_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_make_builtin.gperfrename to llamafile/highlight/is_keyword_make_builtin.gperfdiff --git a/llamafile/is_keyword_matlab.gperf b/llamafile/highlight/is_keyword_matlab.gperfsimilarity index 100%rename from llamafile/is_keyword_matlab.gperfrename to llamafile/highlight/is_keyword_matlab.gperfdiff --git a/llamafile/is_keyword_matlab_builtin.gperf b/llamafile/highlight/is_keyword_matlab_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_matlab_builtin.gperfrename to llamafile/highlight/is_keyword_matlab_builtin.gperfdiff --git a/llamafile/is_keyword_matlab_constant.gperf b/llamafile/highlight/is_keyword_matlab_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_matlab_constant.gperfrename to llamafile/highlight/is_keyword_matlab_constant.gperfdiff --git a/llamafile/is_keyword_ocaml.gperf b/llamafile/highlight/is_keyword_ocaml.gperfsimilarity index 100%rename from llamafile/is_keyword_ocaml.gperfrename to llamafile/highlight/is_keyword_ocaml.gperfdiff --git a/llamafile/is_keyword_ocaml_builtin.gperf b/llamafile/highlight/is_keyword_ocaml_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_ocaml_builtin.gperfrename to llamafile/highlight/is_keyword_ocaml_builtin.gperfdiff --git a/llamafile/is_keyword_ocaml_constant.gperf b/llamafile/highlight/is_keyword_ocaml_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_ocaml_constant.gperfrename to llamafile/highlight/is_keyword_ocaml_constant.gperfdiff --git a/llamafile/is_keyword_pascal.gperf b/llamafile/highlight/is_keyword_pascal.gperfsimilarity index 100%rename from llamafile/is_keyword_pascal.gperfrename to llamafile/highlight/is_keyword_pascal.gperfdiff --git a/llamafile/is_keyword_pascal_builtin.gperf b/llamafile/highlight/is_keyword_pascal_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_pascal_builtin.gperfrename to llamafile/highlight/is_keyword_pascal_builtin.gperfdiff --git a/llamafile/is_keyword_pascal_type.gperf b/llamafile/highlight/is_keyword_pascal_type.gperfsimilarity index 100%rename from llamafile/is_keyword_pascal_type.gperfrename to llamafile/highlight/is_keyword_pascal_type.gperfdiff --git a/llamafile/is_keyword_perl.gperf b/llamafile/highlight/is_keyword_perl.gperfsimilarity index 100%rename from llamafile/is_keyword_perl.gperfrename to llamafile/highlight/is_keyword_perl.gperfdiff --git a/llamafile/is_keyword_php.gperf b/llamafile/highlight/is_keyword_php.gperfsimilarity index 100%rename from llamafile/is_keyword_php.gperfrename to llamafile/highlight/is_keyword_php.gperfdiff --git a/llamafile/is_keyword_php_constant.gperf b/llamafile/highlight/is_keyword_php_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_php_constant.gperfrename to llamafile/highlight/is_keyword_php_constant.gperfdiff --git a/llamafile/is_keyword_python.gperf b/llamafile/highlight/is_keyword_python.gperfsimilarity index 100%rename from llamafile/is_keyword_python.gperfrename to llamafile/highlight/is_keyword_python.gperfdiff --git a/llamafile/is_keyword_python_builtin.gperf b/llamafile/highlight/is_keyword_python_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_python_builtin.gperfrename to llamafile/highlight/is_keyword_python_builtin.gperfdiff --git a/llamafile/is_keyword_python_constant.gperf b/llamafile/highlight/is_keyword_python_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_python_constant.gperfrename to llamafile/highlight/is_keyword_python_constant.gperfdiff --git a/llamafile/is_keyword_r.gperf b/llamafile/highlight/is_keyword_r.gperfsimilarity index 100%rename from llamafile/is_keyword_r.gperfrename to llamafile/highlight/is_keyword_r.gperfdiff --git a/llamafile/is_keyword_r_builtin.gperf b/llamafile/highlight/is_keyword_r_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_r_builtin.gperfrename to llamafile/highlight/is_keyword_r_builtin.gperfdiff --git a/llamafile/is_keyword_r_constant.gperf b/llamafile/highlight/is_keyword_r_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_r_constant.gperfrename to llamafile/highlight/is_keyword_r_constant.gperfdiff --git a/llamafile/is_keyword_ruby.gperf b/llamafile/highlight/is_keyword_ruby.gperfsimilarity index 100%rename from llamafile/is_keyword_ruby.gperfrename to llamafile/highlight/is_keyword_ruby.gperfdiff --git a/llamafile/is_keyword_ruby_builtin.gperf b/llamafile/highlight/is_keyword_ruby_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_ruby_builtin.gperfrename to llamafile/highlight/is_keyword_ruby_builtin.gperfdiff --git a/llamafile/is_keyword_ruby_constant.gperf b/llamafile/highlight/is_keyword_ruby_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_ruby_constant.gperfrename to llamafile/highlight/is_keyword_ruby_constant.gperfdiff --git a/llamafile/is_keyword_rust.gperf b/llamafile/highlight/is_keyword_rust.gperfsimilarity index 100%rename from llamafile/is_keyword_rust.gperfrename to llamafile/highlight/is_keyword_rust.gperfdiff --git a/llamafile/is_keyword_rust_constant.gperf b/llamafile/highlight/is_keyword_rust_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_rust_constant.gperfrename to llamafile/highlight/is_keyword_rust_constant.gperfdiff --git a/llamafile/is_keyword_rust_type.gperf b/llamafile/highlight/is_keyword_rust_type.gperfsimilarity index 100%rename from llamafile/is_keyword_rust_type.gperfrename to llamafile/highlight/is_keyword_rust_type.gperfdiff --git a/llamafile/is_keyword_scala.gperf b/llamafile/highlight/is_keyword_scala.gperfsimilarity index 100%rename from llamafile/is_keyword_scala.gperfrename to llamafile/highlight/is_keyword_scala.gperfdiff --git a/llamafile/is_keyword_shell.gperf b/llamafile/highlight/is_keyword_shell.gperfsimilarity index 100%rename from llamafile/is_keyword_shell.gperfrename to llamafile/highlight/is_keyword_shell.gperfdiff --git a/llamafile/is_keyword_shell_builtin.gperf b/llamafile/highlight/is_keyword_shell_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_shell_builtin.gperfrename to llamafile/highlight/is_keyword_shell_builtin.gperfdiff --git a/llamafile/is_keyword_sql.gperf b/llamafile/highlight/is_keyword_sql.gperfsimilarity index 100%rename from llamafile/is_keyword_sql.gperfrename to llamafile/highlight/is_keyword_sql.gperfdiff --git a/llamafile/is_keyword_sql_type.gperf b/llamafile/highlight/is_keyword_sql_type.gperfsimilarity index 100%rename from llamafile/is_keyword_sql_type.gperfrename to llamafile/highlight/is_keyword_sql_type.gperfdiff --git a/llamafile/is_keyword_swift.gperf b/llamafile/highlight/is_keyword_swift.gperfsimilarity index 100%rename from llamafile/is_keyword_swift.gperfrename to llamafile/highlight/is_keyword_swift.gperfdiff --git a/llamafile/is_keyword_swift_builtin.gperf b/llamafile/highlight/is_keyword_swift_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_swift_builtin.gperfrename to llamafile/highlight/is_keyword_swift_builtin.gperfdiff --git a/llamafile/is_keyword_swift_constant.gperf b/llamafile/highlight/is_keyword_swift_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_swift_constant.gperfrename to llamafile/highlight/is_keyword_swift_constant.gperfdiff --git a/llamafile/is_keyword_swift_type.gperf b/llamafile/highlight/is_keyword_swift_type.gperfsimilarity index 100%rename from llamafile/is_keyword_swift_type.gperfrename to llamafile/highlight/is_keyword_swift_type.gperfdiff --git a/llamafile/is_keyword_tcl.gperf b/llamafile/highlight/is_keyword_tcl.gperfsimilarity index 100%rename from llamafile/is_keyword_tcl.gperfrename to llamafile/highlight/is_keyword_tcl.gperfdiff --git a/llamafile/is_keyword_tcl_builtin.gperf b/llamafile/highlight/is_keyword_tcl_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_tcl_builtin.gperfrename to llamafile/highlight/is_keyword_tcl_builtin.gperfdiff --git a/llamafile/is_keyword_tcl_type.gperf b/llamafile/highlight/is_keyword_tcl_type.gperfsimilarity index 100%rename from llamafile/is_keyword_tcl_type.gperfrename to llamafile/highlight/is_keyword_tcl_type.gperfdiff --git a/llamafile/is_keyword_typescript.gperf b/llamafile/highlight/is_keyword_typescript.gperfsimilarity index 100%rename from llamafile/is_keyword_typescript.gperfrename to llamafile/highlight/is_keyword_typescript.gperfdiff --git a/llamafile/is_keyword_typescript_type.gperf b/llamafile/highlight/is_keyword_typescript_type.gperfsimilarity index 100%rename from llamafile/is_keyword_typescript_type.gperfrename to llamafile/highlight/is_keyword_typescript_type.gperfdiff --git a/llamafile/is_keyword_zig.gperf b/llamafile/highlight/is_keyword_zig.gperfsimilarity index 100%rename from llamafile/is_keyword_zig.gperfrename to llamafile/highlight/is_keyword_zig.gperfdiff --git a/llamafile/is_keyword_zig_builtin.gperf b/llamafile/highlight/is_keyword_zig_builtin.gperfsimilarity index 100%rename from llamafile/is_keyword_zig_builtin.gperfrename to llamafile/highlight/is_keyword_zig_builtin.gperfdiff --git a/llamafile/is_keyword_zig_constant.gperf b/llamafile/highlight/is_keyword_zig_constant.gperfsimilarity index 100%rename from llamafile/is_keyword_zig_constant.gperfrename to llamafile/highlight/is_keyword_zig_constant.gperfdiff --git a/llamafile/is_keyword_zig_type.gperf b/llamafile/highlight/is_keyword_zig_type.gperfsimilarity index 100%rename from llamafile/is_keyword_zig_type.gperfrename to llamafile/highlight/is_keyword_zig_type.gperfdiff --git a/llamafile/highlight/util.cpp b/llamafile/highlight/util.cppnew file mode 100644index 00000000..71072815--- /dev/null+++ b/llamafile/highlight/util.cpp@@ -0,0 +1,38 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "util.h"+#include <cctype>+#include <cosmo.h>+#include <string>++namespace lf {++void+append_wchar(std::string* r, wchar_t c)+{+ if (isascii(c)) {+ *r += c;+ } else {+ char s[8];+ uint64_t w = tpenc(c);+ WRITE64LE(s, w);+ *r += s;+ }+}++} // namespace lfdiff --git a/llamafile/highlight/util.h b/llamafile/highlight/util.hnew file mode 100644index 00000000..0b25c94e--- /dev/null+++ b/llamafile/highlight/util.h@@ -0,0 +1,26 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <__fwd/string.h>++namespace lf {++void+append_wchar(std::string*, wchar_t);++} // namespace lfdiff --git a/llamafile/image.cpp b/llamafile/image.cppindex 63cb14b2..2d9ca22b 100644--- a/llamafile/image.cpp+++ b/llamafile/image.cpp@@ -25,8 +25,8 @@#include "llama.cpp/base64.h"#include "llamafile/macros.h"#include "llamafile/xterm.h"-#include "stb/stb_image.h"-#include "stb/stb_image_resize2.h"+#include "third_party/stb/stb_image.h"+#include "third_party/stb/stb_image_resize2.h"namespace lf {diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mkindex 54aa5eb0..a7f76ce7 100644--- a/llamafile/server/BUILD.mk+++ b/llamafile/server/BUILD.mk@@ -12,18 +12,18 @@ LLAMAFILE_SERVER_ASSETS = $(wildcard llamafile/server/www/*)$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g-o/$(MODE)/llamafile/server/server.a: \+o/$(MODE)/llamafile/server/server.a: \$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))-o/$(MODE)/llamafile/server/main: \- o/$(MODE)/llamafile/server/main.o \- o/$(MODE)/llamafile/server/main.1.asc.zip.o \- o/$(MODE)/llamafile/server/server.a \- o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/llama.cpp/llava/llava.a \- o/$(MODE)/double-conversion/double-conversion.a \- o/$(MODE)/stb/stb.a \- $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)+o/$(MODE)/llamafile/server/main: \+ o/$(MODE)/llamafile/server/main.o \+ o/$(MODE)/llamafile/server/main.1.asc.zip.o \+ o/$(MODE)/llamafile/server/server.a \+ o/$(MODE)/llama.cpp/llama.cpp.a \+ o/$(MODE)/llama.cpp/llava/llava.a \+ o/$(MODE)/third_party/double-conversion/double-conversion.a \+ o/$(MODE)/third_party/stb/stb.a \+ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o) \# turn /zip/llamafile/server/www/...# into /zip/www/...@@ -31,37 +31,37 @@ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o): private ZIPOBJ_FLAGS += -C2$(LLAMAFILE_SERVER_OBJS): llamafile/server/BUILD.mk-o/$(MODE)/llamafile/server/atom_test: \- o/$(MODE)/llamafile/server/atom_test.o \- o/$(MODE)/llamafile/server/atom.o \- o/$(MODE)/llamafile/server/image.o \+o/$(MODE)/llamafile/server/atom_test: \+ o/$(MODE)/llamafile/server/atom_test.o \+ o/$(MODE)/llamafile/server/atom.o \+ o/$(MODE)/llamafile/server/image.o \-o/$(MODE)/llamafile/server/image_test: \- o/$(MODE)/llamafile/server/image_test.o \- o/$(MODE)/llamafile/server/image.o \+o/$(MODE)/llamafile/server/image_test: \+ o/$(MODE)/llamafile/server/image_test.o \+ o/$(MODE)/llamafile/server/image.o \-o/$(MODE)/llamafile/server/fastjson_test: \- o/$(MODE)/llamafile/server/fastjson_test.o \- o/$(MODE)/llamafile/server/fastjson.o \- o/$(MODE)/double-conversion/double-conversion.a \+o/$(MODE)/llamafile/server/fastjson_test: \+ o/$(MODE)/llamafile/server/fastjson_test.o \+ o/$(MODE)/llamafile/server/fastjson.o \+ o/$(MODE)/double-conversion/double-conversion.a \-o/$(MODE)/llamafile/server/json_test: \- o/$(MODE)/llamafile/server/json_test.o \- o/$(MODE)/llamafile/server/json.o \- o/$(MODE)/llamafile/server/hextoint.o \- o/$(MODE)/double-conversion/double-conversion.a \+o/$(MODE)/llamafile/server/json_test: \+ o/$(MODE)/llamafile/server/json_test.o \+ o/$(MODE)/llamafile/server/json.o \+ o/$(MODE)/llamafile/server/hextoint.o \+ o/$(MODE)/double-conversion/double-conversion.a \-o/$(MODE)/llamafile/server/tokenbucket_test: \- o/$(MODE)/llamafile/server/tokenbucket_test.o \- o/$(MODE)/llamafile/server/tokenbucket.o \- o/$(MODE)/llamafile/server/log.o \- o/$(MODE)/llama.cpp/llama.cpp.a \+o/$(MODE)/llamafile/server/tokenbucket_test: \+ o/$(MODE)/llamafile/server/tokenbucket_test.o \+ o/$(MODE)/llamafile/server/tokenbucket.o \+ o/$(MODE)/llamafile/server/log.o \+ o/$(MODE)/llama.cpp/llama.cpp.a \.PHONY: o/$(MODE)/llamafile/server-o/$(MODE)/llamafile/server: \- o/$(MODE)/llamafile/server/main \- o/$(MODE)/llamafile/server/atom_test.runs \- o/$(MODE)/llamafile/server/fastjson_test.runs \- o/$(MODE)/llamafile/server/image_test.runs \- o/$(MODE)/llamafile/server/json_test.runs \- o/$(MODE)/llamafile/server/tokenbucket_test.runs \+o/$(MODE)/llamafile/server: \+ o/$(MODE)/llamafile/server/main \+ o/$(MODE)/llamafile/server/atom_test.runs \+ o/$(MODE)/llamafile/server/fastjson_test.runs \+ o/$(MODE)/llamafile/server/image_test.runs \+ o/$(MODE)/llamafile/server/json_test.runs \+ o/$(MODE)/llamafile/server/tokenbucket_test.runs \diff --git a/llamafile/server/fastjson.cpp b/llamafile/server/fastjson.cppindex 62d46ce8..efb76e47 100644--- a/llamafile/server/fastjson.cpp+++ b/llamafile/server/fastjson.cpp@@ -16,8 +16,8 @@// limitations under the License.#include "fastjson.h"-#include "double-conversion/double-to-string.h"-#include "double-conversion/utils.h"+#include "third_party/double-conversion/double-to-string.h"+#include "third_party/double-conversion/utils.h"#include <cosmo.h>#include <net/http/escape.h>#include <string>diff --git a/llamafile/server/json.cpp b/llamafile/server/json.cppindex a5b03c3b..127b6074 100644--- a/llamafile/server/json.cpp+++ b/llamafile/server/json.cpp@@ -24,8 +24,8 @@#include <cstdlib>#include <stdckdint.h>-#include "double-conversion/double-to-string.h"-#include "double-conversion/string-to-double.h"+#include "third_party/double-conversion/double-to-string.h"+#include "third_party/double-conversion/string-to-double.h"#define KEY 1#define COMMA 2diff --git a/llamafile/string.cpp b/llamafile/string.cppindex ef67d581..a975d430 100644--- a/llamafile/string.cpp+++ b/llamafile/string.cpp@@ -52,17 +52,6 @@ bool startscasewith(const std::string_view &str, const std::string_view &prefix)return true;}-void append_wchar(std::string *r, wchar_t c) {- if (isascii(c)) {- *r += c;- } else {- char s[8];- uint64_t w = tpenc(c);- WRITE64LE(s, w);- *r += s;- }-}-std::string format(const char *fmt, ...) {va_list ap, ap2;va_start(ap, fmt);diff --git a/stable-diffusion.cpp/BUILD.mk b/stable-diffusion.cpp/BUILD.mkindex 2a6c95f1..6e5465cf 100644--- a/stable-diffusion.cpp/BUILD.mk+++ b/stable-diffusion.cpp/BUILD.mk@@ -30,7 +30,7 @@ o/$(MODE)/stable-diffusion.cpp/main: \o/$(MODE)/stable-diffusion.cpp/main.o \o/$(MODE)/stable-diffusion.cpp/stable-diffusion.cpp.a \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a+ o/$(MODE)/third_party/stb/stb.a$(STABLE_DIFFUSION_CPP_OBJS): stable-diffusion.cpp/BUILD.mkdiff --git a/stable-diffusion.cpp/main.cpp b/stable-diffusion.cpp/main.cppindex 2b4c75a9..1d466053 100644--- a/stable-diffusion.cpp/main.cpp+++ b/stable-diffusion.cpp/main.cpp@@ -15,15 +15,15 @@// #define STB_IMAGE_IMPLEMENTATION// #define STB_IMAGE_STATIC-#include "stb/stb_image.h"+#include "third_party/stb/stb_image.h"// #define STB_IMAGE_WRITE_IMPLEMENTATION// #define STB_IMAGE_WRITE_STATIC-#include "stb/stb_image_write.h"+#include "third_party/stb/stb_image_write.h"// #define STB_IMAGE_RESIZE_IMPLEMENTATION// #define STB_IMAGE_RESIZE_STATIC-#include "stb/stb_image_resize2.h"+#include "third_party/stb/stb_image_resize2.h"#include "llamafile/llamafile.h"#include "llamafile/debug.h"diff --git a/stable-diffusion.cpp/stable-diffusion.cpp b/stable-diffusion.cpp/stable-diffusion.cppindex 29dfd739..352adf1a 100644--- a/stable-diffusion.cpp/stable-diffusion.cpp+++ b/stable-diffusion.cpp/stable-diffusion.cpp@@ -18,11 +18,11 @@// #define STB_IMAGE_IMPLEMENTATION// #define STB_IMAGE_STATIC-#include "stb/stb_image.h"+#include "third_party/stb/stb_image.h"// #define STB_IMAGE_WRITE_IMPLEMENTATION// #define STB_IMAGE_WRITE_STATIC-// #include "stb_image_write.h"+// #include "third_party/stb/stb_image_write.h"const char* model_version_to_str[] = {"1.x",diff --git a/stable-diffusion.cpp/util.cpp b/stable-diffusion.cpp/util.cppindex 806b25ec..7aa48631 100644--- a/stable-diffusion.cpp/util.cpp+++ b/stable-diffusion.cpp/util.cpp@@ -25,7 +25,7 @@#include "llama.cpp/ggml.h"#include "stable-diffusion.h"-#include "stb/stb_image_resize2.h"+#include "third_party/stb/stb_image_resize2.h"bool ends_with(const std::string& str, const std::string& ending) {if (str.length() >= ending.length()) {diff --git a/third_party/BUILD.mk b/third_party/BUILD.mknew file mode 100644index 00000000..cb4edaef--- /dev/null+++ b/third_party/BUILD.mk@@ -0,0 +1,7 @@+include third_party/double-conversion/BUILD.mk+include third_party/stb/BUILD.mk++.PHONY: o/$(MODE)/third_party+o/$(MODE)/third_party: \+ o/$(MODE)/third_party/double-conversion \+ o/$(MODE)/third_party/stb \diff --git a/double-conversion/.gitignore b/third_party/double-conversion/.gitignoresimilarity index 100%rename from double-conversion/.gitignorerename to third_party/double-conversion/.gitignorediff --git a/double-conversion/AUTHORS b/third_party/double-conversion/AUTHORSsimilarity index 100%rename from double-conversion/AUTHORSrename to third_party/double-conversion/AUTHORSdiff --git a/double-conversion/BUILD.mk b/third_party/double-conversion/BUILD.mksimilarity index 52%rename from double-conversion/BUILD.mkrename to third_party/double-conversion/BUILD.mkindex 2aca9e7f..689d04f4 100644--- a/double-conversion/BUILD.mk+++ b/third_party/double-conversion/BUILD.mk@@ -3,14 +3,15 @@PKGS += DOUBLE_CONVERSION-DOUBLE_CONVERSION_FILES := $(wildcard double-conversion/*)+DOUBLE_CONVERSION_FILES := $(wildcard third_party/double-conversion/*)DOUBLE_CONVERSION_HDRS = $(filter %.h,$(DOUBLE_CONVERSION_FILES))DOUBLE_CONVERSION_SRCS = $(filter %.cc,$(DOUBLE_CONVERSION_FILES))DOUBLE_CONVERSION_OBJS = $(DOUBLE_CONVERSION_SRCS:%.cc=o/$(MODE)/%.o)-o/$(MODE)/double-conversion/double-conversion.a: $(DOUBLE_CONVERSION_OBJS)+o/$(MODE)/third_party/double-conversion/double-conversion.a: $(DOUBLE_CONVERSION_OBJS)-$(DOUBLE_CONVERSION_OBJS): double-conversion/BUILD.mk+$(DOUBLE_CONVERSION_OBJS): third_party/double-conversion/BUILD.mk-.PHONY: o/$(MODE)/double-conversion-o/$(MODE)/double-conversion: o/$(MODE)/double-conversion/double-conversion.a+.PHONY: o/$(MODE)/third_party/double-conversion+o/$(MODE)/third_party/double-conversion: \+ o/$(MODE)/third_party/double-conversion/double-conversion.adiff --git a/double-conversion/LICENSE b/third_party/double-conversion/LICENSEsimilarity index 100%rename from double-conversion/LICENSErename to third_party/double-conversion/LICENSEdiff --git a/double-conversion/README.llamafile b/third_party/double-conversion/README.llamafilesimilarity index 100%rename from double-conversion/README.llamafilerename to third_party/double-conversion/README.llamafilediff --git a/double-conversion/SConscript b/third_party/double-conversion/SConscriptsimilarity index 100%rename from double-conversion/SConscriptrename to third_party/double-conversion/SConscriptdiff --git a/double-conversion/bignum-dtoa.cc b/third_party/double-conversion/bignum-dtoa.ccsimilarity index 100%rename from double-conversion/bignum-dtoa.ccrename to third_party/double-conversion/bignum-dtoa.ccdiff --git a/double-conversion/bignum-dtoa.h b/third_party/double-conversion/bignum-dtoa.hsimilarity index 100%rename from double-conversion/bignum-dtoa.hrename to third_party/double-conversion/bignum-dtoa.hdiff --git a/double-conversion/bignum.cc b/third_party/double-conversion/bignum.ccsimilarity index 100%rename from double-conversion/bignum.ccrename to third_party/double-conversion/bignum.ccdiff --git a/double-conversion/bignum.h b/third_party/double-conversion/bignum.hsimilarity index 100%rename from double-conversion/bignum.hrename to third_party/double-conversion/bignum.hdiff --git a/double-conversion/cached-powers.cc b/third_party/double-conversion/cached-powers.ccsimilarity index 100%rename from double-conversion/cached-powers.ccrename to third_party/double-conversion/cached-powers.ccdiff --git a/double-conversion/cached-powers.h b/third_party/double-conversion/cached-powers.hsimilarity index 100%rename from double-conversion/cached-powers.hrename to third_party/double-conversion/cached-powers.hdiff --git a/double-conversion/diy-fp.h b/third_party/double-conversion/diy-fp.hsimilarity index 100%rename from double-conversion/diy-fp.hrename to third_party/double-conversion/diy-fp.hdiff --git a/double-conversion/double-conversion.h b/third_party/double-conversion/double-conversion.hsimilarity index 100%rename from double-conversion/double-conversion.hrename to third_party/double-conversion/double-conversion.hdiff --git a/double-conversion/double-to-string.cc b/third_party/double-conversion/double-to-string.ccsimilarity index 100%rename from double-conversion/double-to-string.ccrename to third_party/double-conversion/double-to-string.ccdiff --git a/double-conversion/double-to-string.h b/third_party/double-conversion/double-to-string.hsimilarity index 100%rename from double-conversion/double-to-string.hrename to third_party/double-conversion/double-to-string.hdiff --git a/double-conversion/fast-dtoa.cc b/third_party/double-conversion/fast-dtoa.ccsimilarity index 100%rename from double-conversion/fast-dtoa.ccrename to third_party/double-conversion/fast-dtoa.ccdiff --git a/double-conversion/fast-dtoa.h b/third_party/double-conversion/fast-dtoa.hsimilarity index 100%rename from double-conversion/fast-dtoa.hrename to third_party/double-conversion/fast-dtoa.hdiff --git a/double-conversion/fixed-dtoa.cc b/third_party/double-conversion/fixed-dtoa.ccsimilarity index 100%rename from double-conversion/fixed-dtoa.ccrename to third_party/double-conversion/fixed-dtoa.ccdiff --git a/double-conversion/fixed-dtoa.h b/third_party/double-conversion/fixed-dtoa.hsimilarity index 100%rename from double-conversion/fixed-dtoa.hrename to third_party/double-conversion/fixed-dtoa.hdiff --git a/double-conversion/ieee.h b/third_party/double-conversion/ieee.hsimilarity index 100%rename from double-conversion/ieee.hrename to third_party/double-conversion/ieee.hdiff --git a/double-conversion/string-to-double.cc b/third_party/double-conversion/string-to-double.ccsimilarity index 100%rename from double-conversion/string-to-double.ccrename to third_party/double-conversion/string-to-double.ccdiff --git a/double-conversion/string-to-double.h b/third_party/double-conversion/string-to-double.hsimilarity index 100%rename from double-conversion/string-to-double.hrename to third_party/double-conversion/string-to-double.hdiff --git a/double-conversion/strtod.cc b/third_party/double-conversion/strtod.ccsimilarity index 100%rename from double-conversion/strtod.ccrename to third_party/double-conversion/strtod.ccdiff --git a/double-conversion/strtod.h b/third_party/double-conversion/strtod.hsimilarity index 100%rename from double-conversion/strtod.hrename to third_party/double-conversion/strtod.hdiff --git a/double-conversion/utils.h b/third_party/double-conversion/utils.hsimilarity index 100%rename from double-conversion/utils.hrename to third_party/double-conversion/utils.hdiff --git a/stb/BUILD.mk b/third_party/stb/BUILD.mksimilarity index 62%rename from stb/BUILD.mkrename to third_party/stb/BUILD.mkindex aa8c5be7..af276a6a 100644--- a/stb/BUILD.mk+++ b/third_party/stb/BUILD.mk@@ -3,15 +3,15 @@PKGS += STB-STB_FILES := $(wildcard stb/*)+STB_FILES := $(wildcard third_party/stb/*)STB_HDRS = $(filter %.h,$(STB_FILES))STB_INCS = $(filter %.inc,$(STB_FILES))STB_SRCS = $(filter %.c,$(STB_FILES))STB_OBJS = $(STB_SRCS:%.c=o/$(MODE)/%.o)-o/$(MODE)/stb/stb.a: $(STB_OBJS)+o/$(MODE)/third_party/stb/stb.a: $(STB_OBJS)-$(STB_OBJS): stb/BUILD.mk+$(STB_OBJS): third_party/stb/BUILD.mk-.PHONY: o/$(MODE)/stb-o/$(MODE)/stb: o/$(MODE)/stb/stb.a+.PHONY: o/$(MODE)/third_party/stb+o/$(MODE)/third_party/stb: o/$(MODE)/third_party/stb/stb.adiff --git a/stb/README.llamafile b/third_party/stb/README.llamafilesimilarity index 100%rename from stb/README.llamafilerename to third_party/stb/README.llamafilediff --git a/stb/stb_image.c b/third_party/stb/stb_image.csimilarity index 100%rename from stb/stb_image.crename to third_party/stb/stb_image.cdiff --git a/stb/stb_image.h b/third_party/stb/stb_image.hsimilarity index 100%rename from stb/stb_image.hrename to third_party/stb/stb_image.hdiff --git a/stb/stb_image_resize2.c b/third_party/stb/stb_image_resize2.csimilarity index 100%rename from stb/stb_image_resize2.crename to third_party/stb/stb_image_resize2.cdiff --git a/stb/stb_image_resize2.h b/third_party/stb/stb_image_resize2.hsimilarity index 100%rename from stb/stb_image_resize2.hrename to third_party/stb/stb_image_resize2.hdiff --git a/stb/stb_image_write.c b/third_party/stb/stb_image_write.csimilarity index 100%rename from stb/stb_image_write.crename to third_party/stb/stb_image_write.cdiff --git a/stb/stb_image_write.h b/third_party/stb/stb_image_write.hsimilarity index 100%rename from stb/stb_image_write.hrename to third_party/stb/stb_image_write.hdiff --git a/stb/stb_vorbis.c b/third_party/stb/stb_vorbis.csimilarity index 100%rename from stb/stb_vorbis.crename to third_party/stb/stb_vorbis.cdiff --git a/stb/stb_vorbis.h b/third_party/stb/stb_vorbis.hsimilarity index 100%rename from stb/stb_vorbis.hrename to third_party/stb/stb_vorbis.hdiff --git a/whisper.cpp/BUILD.mk b/whisper.cpp/BUILD.mkindex 6e692c2a..af6aebc6 100644--- a/whisper.cpp/BUILD.mk+++ b/whisper.cpp/BUILD.mk@@ -33,12 +33,12 @@ o/$(MODE)/whisper.cpp/main: \o/$(MODE)/whisper.cpp/main.1.asc.zip.o \o/$(MODE)/whisper.cpp/whisper.cpp.a \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a \+ o/$(MODE)/third_party/stb/stb.a \o/$(MODE)/whisper.cpp/stream: \o/$(MODE)/whisper.cpp/whisper.cpp.a \o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/stb/stb.a \+ o/$(MODE)/third_party/stb/stb.a \o/$(MODE)/whisper.cpp/mic2txt: \o/$(MODE)/whisper.cpp/whisper.cpp.a \diff --git a/whisper.cpp/common.cpp b/whisper.cpp/common.cppindex 1492c148..06e9ed70 100644--- a/whisper.cpp/common.cpp+++ b/whisper.cpp/common.cpp@@ -24,7 +24,7 @@#include <stdlib.h>#include <unistd.h>-#include "stb/stb_vorbis.h"+#include "third_party/stb/stb_vorbis.h"#include "miniaudio.h"void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {diff --git a/whisper.cpp/miniaudio.c b/whisper.cpp/miniaudio.cindex a5e285b3..5629fc99 100644--- a/whisper.cpp/miniaudio.c+++ b/whisper.cpp/miniaudio.c@@ -1,4 +1,4 @@-#include "stb/stb_vorbis.h"+#include "third_party/stb/stb_vorbis.h"#define MA_NO_DEVICE_IO#define MA_NO_RUNTIME_LINKING