Skip to content

Distribute and run LLMs with a single file.

Refactor package structure

Justine Tunney05fa4f35c0fc
diff --git a/Makefile b/Makefile
index e3f6f6ef..c5cf66f4 100644
--- a/Makefile
+++ b/Makefile
@@ -11,12 +11,11 @@ MAKEFLAGS += --no-builtin-rules
include build/config.mk
include build/rules.mk
+include third_party/BUILD.mk
include llamafile/BUILD.mk
include llama.cpp/BUILD.mk
include stable-diffusion.cpp/BUILD.mk
include whisper.cpp/BUILD.mk
-include double-conversion/BUILD.mk
-include stb/BUILD.mk
# the root package is `o//` by default
# building a package also builds its sub-packages
@@ -25,7 +24,7 @@ o/$(MODE)/: o/$(MODE)/llamafile \
o/$(MODE)/llama.cpp \
o/$(MODE)/stable-diffusion.cpp \
o/$(MODE)/whisper.cpp \
- o/$(MODE)/stb \
+ o/$(MODE)/third_party \
o/$(MODE)/depend.test
# for installing to `make PREFIX=/usr/local`
diff --git a/llama.cpp/llava/BUILD.mk b/llama.cpp/llava/BUILD.mk
index ed9e0e17..83183b00 100644
--- a/llama.cpp/llava/BUILD.mk
+++ b/llama.cpp/llava/BUILD.mk
@@ -16,7 +16,7 @@ o/$(MODE)/llama.cpp/llava/llava-quantize: \
o/$(MODE)/llama.cpp/llava/llava-quantize.1.asc.zip.o \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a
+ o/$(MODE)/third_party/stb/stb.a
$(LLAMA_CPP_LLAVA_OBJS): llama.cpp/llava/BUILD.mk
diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cpp
index 85a8db0e..0c0f36af 100644
--- a/llama.cpp/llava/clip.cpp
+++ b/llama.cpp/llava/clip.cpp
@@ -13,7 +13,7 @@
#include "llama.cpp/ggml-metal.h"
#include "llama.cpp/string.h"
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
#include <cassert>
#include <cmath>
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index 5985629e..fb432e16 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -14,7 +14,8 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/server/server.a \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a \
+ o/$(MODE)/llamafile/highlight/highlight.a \
+ o/$(MODE)/third_party/stb/stb.a \
o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 76978305..555b0e10 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -5,7 +5,7 @@
#include "llama.cpp/llama.h"
#include "llama.cpp/grammar-parser.h"
#include "llama.cpp/llava/llava.h"
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
#include "utils.h"
#include "oai.h"
#include "llamafile/micros.h"
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index a374cfd3..f5734415 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -9,16 +9,13 @@ LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS_GPERF_C = $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)
-LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU) $(LLAMAFILE_SRCS_GPERF)
+LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)
LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))
LLAMAFILE_OBJS := \
$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o) \
$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o) \
- $(LLAMAFILE_SRCS_GPERF_C:%.c=%.o) \
$(LLAMAFILE_OBJS): private CCFLAGS += -g
@@ -26,6 +23,7 @@ $(LLAMAFILE_OBJS): private CCFLAGS += -g
# therefore we want to avoid it going inside the .a file
LLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))
+include llamafile/highlight/BUILD.mk
include llamafile/server/BUILD.mk
o/$(MODE)/llamafile/zipalign: \
@@ -57,9 +55,6 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/addnl \
o/$(MODE)/llamafile/high \
o/$(MODE)/llamafile/datauri_test.runs \
- o/$(MODE)/llamafile/highlight_c_test.runs \
- o/$(MODE)/llamafile/highlight_python_test.runs \
- o/$(MODE)/llamafile/highlight_test.runs \
o/$(MODE)/llamafile/parse_cidr_test.runs \
o/$(MODE)/llamafile/pool_cancel_test.runs \
o/$(MODE)/llamafile/pool_test.runs \
@@ -170,34 +165,15 @@ o/$(MODE)/llamafile/pool_test: \
o/$(MODE)/llamafile/crash.o \
o/$(MODE)/llamafile/pool.o \
-o/$(MODE)/llamafile/highlight_test: \
- o/$(MODE)/llamafile/highlight_test.o \
- o/$(MODE)/llama.cpp/llama.cpp.a \
-
-o/$(MODE)/llamafile/highlight_c_test: \
- o/$(MODE)/llamafile/highlight_c_test.o \
- o/$(MODE)/llamafile/highlight_c.o \
- o/$(MODE)/llamafile/is_keyword_c.o \
- o/$(MODE)/llamafile/is_keyword_c_constant.o \
- o/$(MODE)/llamafile/is_keyword_c_type.o \
- o/$(MODE)/llamafile/is_keyword_c_pod.o \
- o/$(MODE)/llamafile/is_keyword_cpp.o \
-
-o/$(MODE)/llamafile/highlight_python_test: \
- o/$(MODE)/llamafile/highlight_python_test.o \
- o/$(MODE)/llamafile/highlight_python.o \
- o/$(MODE)/llamafile/is_keyword_python.o \
- o/$(MODE)/llamafile/is_keyword_python_builtin.o \
- o/$(MODE)/llamafile/is_keyword_python_constant.o \
-
o/$(MODE)/llamafile/datauri_test: \
o/$(MODE)/llamafile/datauri_test.o \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a \
+ o/$(MODE)/third_party/stb/stb.a \
-o/$(MODE)/llamafile/high: \
- o/$(MODE)/llamafile/high.o \
- o/$(MODE)/llama.cpp/llama.cpp.a \
+o/$(MODE)/llamafile/high: \
+ o/$(MODE)/llamafile/high.o \
+ o/$(MODE)/llamafile/highlight/highlight.a \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
o/$(MODE)/llamafile/hex2xterm: \
o/$(MODE)/llamafile/hex2xterm.o \
@@ -257,5 +233,4 @@ o/$(MODE)/llamafile/pick_a_warp_kernel.o: private CFLAGS += -fopenmp
.PHONY: o/$(MODE)/llamafile/check
o/$(MODE)/llamafile/check: \
- $(LLAMAFILE_SRCS_GPERF_C) \
o/$(MODE)/llamafile/tinyblas_test.runs
diff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cpp
index 41daea8c..ad29dbd3 100644
--- a/llamafile/chatbot_repl.cpp
+++ b/llamafile/chatbot_repl.cpp
@@ -26,7 +26,7 @@
#include "llama.cpp/llama.h"
#include "llamafile/bestline.h"
#include "llamafile/color.h"
-#include "llamafile/highlight.h"
+#include "llamafile/highlight/highlight.h"
#include "llamafile/llama.h"
namespace lf {
diff --git a/llamafile/high.cpp b/llamafile/high.cpp
index c19b9d46..24c0deae 100644
--- a/llamafile/high.cpp
+++ b/llamafile/high.cpp
@@ -15,7 +15,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include "highlight.h"
+#include "llamafile/highlight/highlight.h"
#include "string.h"
#include <fcntl.h>
#include <stdio.h>
diff --git a/llamafile/highlight.h b/llamafile/highlight.h
deleted file mode 100644
index 99cc6ce4..00000000
--- a/llamafile/highlight.h
+++ /dev/null
@@ -1,813 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-#include <string>
-#include <string_view>
-
-#define HI_RESET "\033[0m"
-#define HI_BOLD "\033[1m"
-#define HI_UNBOLD "\033[22m"
-#define HI_ITALIC HI_INVERT
-#define HI_INVERT "\033[7m"
-#define HI_KEYWORD "\033[1;34m" // bold blue
-#define HI_STRING "\033[32m" // green
-#define HI_COMMENT "\033[31m" // red
-#define HI_VAR "\033[1;35m" // magenta
-#define HI_MACRO "\033[35m" // magenta
-#define HI_ATTRIB "\033[35m" // magenta
-#define HI_LINENO "\033[2m" // fade
-#define HI_CONTIN "\033[33m" // yellow
-#define HI_LABEL "\033[33m" // yellow
-#define HI_DEF "\033[2;33m" // dim yellow
-#define HI_TYPE "\033[36m" // cyan
-#define HI_CLASS "\033[1;36m" // bold cyan
-#define HI_SELECTOR "\033[33m" // yellow
-#define HI_PROPERTY "\033[36m" // cyan
-#define HI_TAG "\033[33m" // yellow
-#define HI_INCODE "\033[1;35m" // magenta
-#define HI_BUILTIN "\033[35m" // magenta
-#define HI_CONSTANT "\033[1;35m" // bold magenta
-#define HI_LISPKW "\033[35m" // magenta
-#define HI_ENTITY "\033[36m" // cyan
-#define HI_OPERATOR "\033[36m" // cyan
-#define HI_ESCAPE "\033[33m" // yellow
-#define HI_QUALIFIER "\033[35m" // magenta
-#define HI_IMMEDIATE "\033[36m" // cyan
-#define HI_REGISTER "\033[1;35m" // bold magenta
-#define HI_DIRECTIVE "\033[1;31m" // bold red
-#define HI_WARNING "\033[1;31m" // bold red
-#define HI_MATH "\033[1;35m" // bold magenta
-
-typedef const char *is_keyword_f(const char *, size_t);
-
-extern "C" {
-is_keyword_f is_keyword_c;
-is_keyword_f is_keyword_c_pod;
-is_keyword_f is_keyword_c_type;
-is_keyword_f is_keyword_c_builtin;
-is_keyword_f is_keyword_c_constant;
-is_keyword_f is_keyword_cpp;
-is_keyword_f is_keyword_cxx;
-is_keyword_f is_keyword_js;
-is_keyword_f is_keyword_js_builtin;
-is_keyword_f is_keyword_js_constant;
-is_keyword_f is_keyword_java;
-is_keyword_f is_keyword_java_constant;
-is_keyword_f is_keyword_python;
-is_keyword_f is_keyword_python_builtin;
-is_keyword_f is_keyword_python_constant;
-is_keyword_f is_keyword_rust;
-is_keyword_f is_keyword_rust_type;
-is_keyword_f is_keyword_rust_constant;
-is_keyword_f is_keyword_fortran;
-is_keyword_f is_keyword_fortran_type;
-is_keyword_f is_keyword_fortran_builtin;
-is_keyword_f is_keyword_cobol;
-is_keyword_f is_keyword_pascal;
-is_keyword_f is_keyword_pascal_type;
-is_keyword_f is_keyword_pascal_builtin;
-is_keyword_f is_keyword_go;
-is_keyword_f is_keyword_go_type;
-is_keyword_f is_keyword_sql;
-is_keyword_f is_keyword_sql_type;
-is_keyword_f is_keyword_php;
-is_keyword_f is_keyword_php_constant;
-is_keyword_f is_keyword_csharp;
-is_keyword_f is_keyword_csharp_constant;
-is_keyword_f is_keyword_kotlin;
-is_keyword_f is_keyword_kotlin_type;
-is_keyword_f is_keyword_lua;
-is_keyword_f is_keyword_lua_builtin;
-is_keyword_f is_keyword_lua_constant;
-is_keyword_f is_keyword_lisp;
-is_keyword_f is_keyword_ada;
-is_keyword_f is_keyword_ada_constant;
-is_keyword_f is_keyword_haskell;
-is_keyword_f is_keyword_perl;
-is_keyword_f is_keyword_shell;
-is_keyword_f is_keyword_shell_builtin;
-is_keyword_f is_keyword_swift;
-is_keyword_f is_keyword_swift_type;
-is_keyword_f is_keyword_swift_builtin;
-is_keyword_f is_keyword_swift_constant;
-is_keyword_f is_keyword_d;
-is_keyword_f is_keyword_d_constant;
-is_keyword_f is_keyword_zig;
-is_keyword_f is_keyword_zig_type;
-is_keyword_f is_keyword_zig_builtin;
-is_keyword_f is_keyword_zig_constant;
-is_keyword_f is_keyword_tcl;
-is_keyword_f is_keyword_tcl_type;
-is_keyword_f is_keyword_tcl_builtin;
-is_keyword_f is_keyword_ruby;
-is_keyword_f is_keyword_ruby_builtin;
-is_keyword_f is_keyword_ruby_constant;
-is_keyword_f is_keyword_typescript;
-is_keyword_f is_keyword_typescript_type;
-is_keyword_f is_keyword_typescript_constant;
-is_keyword_f is_keyword_forth;
-is_keyword_f is_keyword_forth_def;
-is_keyword_f is_keyword_m4;
-is_keyword_f is_keyword_make;
-is_keyword_f is_keyword_make_builtin;
-is_keyword_f is_keyword_asm_prefix;
-is_keyword_f is_keyword_asm_qualifier;
-is_keyword_f is_keyword_basic;
-is_keyword_f is_keyword_basic_type;
-is_keyword_f is_keyword_basic_builtin;
-is_keyword_f is_keyword_basic_constant;
-is_keyword_f is_keyword_ld;
-is_keyword_f is_keyword_ld_builtin;
-is_keyword_f is_keyword_ld_warning;
-is_keyword_f is_keyword_matlab;
-is_keyword_f is_keyword_matlab_builtin;
-is_keyword_f is_keyword_matlab_constant;
-is_keyword_f is_keyword_r;
-is_keyword_f is_keyword_r_builtin;
-is_keyword_f is_keyword_r_constant;
-is_keyword_f is_keyword_scala;
-is_keyword_f is_keyword_julia;
-is_keyword_f is_keyword_ocaml;
-is_keyword_f is_keyword_ocaml_builtin;
-is_keyword_f is_keyword_ocaml_constant;
-is_keyword_f is_keyword_cmake;
-is_keyword_f is_keyword_css_at;
-is_keyword_f is_keyword_css_bang;
-}
-
-class Highlight {
- public:
- static Highlight *create(const std::string_view &lang);
- virtual ~Highlight() = default;
- virtual void feed(std::string *result, std::string_view input) = 0;
- virtual void flush(std::string *result) = 0;
-};
-
-class HighlightTxt : public Highlight {
- public:
- HighlightTxt();
- ~HighlightTxt() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-};
-
-class ColorBleeder : public Highlight {
- public:
- ColorBleeder(Highlight *h);
- ~ColorBleeder() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- void relay(std::string *r, const std::string &s);
- void restore(std::string *r);
- unsigned char t_ = 0;
- unsigned char x_ = 0;
- unsigned char n_ = 0;
- unsigned char inverted_ = 0;
- unsigned char intensity_ = 0;
- unsigned char foreground_ = 0;
- unsigned char background_ = 0;
- unsigned char sgr_codes_[8];
- Highlight *h_;
-};
-
-class HighlightC : public Highlight {
- public:
- HighlightC(is_keyword_f *is_keyword = is_keyword_c, is_keyword_f *is_type = nullptr,
- is_keyword_f *is_builtin = nullptr, is_keyword_f *is_constant = nullptr);
- ~HighlightC() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int i_ = 0;
- bool is_pod_ = 0;
- bool is_bol_ = true;
- bool is_cpp_ = false;
- bool is_define_ = false;
- bool is_include_ = false;
- std::string word_;
- std::string heredoc_;
- is_keyword_f *is_type_;
- is_keyword_f *is_keyword_;
- is_keyword_f *is_builtin_;
- is_keyword_f *is_constant_;
-};
-
-class HighlightD : public Highlight {
- public:
- HighlightD();
- ~HighlightD() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int depth_ = 0;
- unsigned char opener_ = 0;
- unsigned char closer_ = 0;
- std::string heredoc_;
- std::string heredoc2_;
- std::string word_;
-};
-
-class HighlightJava : public Highlight {
- public:
- HighlightJava();
- ~HighlightJava() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightGo : public Highlight {
- public:
- HighlightGo();
- ~HighlightGo() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightJs : public Highlight {
- public:
- HighlightJs();
- ~HighlightJs() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int expect_;
- int nesti_ = 0;
- std::string word_;
- unsigned char nest_[16];
-};
-
-class HighlightTypescript : public Highlight {
- public:
- HighlightTypescript();
- ~HighlightTypescript() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int expect_;
- int nesti_ = 0;
- std::string word_;
- unsigned char nest_[16];
-};
-
-class HighlightPython : public Highlight {
- public:
- HighlightPython();
- ~HighlightPython() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightMarkdown : public Highlight {
- public:
- HighlightMarkdown();
- ~HighlightMarkdown() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- bool bol_ = true;
- bool tail_ = false;
- std::string lang_;
- Highlight *highlighter_ = nullptr;
-};
-
-class HighlightRust : public Highlight {
- public:
- HighlightRust();
- ~HighlightRust() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int nest_ = 0;
- std::string word_;
-};
-
-class HighlightFortran : public Highlight {
- public:
- HighlightFortran();
- ~HighlightFortran() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int col_ = -1;
- std::string word_;
-};
-
-class HighlightCobol : public Highlight {
- public:
- HighlightCobol();
- ~HighlightCobol() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int col_ = -1;
- std::string word_;
-};
-
-class HighlightPascal : public Highlight {
- public:
- HighlightPascal();
- ~HighlightPascal() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightSql : public Highlight {
- public:
- HighlightSql();
- ~HighlightSql() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightCss : public Highlight {
- public:
- HighlightCss();
- ~HighlightCss() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightHtml : public Highlight {
- public:
- HighlightHtml();
- ~HighlightHtml() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int i_ = 0;
- std::string name_;
- std::string closer_;
- std::string pending_;
- Highlight *highlighter_ = nullptr;
-};
-
-class HighlightPhp : public Highlight {
- public:
- HighlightPhp();
- ~HighlightPhp() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightLua : public Highlight {
- public:
- HighlightLua();
- ~HighlightLua() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int level1_;
- int level2_;
- std::string word_;
-};
-
-class HighlightLisp : public Highlight {
- public:
- HighlightLisp();
- ~HighlightLisp() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- bool is_first_ = false;
- std::string symbol_;
-};
-
-class HighlightAda : public Highlight {
- public:
- HighlightAda();
- ~HighlightAda() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int t_ = 0;
- int last_ = 0;
- std::string symbol_;
-};
-
-class HighlightHaskell : public Highlight {
- public:
- HighlightHaskell();
- ~HighlightHaskell() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int level_ = 0;
- std::string symbol_;
-};
-
-class HighlightPerl : public Highlight {
- public:
- HighlightPerl();
- ~HighlightPerl() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int i_ = 0;
- int last_ = 0;
- int expect_ = 0;
- unsigned char opener_ = 0;
- unsigned char closer_ = 0;
- bool pending_heredoc_ = false;
- bool indented_heredoc_ = false;
- std::string word_;
- std::string heredoc_;
-};
-
-class HighlightShell : public Highlight {
- public:
- HighlightShell();
- ~HighlightShell() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int i_ = 0;
- int curl_ = 0;
- int last_ = 0;
- bool pending_heredoc_ = false;
- bool indented_heredoc_ = false;
- bool no_interpolation_ = false;
- std::string word_;
- std::string heredoc_;
-};
-
-class HighlightZig : public Highlight {
- public:
- HighlightZig();
- ~HighlightZig() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightTcl : public Highlight {
- public:
- HighlightTcl();
- ~HighlightTcl() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightCsharp : public Highlight {
- public:
- HighlightCsharp();
- ~HighlightCsharp() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int trips1_;
- int trips2_;
- std::string word_;
-};
-
-class HighlightRuby : public Highlight {
- public:
- HighlightRuby();
- ~HighlightRuby() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int i_ = 0;
- int level_ = 0;
- int nesti_ = 0;
- int expect_ = 0;
- unsigned char q_ = 0;
- unsigned char opener_ = 0;
- unsigned char closer_ = 0;
- bool is_definition_ = 0;
- bool pending_heredoc_ = false;
- bool indented_heredoc_ = false;
- unsigned char nest_[16];
- unsigned char levels_[16];
- unsigned char openers_[16];
- unsigned char closers_[16];
- std::string word_;
- std::string heredoc_;
-};
-
-class HighlightForth : public Highlight {
- public:
- HighlightForth();
- ~HighlightForth() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- char closer_ = 0;
- bool is_label_ = false;
- std::string word_;
-};
-
-class HighlightM4 : public Highlight {
- public:
- HighlightM4();
- ~HighlightM4() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightMake : public Highlight {
- public:
- HighlightMake();
- ~HighlightMake() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightAsm : public Highlight {
- public:
- HighlightAsm();
- ~HighlightAsm() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int col_ = 0;
- int last_ = 0;
- bool is_preprocessor_ = false;
- bool is_first_thing_on_line_ = true;
- std::string word_;
-};
-
-class HighlightBasic : public Highlight {
- public:
- HighlightBasic();
- ~HighlightBasic() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- bool is_bol_ = true;
- std::string word_;
-};
-
-class HighlightLd : public Highlight {
- public:
- HighlightLd();
- ~HighlightLd() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- bool is_bol_ = true;
- bool is_cpp_ = false;
- std::string word_;
-};
-
-class HighlightTex : public Highlight {
- public:
- HighlightTex();
- ~HighlightTex() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightKotlin : public Highlight {
- public:
- HighlightKotlin();
- ~HighlightKotlin() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int nesti_ = 0;
- std::string word_;
- unsigned char nest_[16];
-};
-
-class HighlightMatlab : public Highlight {
- public:
- HighlightMatlab();
- ~HighlightMatlab() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightR : public Highlight {
- public:
- HighlightR();
- ~HighlightR() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightSwift : public Highlight {
- public:
- HighlightSwift();
- ~HighlightSwift() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int hash1_ = 0;
- int hash2_ = 0;
- int nesti_ = 0;
- int expect_ = 0;
- std::string word_;
- unsigned char nest_[16];
- unsigned char hash_[16];
-};
-
-class HighlightScala : public Highlight {
- public:
- HighlightScala();
- ~HighlightScala() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int nesti_ = 0;
- std::string word_;
- unsigned char nest_[16];
-};
-
-class HighlightJulia : public Highlight {
- public:
- HighlightJulia();
- ~HighlightJulia() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string word_;
-};
-
-class HighlightOcaml : public Highlight {
- public:
- HighlightOcaml();
- ~HighlightOcaml() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- int nest_ = 0;
- std::string word_;
- std::string word2_;
-};
-
-class HighlightCmake : public Highlight {
- public:
- HighlightCmake();
- ~HighlightCmake() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int c_ = 0;
- int u_ = 0;
- int t_ = 0;
- int spaces_ = 0;
- std::string word_;
-};
-
-class HighlightBnf : public Highlight {
- public:
- HighlightBnf();
- ~HighlightBnf() override;
- void feed(std::string *result, std::string_view input) override;
- void flush(std::string *result) override;
-
- private:
- int t_ = 0;
- std::string operator_;
-};
diff --git a/llamafile/highlight/.clang-format b/llamafile/highlight/.clang-format
new file mode 100644
index 00000000..d1ee89c9
--- /dev/null
+++ b/llamafile/highlight/.clang-format
@@ -0,0 +1,14 @@
+---
+BasedOnStyle: Mozilla
+IndentWidth: 4
+ColumnLimit: 80
+---
+Language: Cpp
+AllowShortFunctionsOnASingleLine: false
+AlignTrailingComments: false
+AlignEscapedNewlines: DontAlign
+AlwaysBreakTemplateDeclarations: true
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+FixNamespaceComments: true
+IndentCaseLabels: false
+---
diff --git a/llamafile/highlight/BUILD.mk b/llamafile/highlight/BUILD.mk
new file mode 100644
index 00000000..e37b604c
--- /dev/null
+++ b/llamafile/highlight/BUILD.mk
@@ -0,0 +1,54 @@
+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
+#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
+
+PKGS += LLAMAFILE_HIGHLIGHT
+
+LLAMAFILE_HIGHLIGHT_FILES := $(wildcard llamafile/highlight/*)
+LLAMAFILE_HIGHLIGHT_HDRS = $(filter %.h,$(LLAMAFILE_HIGHLIGHT_FILES))
+LLAMAFILE_HIGHLIGHT_INCS = $(filter %.inc,$(LLAMAFILE_HIGHLIGHT_FILES))
+LLAMAFILE_HIGHLIGHT_SRCS_C = $(filter %.c,$(LLAMAFILE_HIGHLIGHT_FILES))
+LLAMAFILE_HIGHLIGHT_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_HIGHLIGHT_FILES))
+LLAMAFILE_HIGHLIGHT_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_HIGHLIGHT_FILES))
+LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C = $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)
+
+LLAMAFILE_HIGHLIGHT_SRCS = \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_C) \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_CPP) \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF) \
+
+LLAMAFILE_HIGHLIGHT_OBJS = \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_C:%.c=o/$(MODE)/%.o) \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C:%.c=%.o) \
+
+o/$(MODE)/llamafile/highlight/highlight.a: $(LLAMAFILE_HIGHLIGHT_OBJS)
+
+$(LLAMAFILE_HIGHLIGHT_OBJS): llamafile/highlight/BUILD.mk
+
+o/$(MODE)/llamafile/highlight/highlight_test: \
+ o/$(MODE)/llamafile/highlight/highlight_test.o \
+ o/$(MODE)/llamafile/highlight/highlight.a \
+
+o/$(MODE)/llamafile/highlight/highlight_c_test: \
+ o/$(MODE)/llamafile/highlight/highlight_c_test.o \
+ o/$(MODE)/llamafile/highlight/highlight_c.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_c.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_c_constant.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_c_type.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_c_pod.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_cpp.o \
+
+o/$(MODE)/llamafile/highlight/highlight_python_test: \
+ o/$(MODE)/llamafile/highlight/highlight_python_test.o \
+ o/$(MODE)/llamafile/highlight/highlight_python.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_python.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_python_builtin.o \
+ o/$(MODE)/llamafile/highlight/is_keyword_python_constant.o \
+
+.PHONY: o/$(MODE)/llamafile/highlight
+o/$(MODE)/llamafile/highlight: \
+ $(LLAMAFILE_HIGHLIGHT_SRCS_GPERF_C) \
+ o/$(MODE)/llamafile/highlight/highlight.a \
+ o/$(MODE)/llamafile/highlight/highlight_c_test.runs \
+ o/$(MODE)/llamafile/highlight/highlight_python_test.runs \
+ o/$(MODE)/llamafile/highlight/highlight_test.runs \
diff --git a/llamafile/color_bleeder.cpp b/llamafile/highlight/color_bleeder.cpp
similarity index 94%
rename from llamafile/color_bleeder.cpp
rename to llamafile/highlight/color_bleeder.cpp
index 7cc33b6c..e338fcc8 100644
--- a/llamafile/color_bleeder.cpp
+++ b/llamafile/highlight/color_bleeder.cpp
@@ -37,19 +37,24 @@
//
// only the subset of ansi escape codes that we use are supported.
-enum {
+enum
+{
NORMAL,
ESC,
CSI,
};
-ColorBleeder::ColorBleeder(Highlight *h) : h_(h) {
+ColorBleeder::ColorBleeder(Highlight* h) : h_(h)
+{
}
-ColorBleeder::~ColorBleeder() {
+ColorBleeder::~ColorBleeder()
+{
}
-void ColorBleeder::restore(std::string *r) {
+void
+ColorBleeder::restore(std::string* r)
+{
bool got_some = false;
if (!intensity_ && !inverted_ && !foreground_ && !background_)
return;
@@ -78,7 +83,9 @@ void ColorBleeder::restore(std::string *r) {
*r += 'm';
}
-void ColorBleeder::relay(std::string *r, const std::string &s) {
+void
+ColorBleeder::relay(std::string* r, const std::string& s)
+{
for (char c : s) {
*r += c;
switch (t_) {
@@ -171,13 +178,17 @@ void ColorBleeder::relay(std::string *r, const std::string &s) {
}
}
-void ColorBleeder::feed(std::string *r, std::string_view input) {
+void
+ColorBleeder::feed(std::string* r, std::string_view input)
+{
std::string s;
h_->feed(&s, input);
relay(r, s);
}
-void ColorBleeder::flush(std::string *r) {
+void
+ColorBleeder::flush(std::string* r)
+{
std::string s;
h_->flush(&s);
relay(r, s);
diff --git a/llamafile/highlight.cpp b/llamafile/highlight/highlight.cpp
similarity index 50%
rename from llamafile/highlight.cpp
rename to llamafile/highlight/highlight.cpp
index c4d3b85e..c0e3de60 100644
--- a/llamafile/highlight.cpp
+++ b/llamafile/highlight/highlight.cpp
@@ -17,197 +17,289 @@
#include "highlight.h"
-struct HighlightEntry {
- char *name;
- Highlight *(*ctor)(void);
+struct HighlightEntry
+{
+ char* name;
+ Highlight* (*ctor)(void);
};
-extern "C" Highlight *highlight_create_ada_(void) {
+extern "C" Highlight*
+highlight_create_ada_(void)
+{
return new HighlightAda;
}
-extern "C" Highlight *highlight_create_asm_(void) {
+extern "C" Highlight*
+highlight_create_asm_(void)
+{
return new HighlightAsm;
}
-extern "C" Highlight *highlight_create_basic_(void) {
+extern "C" Highlight*
+highlight_create_basic_(void)
+{
return new HighlightBasic;
}
-extern "C" Highlight *highlight_create_bnf_(void) {
+extern "C" Highlight*
+highlight_create_bnf_(void)
+{
return new HighlightBnf;
}
-extern "C" Highlight *highlight_create_c_(void) {
+extern "C" Highlight*
+highlight_create_c_(void)
+{
return new HighlightC(is_keyword_c, //
is_keyword_c_type, //
is_keyword_c_builtin, //
is_keyword_c_constant);
}
-extern "C" Highlight *highlight_create_cxx_(void) {
+extern "C" Highlight*
+highlight_create_cxx_(void)
+{
return new HighlightC(is_keyword_cxx, //
is_keyword_c_type, //
is_keyword_c_builtin, //
is_keyword_c_constant);
}
-extern "C" Highlight *highlight_create_cmake_(void) {
+extern "C" Highlight*
+highlight_create_cmake_(void)
+{
return new HighlightCmake;
}
-extern "C" Highlight *highlight_create_cobol_(void) {
+extern "C" Highlight*
+highlight_create_cobol_(void)
+{
return new HighlightCobol;
}
-extern "C" Highlight *highlight_create_csharp_(void) {
+extern "C" Highlight*
+highlight_create_csharp_(void)
+{
return new HighlightCsharp;
}
-extern "C" Highlight *highlight_create_css_(void) {
+extern "C" Highlight*
+highlight_create_css_(void)
+{
return new HighlightCss;
}
-extern "C" Highlight *highlight_create_d_(void) {
+extern "C" Highlight*
+highlight_create_d_(void)
+{
return new HighlightD;
}
-extern "C" Highlight *highlight_create_forth_(void) {
+extern "C" Highlight*
+highlight_create_forth_(void)
+{
return new HighlightForth;
}
-extern "C" Highlight *highlight_create_fortran_(void) {
+extern "C" Highlight*
+highlight_create_fortran_(void)
+{
return new HighlightFortran;
}
-extern "C" Highlight *highlight_create_go_(void) {
+extern "C" Highlight*
+highlight_create_go_(void)
+{
return new HighlightGo;
}
-extern "C" Highlight *highlight_create_haskell_(void) {
+extern "C" Highlight*
+highlight_create_haskell_(void)
+{
return new HighlightHaskell;
}
-extern "C" Highlight *highlight_create_html_(void) {
+extern "C" Highlight*
+highlight_create_html_(void)
+{
return new HighlightHtml;
}
-extern "C" Highlight *highlight_create_java_(void) {
+extern "C" Highlight*
+highlight_create_java_(void)
+{
return new HighlightJava;
}
-extern "C" Highlight *highlight_create_js_(void) {
+extern "C" Highlight*
+highlight_create_js_(void)
+{
return new HighlightJs;
}
-extern "C" Highlight *highlight_create_julia_(void) {
+extern "C" Highlight*
+highlight_create_julia_(void)
+{
return new HighlightJulia;
}
-extern "C" Highlight *highlight_create_kotlin_(void) {
+extern "C" Highlight*
+highlight_create_kotlin_(void)
+{
return new HighlightKotlin;
}
-extern "C" Highlight *highlight_create_ld_(void) {
+extern "C" Highlight*
+highlight_create_ld_(void)
+{
return new HighlightLd;
}
-extern "C" Highlight *highlight_create_lisp_(void) {
+extern "C" Highlight*
+highlight_create_lisp_(void)
+{
return new HighlightLisp;
}
-extern "C" Highlight *highlight_create_lua_(void) {
+extern "C" Highlight*
+highlight_create_lua_(void)
+{
return new HighlightLua;
}
-extern "C" Highlight *highlight_create_m4_(void) {
+extern "C" Highlight*
+highlight_create_m4_(void)
+{
return new HighlightM4;
}
-extern "C" Highlight *highlight_create_make_(void) {
+extern "C" Highlight*
+highlight_create_make_(void)
+{
return new HighlightMake;
}
-extern "C" Highlight *highlight_create_markdown_(void) {
+extern "C" Highlight*
+highlight_create_markdown_(void)
+{
return new HighlightMarkdown;
}
-extern "C" Highlight *highlight_create_matlab_(void) {
+extern "C" Highlight*
+highlight_create_matlab_(void)
+{
return new HighlightMatlab;
}
-extern "C" Highlight *highlight_create_ocaml_(void) {
+extern "C" Highlight*
+highlight_create_ocaml_(void)
+{
return new HighlightOcaml;
}
-extern "C" Highlight *highlight_create_pascal_(void) {
+extern "C" Highlight*
+highlight_create_pascal_(void)
+{
return new HighlightPascal;
}
-extern "C" Highlight *highlight_create_perl_(void) {
+extern "C" Highlight*
+highlight_create_perl_(void)
+{
return new HighlightPerl;
}
-extern "C" Highlight *highlight_create_php_(void) {
+extern "C" Highlight*
+highlight_create_php_(void)
+{
return new HighlightPhp;
}
-extern "C" Highlight *highlight_create_python_(void) {
+extern "C" Highlight*
+highlight_create_python_(void)
+{
return new HighlightPython;
}
-extern "C" Highlight *highlight_create_r_(void) {
+extern "C" Highlight*
+highlight_create_r_(void)
+{
return new HighlightR;
}
-extern "C" Highlight *highlight_create_ruby_(void) {
+extern "C" Highlight*
+highlight_create_ruby_(void)
+{
return new HighlightRuby;
}
-extern "C" Highlight *highlight_create_rust_(void) {
+extern "C" Highlight*
+highlight_create_rust_(void)
+{
return new HighlightRust;
}
-extern "C" Highlight *highlight_create_scala_(void) {
+extern "C" Highlight*
+highlight_create_scala_(void)
+{
return new HighlightScala;
}
-extern "C" Highlight *highlight_create_shell_(void) {
+extern "C" Highlight*
+highlight_create_shell_(void)
+{
return new HighlightShell;
}
-extern "C" Highlight *highlight_create_sql_(void) {
+extern "C" Highlight*
+highlight_create_sql_(void)
+{
return new HighlightSql;
}
-extern "C" Highlight *highlight_create_swift_(void) {
+extern "C" Highlight*
+highlight_create_swift_(void)
+{
return new HighlightSwift;
}
-extern "C" Highlight *highlight_create_tcl_(void) {
+extern "C" Highlight*
+highlight_create_tcl_(void)
+{
return new HighlightTcl;
}
-extern "C" Highlight *highlight_create_tex_(void) {
+extern "C" Highlight*
+highlight_create_tex_(void)
+{
return new HighlightTex;
}
-extern "C" Highlight *highlight_create_txt_(void) {
+extern "C" Highlight*
+highlight_create_txt_(void)
+{
return new HighlightTxt;
}
-extern "C" Highlight *highlight_create_typescript_(void) {
+extern "C" Highlight*
+highlight_create_typescript_(void)
+{
return new HighlightTypescript;
}
-extern "C" Highlight *highlight_create_zig_(void) {
+extern "C" Highlight*
+highlight_create_zig_(void)
+{
return new HighlightZig;
}
-extern "C" const HighlightEntry *highlight_lookup_(const char *str, size_t len);
+extern "C" const HighlightEntry*
+highlight_lookup_(const char* str, size_t len);
-Highlight *Highlight::create(const std::string_view &lang) {
- const HighlightEntry *slot;
+Highlight*
+Highlight::create(const std::string_view& lang)
+{
+ const HighlightEntry* slot;
if ((slot = highlight_lookup_(lang.data(), lang.size())))
return slot->ctor();
return nullptr;
diff --git a/llamafile/highlight/highlight.h b/llamafile/highlight/highlight.h
new file mode 100644
index 00000000..1cc34cda
--- /dev/null
+++ b/llamafile/highlight/highlight.h
@@ -0,0 +1,862 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <string>
+#include <string_view>
+
+#define HI_RESET "\033[0m"
+#define HI_BOLD "\033[1m"
+#define HI_UNBOLD "\033[22m"
+#define HI_ITALIC HI_INVERT
+#define HI_INVERT "\033[7m"
+#define HI_KEYWORD "\033[1;34m" // bold blue
+#define HI_STRING "\033[32m" // green
+#define HI_COMMENT "\033[31m" // red
+#define HI_VAR "\033[1;35m" // magenta
+#define HI_MACRO "\033[35m" // magenta
+#define HI_ATTRIB "\033[35m" // magenta
+#define HI_LINENO "\033[2m" // fade
+#define HI_CONTIN "\033[33m" // yellow
+#define HI_LABEL "\033[33m" // yellow
+#define HI_DEF "\033[2;33m" // dim yellow
+#define HI_TYPE "\033[36m" // cyan
+#define HI_CLASS "\033[1;36m" // bold cyan
+#define HI_SELECTOR "\033[33m" // yellow
+#define HI_PROPERTY "\033[36m" // cyan
+#define HI_TAG "\033[33m" // yellow
+#define HI_INCODE "\033[1;35m" // magenta
+#define HI_BUILTIN "\033[35m" // magenta
+#define HI_CONSTANT "\033[1;35m" // bold magenta
+#define HI_LISPKW "\033[35m" // magenta
+#define HI_ENTITY "\033[36m" // cyan
+#define HI_OPERATOR "\033[36m" // cyan
+#define HI_ESCAPE "\033[33m" // yellow
+#define HI_QUALIFIER "\033[35m" // magenta
+#define HI_IMMEDIATE "\033[36m" // cyan
+#define HI_REGISTER "\033[1;35m" // bold magenta
+#define HI_DIRECTIVE "\033[1;31m" // bold red
+#define HI_WARNING "\033[1;31m" // bold red
+#define HI_MATH "\033[1;35m" // bold magenta
+
+typedef const char*
+is_keyword_f(const char*, size_t);
+
+extern "C"
+{
+ is_keyword_f is_keyword_c;
+ is_keyword_f is_keyword_c_pod;
+ is_keyword_f is_keyword_c_type;
+ is_keyword_f is_keyword_c_builtin;
+ is_keyword_f is_keyword_c_constant;
+ is_keyword_f is_keyword_cpp;
+ is_keyword_f is_keyword_cxx;
+ is_keyword_f is_keyword_js;
+ is_keyword_f is_keyword_js_builtin;
+ is_keyword_f is_keyword_js_constant;
+ is_keyword_f is_keyword_java;
+ is_keyword_f is_keyword_java_constant;
+ is_keyword_f is_keyword_python;
+ is_keyword_f is_keyword_python_builtin;
+ is_keyword_f is_keyword_python_constant;
+ is_keyword_f is_keyword_rust;
+ is_keyword_f is_keyword_rust_type;
+ is_keyword_f is_keyword_rust_constant;
+ is_keyword_f is_keyword_fortran;
+ is_keyword_f is_keyword_fortran_type;
+ is_keyword_f is_keyword_fortran_builtin;
+ is_keyword_f is_keyword_cobol;
+ is_keyword_f is_keyword_pascal;
+ is_keyword_f is_keyword_pascal_type;
+ is_keyword_f is_keyword_pascal_builtin;
+ is_keyword_f is_keyword_go;
+ is_keyword_f is_keyword_go_type;
+ is_keyword_f is_keyword_sql;
+ is_keyword_f is_keyword_sql_type;
+ is_keyword_f is_keyword_php;
+ is_keyword_f is_keyword_php_constant;
+ is_keyword_f is_keyword_csharp;
+ is_keyword_f is_keyword_csharp_constant;
+ is_keyword_f is_keyword_kotlin;
+ is_keyword_f is_keyword_kotlin_type;
+ is_keyword_f is_keyword_lua;
+ is_keyword_f is_keyword_lua_builtin;
+ is_keyword_f is_keyword_lua_constant;
+ is_keyword_f is_keyword_lisp;
+ is_keyword_f is_keyword_ada;
+ is_keyword_f is_keyword_ada_constant;
+ is_keyword_f is_keyword_haskell;
+ is_keyword_f is_keyword_perl;
+ is_keyword_f is_keyword_shell;
+ is_keyword_f is_keyword_shell_builtin;
+ is_keyword_f is_keyword_swift;
+ is_keyword_f is_keyword_swift_type;
+ is_keyword_f is_keyword_swift_builtin;
+ is_keyword_f is_keyword_swift_constant;
+ is_keyword_f is_keyword_d;
+ is_keyword_f is_keyword_d_constant;
+ is_keyword_f is_keyword_zig;
+ is_keyword_f is_keyword_zig_type;
+ is_keyword_f is_keyword_zig_builtin;
+ is_keyword_f is_keyword_zig_constant;
+ is_keyword_f is_keyword_tcl;
+ is_keyword_f is_keyword_tcl_type;
+ is_keyword_f is_keyword_tcl_builtin;
+ is_keyword_f is_keyword_ruby;
+ is_keyword_f is_keyword_ruby_builtin;
+ is_keyword_f is_keyword_ruby_constant;
+ is_keyword_f is_keyword_typescript;
+ is_keyword_f is_keyword_typescript_type;
+ is_keyword_f is_keyword_typescript_constant;
+ is_keyword_f is_keyword_forth;
+ is_keyword_f is_keyword_forth_def;
+ is_keyword_f is_keyword_m4;
+ is_keyword_f is_keyword_make;
+ is_keyword_f is_keyword_make_builtin;
+ is_keyword_f is_keyword_asm_prefix;
+ is_keyword_f is_keyword_asm_qualifier;
+ is_keyword_f is_keyword_basic;
+ is_keyword_f is_keyword_basic_type;
+ is_keyword_f is_keyword_basic_builtin;
+ is_keyword_f is_keyword_basic_constant;
+ is_keyword_f is_keyword_ld;
+ is_keyword_f is_keyword_ld_builtin;
+ is_keyword_f is_keyword_ld_warning;
+ is_keyword_f is_keyword_matlab;
+ is_keyword_f is_keyword_matlab_builtin;
+ is_keyword_f is_keyword_matlab_constant;
+ is_keyword_f is_keyword_r;
+ is_keyword_f is_keyword_r_builtin;
+ is_keyword_f is_keyword_r_constant;
+ is_keyword_f is_keyword_scala;
+ is_keyword_f is_keyword_julia;
+ is_keyword_f is_keyword_ocaml;
+ is_keyword_f is_keyword_ocaml_builtin;
+ is_keyword_f is_keyword_ocaml_constant;
+ is_keyword_f is_keyword_cmake;
+ is_keyword_f is_keyword_css_at;
+ is_keyword_f is_keyword_css_bang;
+}
+
+class Highlight
+{
+ public:
+ static Highlight* create(const std::string_view& lang);
+ virtual ~Highlight() = default;
+ virtual void feed(std::string* result, std::string_view input) = 0;
+ virtual void flush(std::string* result) = 0;
+};
+
+class HighlightTxt : public Highlight
+{
+ public:
+ HighlightTxt();
+ ~HighlightTxt() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+};
+
+class ColorBleeder : public Highlight
+{
+ public:
+ ColorBleeder(Highlight* h);
+ ~ColorBleeder() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ void relay(std::string* r, const std::string& s);
+ void restore(std::string* r);
+ unsigned char t_ = 0;
+ unsigned char x_ = 0;
+ unsigned char n_ = 0;
+ unsigned char inverted_ = 0;
+ unsigned char intensity_ = 0;
+ unsigned char foreground_ = 0;
+ unsigned char background_ = 0;
+ unsigned char sgr_codes_[8];
+ Highlight* h_;
+};
+
+class HighlightC : public Highlight
+{
+ public:
+ HighlightC(is_keyword_f* is_keyword = is_keyword_c,
+ is_keyword_f* is_type = nullptr,
+ is_keyword_f* is_builtin = nullptr,
+ is_keyword_f* is_constant = nullptr);
+ ~HighlightC() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int i_ = 0;
+ bool is_pod_ = 0;
+ bool is_bol_ = true;
+ bool is_cpp_ = false;
+ bool is_define_ = false;
+ bool is_include_ = false;
+ std::string word_;
+ std::string heredoc_;
+ is_keyword_f* is_type_;
+ is_keyword_f* is_keyword_;
+ is_keyword_f* is_builtin_;
+ is_keyword_f* is_constant_;
+};
+
+class HighlightD : public Highlight
+{
+ public:
+ HighlightD();
+ ~HighlightD() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int depth_ = 0;
+ unsigned char opener_ = 0;
+ unsigned char closer_ = 0;
+ std::string heredoc_;
+ std::string heredoc2_;
+ std::string word_;
+};
+
+class HighlightJava : public Highlight
+{
+ public:
+ HighlightJava();
+ ~HighlightJava() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightGo : public Highlight
+{
+ public:
+ HighlightGo();
+ ~HighlightGo() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightJs : public Highlight
+{
+ public:
+ HighlightJs();
+ ~HighlightJs() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int expect_;
+ int nesti_ = 0;
+ std::string word_;
+ unsigned char nest_[16];
+};
+
+class HighlightTypescript : public Highlight
+{
+ public:
+ HighlightTypescript();
+ ~HighlightTypescript() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int expect_;
+ int nesti_ = 0;
+ std::string word_;
+ unsigned char nest_[16];
+};
+
+class HighlightPython : public Highlight
+{
+ public:
+ HighlightPython();
+ ~HighlightPython() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightMarkdown : public Highlight
+{
+ public:
+ HighlightMarkdown();
+ ~HighlightMarkdown() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ bool bol_ = true;
+ bool tail_ = false;
+ std::string lang_;
+ Highlight* highlighter_ = nullptr;
+};
+
+class HighlightRust : public Highlight
+{
+ public:
+ HighlightRust();
+ ~HighlightRust() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int nest_ = 0;
+ std::string word_;
+};
+
+class HighlightFortran : public Highlight
+{
+ public:
+ HighlightFortran();
+ ~HighlightFortran() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int col_ = -1;
+ std::string word_;
+};
+
+class HighlightCobol : public Highlight
+{
+ public:
+ HighlightCobol();
+ ~HighlightCobol() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int col_ = -1;
+ std::string word_;
+};
+
+class HighlightPascal : public Highlight
+{
+ public:
+ HighlightPascal();
+ ~HighlightPascal() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightSql : public Highlight
+{
+ public:
+ HighlightSql();
+ ~HighlightSql() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightCss : public Highlight
+{
+ public:
+ HighlightCss();
+ ~HighlightCss() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightHtml : public Highlight
+{
+ public:
+ HighlightHtml();
+ ~HighlightHtml() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int i_ = 0;
+ std::string name_;
+ std::string closer_;
+ std::string pending_;
+ Highlight* highlighter_ = nullptr;
+};
+
+class HighlightPhp : public Highlight
+{
+ public:
+ HighlightPhp();
+ ~HighlightPhp() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightLua : public Highlight
+{
+ public:
+ HighlightLua();
+ ~HighlightLua() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int level1_;
+ int level2_;
+ std::string word_;
+};
+
+class HighlightLisp : public Highlight
+{
+ public:
+ HighlightLisp();
+ ~HighlightLisp() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ bool is_first_ = false;
+ std::string symbol_;
+};
+
+class HighlightAda : public Highlight
+{
+ public:
+ HighlightAda();
+ ~HighlightAda() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int t_ = 0;
+ int last_ = 0;
+ std::string symbol_;
+};
+
+class HighlightHaskell : public Highlight
+{
+ public:
+ HighlightHaskell();
+ ~HighlightHaskell() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int level_ = 0;
+ std::string symbol_;
+};
+
+class HighlightPerl : public Highlight
+{
+ public:
+ HighlightPerl();
+ ~HighlightPerl() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int i_ = 0;
+ int last_ = 0;
+ int expect_ = 0;
+ unsigned char opener_ = 0;
+ unsigned char closer_ = 0;
+ bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
+ std::string word_;
+ std::string heredoc_;
+};
+
+class HighlightShell : public Highlight
+{
+ public:
+ HighlightShell();
+ ~HighlightShell() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int i_ = 0;
+ int curl_ = 0;
+ int last_ = 0;
+ bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
+ bool no_interpolation_ = false;
+ std::string word_;
+ std::string heredoc_;
+};
+
+class HighlightZig : public Highlight
+{
+ public:
+ HighlightZig();
+ ~HighlightZig() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightTcl : public Highlight
+{
+ public:
+ HighlightTcl();
+ ~HighlightTcl() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightCsharp : public Highlight
+{
+ public:
+ HighlightCsharp();
+ ~HighlightCsharp() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int trips1_;
+ int trips2_;
+ std::string word_;
+};
+
+class HighlightRuby : public Highlight
+{
+ public:
+ HighlightRuby();
+ ~HighlightRuby() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int i_ = 0;
+ int level_ = 0;
+ int nesti_ = 0;
+ int expect_ = 0;
+ unsigned char q_ = 0;
+ unsigned char opener_ = 0;
+ unsigned char closer_ = 0;
+ bool is_definition_ = 0;
+ bool pending_heredoc_ = false;
+ bool indented_heredoc_ = false;
+ unsigned char nest_[16];
+ unsigned char levels_[16];
+ unsigned char openers_[16];
+ unsigned char closers_[16];
+ std::string word_;
+ std::string heredoc_;
+};
+
+class HighlightForth : public Highlight
+{
+ public:
+ HighlightForth();
+ ~HighlightForth() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ char closer_ = 0;
+ bool is_label_ = false;
+ std::string word_;
+};
+
+class HighlightM4 : public Highlight
+{
+ public:
+ HighlightM4();
+ ~HighlightM4() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightMake : public Highlight
+{
+ public:
+ HighlightMake();
+ ~HighlightMake() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightAsm : public Highlight
+{
+ public:
+ HighlightAsm();
+ ~HighlightAsm() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int col_ = 0;
+ int last_ = 0;
+ bool is_preprocessor_ = false;
+ bool is_first_thing_on_line_ = true;
+ std::string word_;
+};
+
+class HighlightBasic : public Highlight
+{
+ public:
+ HighlightBasic();
+ ~HighlightBasic() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ bool is_bol_ = true;
+ std::string word_;
+};
+
+class HighlightLd : public Highlight
+{
+ public:
+ HighlightLd();
+ ~HighlightLd() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ bool is_bol_ = true;
+ bool is_cpp_ = false;
+ std::string word_;
+};
+
+class HighlightTex : public Highlight
+{
+ public:
+ HighlightTex();
+ ~HighlightTex() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightKotlin : public Highlight
+{
+ public:
+ HighlightKotlin();
+ ~HighlightKotlin() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int nesti_ = 0;
+ std::string word_;
+ unsigned char nest_[16];
+};
+
+class HighlightMatlab : public Highlight
+{
+ public:
+ HighlightMatlab();
+ ~HighlightMatlab() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightR : public Highlight
+{
+ public:
+ HighlightR();
+ ~HighlightR() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightSwift : public Highlight
+{
+ public:
+ HighlightSwift();
+ ~HighlightSwift() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int hash1_ = 0;
+ int hash2_ = 0;
+ int nesti_ = 0;
+ int expect_ = 0;
+ std::string word_;
+ unsigned char nest_[16];
+ unsigned char hash_[16];
+};
+
+class HighlightScala : public Highlight
+{
+ public:
+ HighlightScala();
+ ~HighlightScala() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int nesti_ = 0;
+ std::string word_;
+ unsigned char nest_[16];
+};
+
+class HighlightJulia : public Highlight
+{
+ public:
+ HighlightJulia();
+ ~HighlightJulia() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string word_;
+};
+
+class HighlightOcaml : public Highlight
+{
+ public:
+ HighlightOcaml();
+ ~HighlightOcaml() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ int nest_ = 0;
+ std::string word_;
+ std::string word2_;
+};
+
+class HighlightCmake : public Highlight
+{
+ public:
+ HighlightCmake();
+ ~HighlightCmake() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int c_ = 0;
+ int u_ = 0;
+ int t_ = 0;
+ int spaces_ = 0;
+ std::string word_;
+};
+
+class HighlightBnf : public Highlight
+{
+ public:
+ HighlightBnf();
+ ~HighlightBnf() override;
+ void feed(std::string* result, std::string_view input) override;
+ void flush(std::string* result) override;
+
+ private:
+ int t_ = 0;
+ std::string operator_;
+};
diff --git a/llamafile/highlight_ada.cpp b/llamafile/highlight/highlight_ada.cpp
similarity index 93%
rename from llamafile/highlight_ada.cpp
rename to llamafile/highlight/highlight_ada.cpp
index 581ec7ca..928fcf50 100644
--- a/llamafile/highlight_ada.cpp
+++ b/llamafile/highlight/highlight_ada.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
SYMBOL,
QUOTE,
@@ -28,13 +29,17 @@ enum {
COMMENT,
};
-HighlightAda::HighlightAda() {
+HighlightAda::HighlightAda()
+{
}
-HighlightAda::~HighlightAda() {
+HighlightAda::~HighlightAda()
+{
}
-void HighlightAda::feed(std::string *r, std::string_view input) {
+void
+HighlightAda::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
last_ = c_;
@@ -71,7 +76,8 @@ void HighlightAda::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += symbol_;
*r += HI_RESET;
- } else if (is_keyword_ada_constant(symbol_.data(), symbol_.size())) {
+ } else if (is_keyword_ada_constant(symbol_.data(),
+ symbol_.size())) {
*r += HI_CONSTANT;
*r += symbol_;
*r += HI_RESET;
@@ -126,7 +132,9 @@ void HighlightAda::feed(std::string *r, std::string_view input) {
}
}
-void HighlightAda::flush(std::string *r) {
+void
+HighlightAda::flush(std::string* r)
+{
switch (t_) {
case SYMBOL:
if (is_keyword_ada(symbol_.data(), symbol_.size())) {
diff --git a/llamafile/highlight_asm.cpp b/llamafile/highlight/highlight_asm.cpp
similarity index 95%
rename from llamafile/highlight_asm.cpp
rename to llamafile/highlight/highlight_asm.cpp
index ee520700..286e2684 100644
--- a/llamafile/highlight_asm.cpp
+++ b/llamafile/highlight/highlight_asm.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
@@ -32,7 +32,8 @@
//
// doing that requires special care
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
@@ -57,7 +58,9 @@ enum {
HASH,
};
-static bool is_immediate(int c) {
+static bool
+is_immediate(int c)
+{
switch (c) {
case '&':
case '*':
@@ -78,13 +81,17 @@ static bool is_immediate(int c) {
}
}
-HighlightAsm::HighlightAsm() {
+HighlightAsm::HighlightAsm()
+{
}
-HighlightAsm::~HighlightAsm() {
+HighlightAsm::~HighlightAsm()
+{
}
-void HighlightAsm::feed(std::string *r, std::string_view input) {
+void
+HighlightAsm::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -240,7 +247,8 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {
// fallthrough
case WORD:
- if (!isascii(c) || isalnum(c) || c == '$' || c == '_' || c == '-' || c == '.') {
+ if (!isascii(c) || isalnum(c) || c == '$' || c == '_' || c == '-' ||
+ c == '.') {
lf::append_wchar(&word_, c);
} else {
if (is_first_thing_on_line_) {
@@ -255,7 +263,8 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {
*r += HI_LABEL;
*r += word_;
*r += HI_RESET;
- } else if (word_ == "C" || word_ == "dnl" || word_ == "m4_dnl") {
+ } else if (word_ == "C" || word_ == "dnl" ||
+ word_ == "m4_dnl") {
*r += HI_COMMENT;
*r += word_;
word_.clear();
@@ -268,11 +277,13 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {
if (!is_keyword_asm_prefix(word_.data(), word_.size()))
is_first_thing_on_line_ = false;
}
- } else if (is_preprocessor_ && is_keyword_c_builtin(word_.data(), word_.size())) {
+ } else if (is_preprocessor_ &&
+ is_keyword_c_builtin(word_.data(), word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_asm_qualifier(word_.data(), word_.size())) {
+ } else if (is_keyword_asm_qualifier(word_.data(),
+ word_.size())) {
*r += HI_QUALIFIER;
*r += word_;
*r += HI_RESET;
@@ -406,7 +417,9 @@ void HighlightAsm::feed(std::string *r, std::string_view input) {
}
}
-void HighlightAsm::flush(std::string *r) {
+void
+HighlightAsm::flush(std::string* r)
+{
switch (t_) {
case WORD:
*r += word_;
diff --git a/llamafile/highlight_basic.cpp b/llamafile/highlight/highlight_basic.cpp
similarity index 89%
rename from llamafile/highlight_basic.cpp
rename to llamafile/highlight/highlight_basic.cpp
index 830ac824..acd1b6d4 100644
--- a/llamafile/highlight_basic.cpp
+++ b/llamafile/highlight/highlight_basic.cpp
@@ -16,8 +16,7 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
-#include <ctype.h>
+#include <cctype>
// beginner's all-purpose symbolic instruction code
//
@@ -29,7 +28,8 @@
//
// doing that requires extra keywords
-enum {
+enum
+{
NORMAL,
WORD,
LINENO,
@@ -38,13 +38,26 @@ enum {
DIRECTIVE,
};
-HighlightBasic::HighlightBasic() {
+static std::string
+ToLower(const std::string_view& s)
+{
+ std::string b;
+ for (char c : s)
+ b += std::tolower(c);
+ return b;
}
-HighlightBasic::~HighlightBasic() {
+HighlightBasic::HighlightBasic()
+{
}
-void HighlightBasic::feed(std::string *r, std::string_view input) {
+HighlightBasic::~HighlightBasic()
+{
+}
+
+void
+HighlightBasic::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -81,7 +94,7 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {
word_ += c;
} else {
if (is_keyword_basic(word_.data(), word_.size())) {
- if (lf::tolower(word_) == "rem") {
+ if (ToLower(word_) == "rem") {
*r += HI_COMMENT;
*r += word_;
t_ = COMMENT;
@@ -96,11 +109,13 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_basic_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_basic_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_basic_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_basic_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -155,11 +170,13 @@ void HighlightBasic::feed(std::string *r, std::string_view input) {
}
}
-void HighlightBasic::flush(std::string *r) {
+void
+HighlightBasic::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_basic(word_.data(), word_.size())) {
- if (lf::tolower(word_) == "rem") {
+ if (ToLower(word_) == "rem") {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
diff --git a/llamafile/highlight_bnf.cpp b/llamafile/highlight/highlight_bnf.cpp
similarity index 94%
rename from llamafile/highlight_bnf.cpp
rename to llamafile/highlight/highlight_bnf.cpp
index ab7f0678..79d3ab3c 100644
--- a/llamafile/highlight_bnf.cpp
+++ b/llamafile/highlight/highlight_bnf.cpp
@@ -17,7 +17,8 @@
#include "highlight.h"
-enum {
+enum
+{
NORMAL,
COMMENT,
DQUOTE,
@@ -28,7 +29,9 @@ enum {
OPERATOR,
};
-static bool is_operator(const std::string &op) {
+static bool
+is_operator(const std::string& op)
+{
return op == "::=" || //
op == "|" || //
op == "?" || //
@@ -36,7 +39,9 @@ static bool is_operator(const std::string &op) {
op == "+";
}
-static bool is_operator_char(int c) {
+static bool
+is_operator_char(int c)
+{
switch (c) {
case '!':
case '$':
@@ -64,13 +69,17 @@ static bool is_operator_char(int c) {
}
}
-HighlightBnf::HighlightBnf() {
+HighlightBnf::HighlightBnf()
+{
}
-HighlightBnf::~HighlightBnf() {
+HighlightBnf::~HighlightBnf()
+{
}
-void HighlightBnf::feed(std::string *r, std::string_view input) {
+void
+HighlightBnf::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -178,7 +187,9 @@ void HighlightBnf::feed(std::string *r, std::string_view input) {
}
}
-void HighlightBnf::flush(std::string *r) {
+void
+HighlightBnf::flush(std::string* r)
+{
switch (t_) {
case OPERATOR:
if (is_operator(operator_)) {
diff --git a/llamafile/highlight_c.cpp b/llamafile/highlight/highlight_c.cpp
similarity index 90%
rename from llamafile/highlight_c.cpp
rename to llamafile/highlight/highlight_c.cpp
index 2cc9b2f6..563a8d69 100644
--- a/llamafile/highlight_c.cpp
+++ b/llamafile/highlight/highlight_c.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -41,20 +42,24 @@ enum {
UNIVERSAL,
};
-HighlightC::HighlightC(is_keyword_f *is_keyword, //
- is_keyword_f *is_type, //
- is_keyword_f *is_builtin, //
- is_keyword_f *is_constant)
- : is_keyword_(is_keyword),
- is_type_(is_type),
- is_builtin_(is_builtin),
- is_constant_(is_constant) {
+HighlightC::HighlightC(is_keyword_f* is_keyword, //
+ is_keyword_f* is_type, //
+ is_keyword_f* is_builtin, //
+ is_keyword_f* is_constant)
+ : is_keyword_(is_keyword)
+ , is_type_(is_type)
+ , is_builtin_(is_builtin)
+ , is_constant_(is_constant)
+{
}
-HighlightC::~HighlightC() {
+HighlightC::~HighlightC()
+{
}
-void HighlightC::feed(std::string *r, std::string_view input) {
+void
+HighlightC::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -121,7 +126,8 @@ void HighlightC::feed(std::string *r, std::string_view input) {
is_define_ = true;
}
*r += HI_RESET;
- } else if (is_keyword_c_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_c_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -135,17 +141,21 @@ void HighlightC::feed(std::string *r, std::string_view input) {
*r += HI_RESET;
if (is_keyword_c_pod(word_.data(), word_.size()))
is_pod_ = true;
- } else if (is_pod_ || (is_type_ && is_type_(word_.data(), word_.size()))) {
+ } else if (is_pod_ ||
+ (is_type_ &&
+ is_type_(word_.data(), word_.size()))) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
is_pod_ = false;
- } else if (is_builtin_ && is_builtin_(word_.data(), word_.size())) {
+ } else if (is_builtin_ &&
+ is_builtin_(word_.data(), word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
is_pod_ = false;
- } else if (is_constant_ && is_constant_(word_.data(), word_.size())) {
+ } else if (is_constant_ &&
+ is_constant_(word_.data(), word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -360,7 +370,9 @@ void HighlightC::feed(std::string *r, std::string_view input) {
}
}
-void HighlightC::flush(std::string *r) {
+void
+HighlightC::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_cpp_) {
@@ -381,7 +393,8 @@ void HighlightC::flush(std::string *r) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_pod_ || (is_type_ && is_type_(word_.data(), word_.size()))) {
+ } else if (is_pod_ ||
+ (is_type_ && is_type_(word_.data(), word_.size()))) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
@@ -389,7 +402,8 @@ void HighlightC::flush(std::string *r) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_constant_ && is_constant_(word_.data(), word_.size())) {
+ } else if (is_constant_ &&
+ is_constant_(word_.data(), word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
diff --git a/llamafile/highlight_c_test.cpp b/llamafile/highlight/highlight_c_test.cpp
similarity index 74%
rename from llamafile/highlight_c_test.cpp
rename to llamafile/highlight/highlight_c_test.cpp
index 227baf46..bd1ad9f6 100644
--- a/llamafile/highlight_c_test.cpp
+++ b/llamafile/highlight/highlight_c_test.cpp
@@ -24,7 +24,9 @@ using namespace std::string_literals;
HighlightC g_highlight;
-void test(std::string input, std::string expect) {
+void
+test(std::string input, std::string expect)
+{
std::string got;
g_highlight.feed(&got, input);
if (got != expect) {
@@ -33,12 +35,16 @@ void test(std::string input, std::string expect) {
"\t input: %`'s\n"
"\texpected: %`'s\n"
"\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
+ input.c_str(),
+ expect.c_str(),
+ got.c_str());
exit(1);
}
}
-void testf(std::string input, std::string expect) {
+void
+testf(std::string input, std::string expect)
+{
std::string got;
g_highlight.feed(&got, input);
g_highlight.flush(&got);
@@ -48,20 +54,28 @@ void testf(std::string input, std::string expect) {
"\t input: %`'s\n"
"\texpected: %`'s\n"
"\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
+ input.c_str(),
+ expect.c_str(),
+ got.c_str());
exit(1);
}
}
-int main(int argc, char *argv[]) {
+int
+main(int argc, char* argv[])
+{
test("", "");
test("vola", "");
test("tile ", ""s + HI_KEYWORD + "volatile" + HI_RESET + " ");
- test("yo // hi\nthere ", "yo "s + HI_COMMENT + "// hi\n" + HI_RESET + "there ");
+ test("yo // hi\nthere ",
+ "yo "s + HI_COMMENT + "// hi\n" + HI_RESET + "there ");
testf("// sup", ""s + HI_COMMENT + "// sup" + HI_RESET);
- test("yo /* hi */ there ", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");
- testf("yo /* hi */ there", "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");
- testf("yo /* h*i */ there", "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");
+ test("yo /* hi */ there ",
+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there ");
+ testf("yo /* hi */ there",
+ "yo "s + HI_COMMENT + "/* hi */" + HI_RESET + " there");
+ testf("yo /* h*i */ there",
+ "yo "s + HI_COMMENT + "/* h*i */" + HI_RESET + " there");
testf("vola", "vola");
testf("volatile", ""s + HI_KEYWORD + "volatile" + HI_RESET);
testf("/hi/", "/hi/");
diff --git a/llamafile/highlight_cmake.cpp b/llamafile/highlight/highlight_cmake.cpp
similarity index 96%
rename from llamafile/highlight_cmake.cpp
rename to llamafile/highlight/highlight_cmake.cpp
index ce34c5ad..1c2c5ff6 100644
--- a/llamafile/highlight_cmake.cpp
+++ b/llamafile/highlight/highlight_cmake.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
BACKSLASH,
WORD,
@@ -34,13 +35,17 @@ enum {
DQUOTE_VAR,
};
-HighlightCmake::HighlightCmake() {
+HighlightCmake::HighlightCmake()
+{
}
-HighlightCmake::~HighlightCmake() {
+HighlightCmake::~HighlightCmake()
+{
}
-void HighlightCmake::feed(std::string *r, std::string_view input) {
+void
+HighlightCmake::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
@@ -234,7 +239,9 @@ void HighlightCmake::feed(std::string *r, std::string_view input) {
}
}
-void HighlightCmake::flush(std::string *r) {
+void
+HighlightCmake::flush(std::string* r)
+{
switch (t_) {
case WORD:
*r += word_;
diff --git a/llamafile/highlight_cobol.cpp b/llamafile/highlight/highlight_cobol.cpp
similarity index 95%
rename from llamafile/highlight_cobol.cpp
rename to llamafile/highlight/highlight_cobol.cpp
index e57f84e2..df2f9727 100644
--- a/llamafile/highlight_cobol.cpp
+++ b/llamafile/highlight/highlight_cobol.cpp
@@ -21,7 +21,8 @@
// COmmon Business-Oriented Language
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -31,13 +32,17 @@ enum {
COMMENT,
};
-HighlightCobol::HighlightCobol() {
+HighlightCobol::HighlightCobol()
+{
}
-HighlightCobol::~HighlightCobol() {
+HighlightCobol::~HighlightCobol()
+{
}
-void HighlightCobol::feed(std::string *r, std::string_view input) {
+void
+HighlightCobol::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -160,7 +165,9 @@ void HighlightCobol::feed(std::string *r, std::string_view input) {
}
}
-void HighlightCobol::flush(std::string *r) {
+void
+HighlightCobol::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_cobol(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_create.gperf b/llamafile/highlight/highlight_create.gperf
similarity index 100%
rename from llamafile/highlight_create.gperf
rename to llamafile/highlight/highlight_create.gperf
diff --git a/llamafile/highlight_csharp.cpp b/llamafile/highlight/highlight_csharp.cpp
similarity index 96%
rename from llamafile/highlight_csharp.cpp
rename to llamafile/highlight/highlight_csharp.cpp
index 2e84ca6c..53e15033 100644
--- a/llamafile/highlight_csharp.cpp
+++ b/llamafile/highlight/highlight_csharp.cpp
@@ -36,7 +36,8 @@
// As we can see above, you can use four, five, or more dquotes so you
// can embed triple quoted strings inside.
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -54,13 +55,17 @@ enum {
TRIPS_DQUOTE,
};
-HighlightCsharp::HighlightCsharp() {
+HighlightCsharp::HighlightCsharp()
+{
}
-HighlightCsharp::~HighlightCsharp() {
+HighlightCsharp::~HighlightCsharp()
+{
}
-void HighlightCsharp::feed(std::string *r, std::string_view input) {
+void
+HighlightCsharp::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -94,7 +99,8 @@ void HighlightCsharp::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_csharp_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_csharp_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -242,7 +248,9 @@ void HighlightCsharp::feed(std::string *r, std::string_view input) {
}
}
-void HighlightCsharp::flush(std::string *r) {
+void
+HighlightCsharp::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_csharp(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_css.cpp b/llamafile/highlight/highlight_css.cpp
similarity index 97%
rename from llamafile/highlight_css.cpp
rename to llamafile/highlight/highlight_css.cpp
index a0f72813..5b45d08a 100644
--- a/llamafile/highlight_css.cpp
+++ b/llamafile/highlight/highlight_css.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
SELECTOR,
PROPERTY,
@@ -35,13 +36,17 @@ enum {
BANG,
};
-HighlightCss::HighlightCss() {
+HighlightCss::HighlightCss()
+{
}
-HighlightCss::~HighlightCss() {
+HighlightCss::~HighlightCss()
+{
}
-void HighlightCss::feed(std::string *r, std::string_view input) {
+void
+HighlightCss::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -273,7 +278,9 @@ void HighlightCss::feed(std::string *r, std::string_view input) {
}
}
-void HighlightCss::flush(std::string *r) {
+void
+HighlightCss::flush(std::string* r)
+{
while (t_) {
switch (t_ & 255) {
case AT:
diff --git a/llamafile/highlight_d.cpp b/llamafile/highlight/highlight_d.cpp
similarity index 98%
rename from llamafile/highlight_d.cpp
rename to llamafile/highlight/highlight_d.cpp
index 0e038529..c9f89269 100644
--- a/llamafile/highlight_d.cpp
+++ b/llamafile/highlight/highlight_d.cpp
@@ -16,14 +16,15 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
// D probably has the best lexical syntax documentation of any language.
// https://dlang.org/spec/lex.html
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -52,7 +53,9 @@ enum {
X_DQUOTE,
};
-static bool is_line_terminator(wchar_t c) {
+static bool
+is_line_terminator(wchar_t c)
+{
switch (c) {
case '\r':
case '\n':
@@ -64,7 +67,9 @@ static bool is_line_terminator(wchar_t c) {
}
}
-static int mirror(int c) {
+static int
+mirror(int c)
+{
switch (c) {
case '(':
return ')';
@@ -79,13 +84,17 @@ static int mirror(int c) {
}
}
-HighlightD::HighlightD() {
+HighlightD::HighlightD()
+{
}
-HighlightD::~HighlightD() {
+HighlightD::~HighlightD()
+{
}
-void HighlightD::feed(std::string *r, std::string_view input) {
+void
+HighlightD::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
@@ -452,7 +461,9 @@ void HighlightD::feed(std::string *r, std::string_view input) {
}
}
-void HighlightD::flush(std::string *r) {
+void
+HighlightD::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_d(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_forth.cpp b/llamafile/highlight/highlight_forth.cpp
similarity index 91%
rename from llamafile/highlight_forth.cpp
rename to llamafile/highlight/highlight_forth.cpp
index 83148298..f6be6a77 100644
--- a/llamafile/highlight_forth.cpp
+++ b/llamafile/highlight/highlight_forth.cpp
@@ -19,18 +19,23 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
SYNTAX,
};
-HighlightForth::HighlightForth() {
+HighlightForth::HighlightForth()
+{
}
-HighlightForth::~HighlightForth() {
+HighlightForth::~HighlightForth()
+{
}
-void HighlightForth::feed(std::string *r, std::string_view input) {
+void
+HighlightForth::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -49,8 +54,10 @@ void HighlightForth::feed(std::string *r, std::string_view input) {
*r += word_;
t_ = SYNTAX;
closer_ = '\n';
- } else if (word_ == "(" || // inline comment, e.g. ( arg1 arg2 -- res1 )
- word_ == ".(") { // printed comment, e.g. .( compiling... )
+ } else if (word_ == "(" || // inline comment, e.g. ( arg1
+ // arg2 -- res1 )
+ word_ == ".(") { // printed comment, e.g. .(
+ // compiling... )
*r += HI_COMMENT;
*r += word_;
t_ = SYNTAX;
@@ -98,7 +105,9 @@ void HighlightForth::feed(std::string *r, std::string_view input) {
}
}
-void HighlightForth::flush(std::string *r) {
+void
+HighlightForth::flush(std::string* r)
+{
switch (t_) {
case NORMAL:
if (is_label_) {
diff --git a/llamafile/highlight_fortran.cpp b/llamafile/highlight/highlight_fortran.cpp
similarity index 93%
rename from llamafile/highlight_fortran.cpp
rename to llamafile/highlight/highlight_fortran.cpp
index d7284b4f..e598c4c5 100644
--- a/llamafile/highlight_fortran.cpp
+++ b/llamafile/highlight/highlight_fortran.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -29,13 +30,17 @@ enum {
COMMENT,
};
-HighlightFortran::HighlightFortran() {
+HighlightFortran::HighlightFortran()
+{
}
-HighlightFortran::~HighlightFortran() {
+HighlightFortran::~HighlightFortran()
+{
}
-void HighlightFortran::feed(std::string *r, std::string_view input) {
+void
+HighlightFortran::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -105,11 +110,13 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_fortran_type(word_.data(), word_.size())) {
+ } else if (is_keyword_fortran_type(word_.data(),
+ word_.size())) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_fortran_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_fortran_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
@@ -166,7 +173,9 @@ void HighlightFortran::feed(std::string *r, std::string_view input) {
}
}
-void HighlightFortran::flush(std::string *r) {
+void
+HighlightFortran::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_fortran(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_go.cpp b/llamafile/highlight/highlight_go.cpp
similarity index 96%
rename from llamafile/highlight_go.cpp
rename to llamafile/highlight/highlight_go.cpp
index 0686edf2..66970de1 100644
--- a/llamafile/highlight_go.cpp
+++ b/llamafile/highlight/highlight_go.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -34,13 +35,17 @@ enum {
TICK_BACKSLASH,
};
-HighlightGo::HighlightGo() {
+HighlightGo::HighlightGo()
+{
}
-HighlightGo::~HighlightGo() {
+HighlightGo::~HighlightGo()
+{
}
-void HighlightGo::feed(std::string *r, std::string_view input) {
+void
+HighlightGo::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -183,7 +188,9 @@ void HighlightGo::feed(std::string *r, std::string_view input) {
}
}
-void HighlightGo::flush(std::string *r) {
+void
+HighlightGo::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_go(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_haskell.cpp b/llamafile/highlight/highlight_haskell.cpp
similarity index 97%
rename from llamafile/highlight_haskell.cpp
rename to llamafile/highlight/highlight_haskell.cpp
index e60b9edc..4a74be49 100644
--- a/llamafile/highlight_haskell.cpp
+++ b/llamafile/highlight/highlight_haskell.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
DQUOTE,
@@ -38,13 +39,17 @@ enum {
LT,
};
-HighlightHaskell::HighlightHaskell() {
+HighlightHaskell::HighlightHaskell()
+{
}
-HighlightHaskell::~HighlightHaskell() {
+HighlightHaskell::~HighlightHaskell()
+{
}
-void HighlightHaskell::feed(std::string *r, std::string_view input) {
+void
+HighlightHaskell::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -272,7 +277,9 @@ void HighlightHaskell::feed(std::string *r, std::string_view input) {
}
}
-void HighlightHaskell::flush(std::string *r) {
+void
+HighlightHaskell::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_haskell(symbol_.data(), symbol_.size())) {
diff --git a/llamafile/highlight_html.cpp b/llamafile/highlight/highlight_html.cpp
similarity index 96%
rename from llamafile/highlight_html.cpp
rename to llamafile/highlight/highlight_html.cpp
index 9bc979d0..b2b0ef38 100644
--- a/llamafile/highlight_html.cpp
+++ b/llamafile/highlight/highlight_html.cpp
@@ -17,7 +17,8 @@
#include "highlight.h"
-enum {
+enum
+{
NORMAL,
TAG,
TAG2,
@@ -37,13 +38,17 @@ enum {
ENTITY,
};
-HighlightHtml::HighlightHtml() {
+HighlightHtml::HighlightHtml()
+{
}
-HighlightHtml::~HighlightHtml() {
+HighlightHtml::~HighlightHtml()
+{
}
-void HighlightHtml::feed(std::string *r, std::string_view input) {
+void
+HighlightHtml::feed(std::string* r, std::string_view input)
+{
char c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i];
@@ -83,7 +88,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {
*r += '<';
*r += HI_TAG;
*r += c;
- name_ += tolower(c);
+ name_ += std::tolower(c);
t_ = TAG2;
}
break;
@@ -99,7 +104,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {
*r += HI_ATTRIB;
} else {
*r += c;
- name_ += tolower(c);
+ name_ += std::tolower(c);
}
break;
@@ -219,7 +224,7 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {
break;
case RELAY:
- if (closer_[i_] == tolower(c)) {
+ if (closer_[i_] == std::tolower(c)) {
pending_ += c;
if (++i_ == closer_.size()) {
highlighter_->flush(r);
@@ -294,7 +299,9 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {
}
}
-void HighlightHtml::flush(std::string *r) {
+void
+HighlightHtml::flush(std::string* r)
+{
switch (t_) {
case TAG:
*r += '<';
diff --git a/llamafile/highlight_java.cpp b/llamafile/highlight/highlight_java.cpp
similarity index 96%
rename from llamafile/highlight_java.cpp
rename to llamafile/highlight/highlight_java.cpp
index 7e436ae4..df537d30 100644
--- a/llamafile/highlight_java.cpp
+++ b/llamafile/highlight/highlight_java.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -40,13 +41,17 @@ enum {
DQUOTE32, // """...""
};
-HighlightJava::HighlightJava() {
+HighlightJava::HighlightJava()
+{
}
-HighlightJava::~HighlightJava() {
+HighlightJava::~HighlightJava()
+{
}
-void HighlightJava::feed(std::string *r, std::string_view input) {
+void
+HighlightJava::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -82,7 +87,8 @@ void HighlightJava::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_java_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_java_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -255,7 +261,9 @@ void HighlightJava::feed(std::string *r, std::string_view input) {
}
}
-void HighlightJava::flush(std::string *r) {
+void
+HighlightJava::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_java(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_js.cpp b/llamafile/highlight/highlight_js.cpp
similarity index 97%
rename from llamafile/highlight_js.cpp
rename to llamafile/highlight/highlight_js.cpp
index fe2a0a41..ee45f32d 100644
--- a/llamafile/highlight_js.cpp
+++ b/llamafile/highlight/highlight_js.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -40,19 +41,24 @@ enum {
REGEX_SQUARE_BACKSLASH,
};
-enum {
+enum
+{
EXPECT_VALUE,
EXPECT_OPERATOR,
};
-HighlightJs::HighlightJs() {
+HighlightJs::HighlightJs()
+{
}
-HighlightJs::~HighlightJs() {
+HighlightJs::~HighlightJs()
+{
}
// https://262.ecma-international.org/12.0/#sec-line-terminators
-static bool is_line_terminator(wchar_t c) {
+static bool
+is_line_terminator(wchar_t c)
+{
switch (c) {
case '\r':
case '\n':
@@ -64,7 +70,9 @@ static bool is_line_terminator(wchar_t c) {
}
}
-void HighlightJs::feed(std::string *r, std::string_view input) {
+void
+HighlightJs::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -329,7 +337,9 @@ void HighlightJs::feed(std::string *r, std::string_view input) {
}
}
-void HighlightJs::flush(std::string *r) {
+void
+HighlightJs::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_js(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_julia.cpp b/llamafile/highlight/highlight_julia.cpp
similarity index 96%
rename from llamafile/highlight_julia.cpp
rename to llamafile/highlight/highlight_julia.cpp
index 05384ab6..2a8263c8 100644
--- a/llamafile/highlight_julia.cpp
+++ b/llamafile/highlight/highlight_julia.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
@@ -38,13 +39,17 @@ enum {
DQUOTE32,
};
-HighlightJulia::HighlightJulia() {
+HighlightJulia::HighlightJulia()
+{
}
-HighlightJulia::~HighlightJulia() {
+HighlightJulia::~HighlightJulia()
+{
}
-void HighlightJulia::feed(std::string *r, std::string_view input) {
+void
+HighlightJulia::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -219,7 +224,9 @@ void HighlightJulia::feed(std::string *r, std::string_view input) {
}
}
-void HighlightJulia::flush(std::string *r) {
+void
+HighlightJulia::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_julia(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_kotlin.cpp b/llamafile/highlight/highlight_kotlin.cpp
similarity index 97%
rename from llamafile/highlight_kotlin.cpp
rename to llamafile/highlight/highlight_kotlin.cpp
index 9824befd..c55ac3d4 100644
--- a/llamafile/highlight_kotlin.cpp
+++ b/llamafile/highlight/highlight_kotlin.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -46,13 +47,17 @@ enum {
DQUOTE32,
};
-HighlightKotlin::HighlightKotlin() {
+HighlightKotlin::HighlightKotlin()
+{
}
-HighlightKotlin::~HighlightKotlin() {
+HighlightKotlin::~HighlightKotlin()
+{
}
-void HighlightKotlin::feed(std::string *r, std::string_view input) {
+void
+HighlightKotlin::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -95,7 +100,8 @@ void HighlightKotlin::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_java_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_java_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -334,7 +340,9 @@ void HighlightKotlin::feed(std::string *r, std::string_view input) {
}
}
-void HighlightKotlin::flush(std::string *r) {
+void
+HighlightKotlin::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_kotlin(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_ld.cpp b/llamafile/highlight/highlight_ld.cpp
similarity index 93%
rename from llamafile/highlight_ld.cpp
rename to llamafile/highlight/highlight_ld.cpp
index 93443b12..823a1283 100644
--- a/llamafile/highlight_ld.cpp
+++ b/llamafile/highlight/highlight_ld.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
DQUOTE,
@@ -30,13 +31,17 @@ enum {
SLASH_STAR_STAR,
};
-HighlightLd::HighlightLd() {
+HighlightLd::HighlightLd()
+{
}
-HighlightLd::~HighlightLd() {
+HighlightLd::~HighlightLd()
+{
}
-void HighlightLd::feed(std::string *r, std::string_view input) {
+void
+HighlightLd::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -70,7 +75,8 @@ void HighlightLd::feed(std::string *r, std::string_view input) {
Word:
case WORD:
- if (!isascii(c) || isalnum(c) || c == '_' || c == '.' || c == '/' || c == ':') {
+ if (!isascii(c) || isalnum(c) || c == '_' || c == '.' || c == '/' ||
+ c == ':') {
word_ += c;
} else {
if (is_cpp_) {
@@ -78,7 +84,8 @@ void HighlightLd::feed(std::string *r, std::string_view input) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_c_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_c_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -90,11 +97,13 @@ void HighlightLd::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_ld_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_ld_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_ld_warning(word_.data(), word_.size())) {
+ } else if (is_keyword_ld_warning(word_.data(),
+ word_.size())) {
*r += HI_WARNING;
*r += word_;
*r += HI_RESET;
@@ -181,7 +190,9 @@ void HighlightLd::feed(std::string *r, std::string_view input) {
}
}
-void HighlightLd::flush(std::string *r) {
+void
+HighlightLd::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_cpp_) {
diff --git a/llamafile/highlight_lisp.cpp b/llamafile/highlight/highlight_lisp.cpp
similarity index 93%
rename from llamafile/highlight_lisp.cpp
rename to llamafile/highlight/highlight_lisp.cpp
index c9ed8eea..db937069 100644
--- a/llamafile/highlight_lisp.cpp
+++ b/llamafile/highlight/highlight_lisp.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
SYMBOL,
DQUOTE,
@@ -27,13 +28,17 @@ enum {
COMMENT,
};
-HighlightLisp::HighlightLisp() {
+HighlightLisp::HighlightLisp()
+{
}
-HighlightLisp::~HighlightLisp() {
+HighlightLisp::~HighlightLisp()
+{
}
-void HighlightLisp::feed(std::string *r, std::string_view input) {
+void
+HighlightLisp::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -82,7 +87,8 @@ void HighlightLisp::feed(std::string *r, std::string_view input) {
c == '`' || //
c == '"' || //
c == '\'') {
- if (is_first_ && is_keyword_lisp(symbol_.data(), symbol_.size())) {
+ if (is_first_ &&
+ is_keyword_lisp(symbol_.data(), symbol_.size())) {
*r += HI_KEYWORD;
*r += symbol_;
*r += HI_RESET;
@@ -131,7 +137,9 @@ void HighlightLisp::feed(std::string *r, std::string_view input) {
}
}
-void HighlightLisp::flush(std::string *r) {
+void
+HighlightLisp::flush(std::string* r)
+{
switch (t_) {
case SYMBOL:
if (is_first_ && is_keyword_lisp(symbol_.data(), symbol_.size())) {
diff --git a/llamafile/highlight_lua.cpp b/llamafile/highlight/highlight_lua.cpp
similarity index 96%
rename from llamafile/highlight_lua.cpp
rename to llamafile/highlight/highlight_lua.cpp
index 9e62e8d2..2759c94b 100644
--- a/llamafile/highlight_lua.cpp
+++ b/llamafile/highlight/highlight_lua.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -36,13 +37,17 @@ enum {
LITERAL_RSB,
};
-HighlightLua::HighlightLua() {
+HighlightLua::HighlightLua()
+{
}
-HighlightLua::~HighlightLua() {
+HighlightLua::~HighlightLua()
+{
}
-void HighlightLua::feed(std::string *r, std::string_view input) {
+void
+HighlightLua::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -79,7 +84,8 @@ void HighlightLua::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_lua_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_lua_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -220,7 +226,9 @@ void HighlightLua::feed(std::string *r, std::string_view input) {
}
}
-void HighlightLua::flush(std::string *r) {
+void
+HighlightLua::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_lua(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_m4.cpp b/llamafile/highlight/highlight_m4.cpp
similarity index 94%
rename from llamafile/highlight_m4.cpp
rename to llamafile/highlight/highlight_m4.cpp
index 7dbc1942..f4a4eb38 100644
--- a/llamafile/highlight_m4.cpp
+++ b/llamafile/highlight/highlight_m4.cpp
@@ -19,20 +19,25 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
DOLLAR,
};
-HighlightM4::HighlightM4() {
+HighlightM4::HighlightM4()
+{
}
-HighlightM4::~HighlightM4() {
+HighlightM4::~HighlightM4()
+{
}
-void HighlightM4::feed(std::string *r, std::string_view input) {
+void
+HighlightM4::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -107,7 +112,9 @@ void HighlightM4::feed(std::string *r, std::string_view input) {
}
}
-void HighlightM4::flush(std::string *r) {
+void
+HighlightM4::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_m4(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_make.cpp b/llamafile/highlight/highlight_make.cpp
similarity index 96%
rename from llamafile/highlight_make.cpp
rename to llamafile/highlight/highlight_make.cpp
index 0696d75d..b82db978 100644
--- a/llamafile/highlight_make.cpp
+++ b/llamafile/highlight/highlight_make.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
@@ -31,7 +32,9 @@ enum {
BACKSLASH,
};
-static bool is_automatic_variable(int c) {
+static bool
+is_automatic_variable(int c)
+{
switch (c) {
case '@':
case '%':
@@ -47,13 +50,17 @@ static bool is_automatic_variable(int c) {
}
}
-HighlightMake::HighlightMake() {
+HighlightMake::HighlightMake()
+{
}
-HighlightMake::~HighlightMake() {
+HighlightMake::~HighlightMake()
+{
}
-void HighlightMake::feed(std::string *r, std::string_view input) {
+void
+HighlightMake::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -208,7 +215,9 @@ void HighlightMake::feed(std::string *r, std::string_view input) {
}
}
-void HighlightMake::flush(std::string *r) {
+void
+HighlightMake::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_make(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight/highlight_markdown.cpp
similarity index 96%
rename from llamafile/highlight_markdown.cpp
rename to llamafile/highlight/highlight_markdown.cpp
index 33c819f4..1ea9aaec 100644
--- a/llamafile/highlight_markdown.cpp
+++ b/llamafile/highlight/highlight_markdown.cpp
@@ -16,10 +16,11 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
-enum {
+enum
+{
NORMAL,
TICK,
TICK_TICK,
@@ -40,13 +41,17 @@ enum {
EMPHASIS_BACKSLASH,
};
-HighlightMarkdown::HighlightMarkdown() {
+HighlightMarkdown::HighlightMarkdown()
+{
}
-HighlightMarkdown::~HighlightMarkdown() {
+HighlightMarkdown::~HighlightMarkdown()
+{
}
-void HighlightMarkdown::feed(std::string *r, std::string_view input) {
+void
+HighlightMarkdown::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -115,7 +120,8 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
t_ = NORMAL;
} else {
// handle *emphasized* text
- // inverted because \e[3m has a poorly supported western bias
+ // inverted because \e[3m has a poorly supported western
+ // bias
*r += '*';
*r += HI_ITALIC;
lf::append_wchar(r, c);
@@ -245,7 +251,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
case LANG:
if (!isascii(c) || !isspace(c)) {
lf::append_wchar(r, c);
- lang_ += tolower(c);
+ lang_ += std::tolower(c);
} else {
if (!(highlighter_ = Highlight::create(lang_)))
highlighter_ = new HighlightTxt;
@@ -304,7 +310,9 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {
}
}
-void HighlightMarkdown::flush(std::string *r) {
+void
+HighlightMarkdown::flush(std::string* r)
+{
switch (t_) {
case LANG:
lang_.clear();
diff --git a/llamafile/highlight_matlab.cpp b/llamafile/highlight/highlight_matlab.cpp
similarity index 92%
rename from llamafile/highlight_matlab.cpp
rename to llamafile/highlight/highlight_matlab.cpp
index e0ee4ad4..678fc4f4 100644
--- a/llamafile/highlight_matlab.cpp
+++ b/llamafile/highlight/highlight_matlab.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
@@ -29,13 +30,17 @@ enum {
DQUOTE_BACKSLASH,
};
-HighlightMatlab::HighlightMatlab() {
+HighlightMatlab::HighlightMatlab()
+{
}
-HighlightMatlab::~HighlightMatlab() {
+HighlightMatlab::~HighlightMatlab()
+{
}
-void HighlightMatlab::feed(std::string *r, std::string_view input) {
+void
+HighlightMatlab::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -71,11 +76,13 @@ void HighlightMatlab::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_matlab_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_matlab_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_matlab_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_matlab_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -132,7 +139,9 @@ void HighlightMatlab::feed(std::string *r, std::string_view input) {
}
}
-void HighlightMatlab::flush(std::string *r) {
+void
+HighlightMatlab::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_matlab(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_ocaml.cpp b/llamafile/highlight/highlight_ocaml.cpp
similarity index 94%
rename from llamafile/highlight_ocaml.cpp
rename to llamafile/highlight/highlight_ocaml.cpp
index 9fd9c194..a4831918 100644
--- a/llamafile/highlight_ocaml.cpp
+++ b/llamafile/highlight/highlight_ocaml.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -35,13 +36,17 @@ enum {
RAWSTR_PIPE,
};
-HighlightOcaml::HighlightOcaml() {
+HighlightOcaml::HighlightOcaml()
+{
}
-HighlightOcaml::~HighlightOcaml() {
+HighlightOcaml::~HighlightOcaml()
+{
}
-void HighlightOcaml::feed(std::string *r, std::string_view input) {
+void
+HighlightOcaml::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -72,18 +77,21 @@ void HighlightOcaml::feed(std::string *r, std::string_view input) {
Word:
case WORD:
- if (!isascii(c) || isalnum(c) || c == '_' || c == '\'' || c == '~') {
+ if (!isascii(c) || isalnum(c) || c == '_' || c == '\'' ||
+ c == '~') {
word_ += c;
} else {
if (is_keyword_ocaml(word_.data(), word_.size())) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_ocaml_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_ocaml_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_ocaml_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_ocaml_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -227,7 +235,9 @@ void HighlightOcaml::feed(std::string *r, std::string_view input) {
}
}
-void HighlightOcaml::flush(std::string *r) {
+void
+HighlightOcaml::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_ocaml(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_pascal.cpp b/llamafile/highlight/highlight_pascal.cpp
similarity index 95%
rename from llamafile/highlight_pascal.cpp
rename to llamafile/highlight/highlight_pascal.cpp
index eceb3535..e304aa5a 100644
--- a/llamafile/highlight_pascal.cpp
+++ b/llamafile/highlight/highlight_pascal.cpp
@@ -28,7 +28,8 @@
//
// doing that requires extra keywords
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -41,13 +42,17 @@ enum {
PAREN_STAR_STAR,
};
-HighlightPascal::HighlightPascal() {
+HighlightPascal::HighlightPascal()
+{
}
-HighlightPascal::~HighlightPascal() {
+HighlightPascal::~HighlightPascal()
+{
}
-void HighlightPascal::feed(std::string *r, std::string_view input) {
+void
+HighlightPascal::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -93,7 +98,8 @@ void HighlightPascal::feed(std::string *r, std::string_view input) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_pascal_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_pascal_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
@@ -184,7 +190,9 @@ void HighlightPascal::feed(std::string *r, std::string_view input) {
}
}
-void HighlightPascal::flush(std::string *r) {
+void
+HighlightPascal::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_pascal(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_perl.cpp b/llamafile/highlight/highlight_perl.cpp
similarity index 97%
rename from llamafile/highlight_perl.cpp
rename to llamafile/highlight/highlight_perl.cpp
index 4380b25c..53bf9771 100644
--- a/llamafile/highlight_perl.cpp
+++ b/llamafile/highlight/highlight_perl.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -48,12 +49,15 @@ enum {
BACKSLASH,
};
-enum {
+enum
+{
EXPECT_VALUE,
EXPECT_OPERATOR,
};
-static int mirror(int c) {
+static int
+mirror(int c)
+{
switch (c) {
case '(':
return ')';
@@ -68,7 +72,9 @@ static int mirror(int c) {
}
}
-static bool is_magic_var(int c) {
+static bool
+is_magic_var(int c)
+{
switch (c) {
case '!':
case '"':
@@ -92,7 +98,9 @@ static bool is_magic_var(int c) {
}
}
-static bool is_regex_punct(int c) {
+static bool
+is_regex_punct(int c)
+{
switch (c) {
case '!':
case '"':
@@ -123,7 +131,9 @@ static bool is_regex_punct(int c) {
}
}
-static bool is_regex_prefix(const std::string_view &s) {
+static bool
+is_regex_prefix(const std::string_view& s)
+{
return s == "m" || //
s == "s" || //
s == "y" || //
@@ -135,19 +145,25 @@ static bool is_regex_prefix(const std::string_view &s) {
s == "qr";
}
-static bool is_double_regex(const std::string_view &s) {
+static bool
+is_double_regex(const std::string_view& s)
+{
return s == "s" || //
s == "y" || //
s == "tr";
}
-HighlightPerl::HighlightPerl() {
+HighlightPerl::HighlightPerl()
+{
}
-HighlightPerl::~HighlightPerl() {
+HighlightPerl::~HighlightPerl()
+{
}
-void HighlightPerl::feed(std::string *r, std::string_view input) {
+void
+HighlightPerl::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -521,7 +537,9 @@ void HighlightPerl::feed(std::string *r, std::string_view input) {
}
}
-void HighlightPerl::flush(std::string *r) {
+void
+HighlightPerl::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_perl(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_php.cpp b/llamafile/highlight/highlight_php.cpp
similarity index 95%
rename from llamafile/highlight_php.cpp
rename to llamafile/highlight/highlight_php.cpp
index d097786e..334856b7 100644
--- a/llamafile/highlight_php.cpp
+++ b/llamafile/highlight/highlight_php.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -35,13 +36,17 @@ enum {
VAR,
};
-HighlightPhp::HighlightPhp() {
+HighlightPhp::HighlightPhp()
+{
}
-HighlightPhp::~HighlightPhp() {
+HighlightPhp::~HighlightPhp()
+{
}
-void HighlightPhp::feed(std::string *r, std::string_view input) {
+void
+HighlightPhp::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -83,7 +88,8 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_php_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_php_constant(word_.data(),
+ word_.size())) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
@@ -197,7 +203,9 @@ void HighlightPhp::feed(std::string *r, std::string_view input) {
}
}
-void HighlightPhp::flush(std::string *r) {
+void
+HighlightPhp::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_php(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_python.cpp b/llamafile/highlight/highlight_python.cpp
similarity index 95%
rename from llamafile/highlight_python.cpp
rename to llamafile/highlight/highlight_python.cpp
index 06f0f950..103d9692 100644
--- a/llamafile/highlight_python.cpp
+++ b/llamafile/highlight/highlight_python.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COM,
@@ -41,13 +42,17 @@ enum {
DQUOTE32,
};
-HighlightPython::HighlightPython() {
+HighlightPython::HighlightPython()
+{
}
-HighlightPython::~HighlightPython() {
+HighlightPython::~HighlightPython()
+{
}
-void HighlightPython::feed(std::string *r, std::string_view input) {
+void
+HighlightPython::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -84,11 +89,13 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_python_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_python_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_python_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_python_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -265,7 +272,9 @@ void HighlightPython::feed(std::string *r, std::string_view input) {
}
}
-void HighlightPython::flush(std::string *r) {
+void
+HighlightPython::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_python(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_python_test.cpp b/llamafile/highlight/highlight_python_test.cpp
similarity index 77%
rename from llamafile/highlight_python_test.cpp
rename to llamafile/highlight/highlight_python_test.cpp
index 0c5f0787..657a5374 100644
--- a/llamafile/highlight_python_test.cpp
+++ b/llamafile/highlight/highlight_python_test.cpp
@@ -24,7 +24,9 @@ using namespace std::string_literals;
HighlightPython g_highlight;
-void test(std::string input, std::string expect) {
+void
+test(std::string input, std::string expect)
+{
std::string got;
g_highlight.feed(&got, input);
if (got != expect) {
@@ -33,12 +35,16 @@ void test(std::string input, std::string expect) {
"\t input: %`'s\n"
"\texpected: %`'s\n"
"\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
+ input.c_str(),
+ expect.c_str(),
+ got.c_str());
exit(1);
}
}
-void testf(std::string input, std::string expect) {
+void
+testf(std::string input, std::string expect)
+{
std::string got;
g_highlight.feed(&got, input);
g_highlight.flush(&got);
@@ -48,16 +54,21 @@ void testf(std::string input, std::string expect) {
"\t input: %`'s\n"
"\texpected: %`'s\n"
"\t got: %`'s\n",
- input.c_str(), expect.c_str(), got.c_str());
+ input.c_str(),
+ expect.c_str(),
+ got.c_str());
exit(1);
}
}
-int main(int argc, char *argv[]) {
+int
+main(int argc, char* argv[])
+{
test("", "");
test("d", "");
test("ef ", ""s + HI_KEYWORD + "def" + HI_RESET + " ");
- test("yo # hi\nthere ", "yo "s + HI_COMMENT + "# hi\n" + HI_RESET + "there ");
+ test("yo # hi\nthere ",
+ "yo "s + HI_COMMENT + "# hi\n" + HI_RESET + "there ");
testf("# sup", ""s + HI_COMMENT + "# sup" + HI_RESET);
testf("de", "de");
testf("def", ""s + HI_KEYWORD + "def" + HI_RESET);
@@ -66,10 +77,13 @@ int main(int argc, char *argv[]) {
testf("x '''''' y", "x "s + HI_STRING + "''''''" + HI_RESET + " y");
testf("x '''xxx''' y", "x "s + HI_STRING + "'''xxx'''" + HI_RESET + " y");
testf("x '''xx'x''' y", "x "s + HI_STRING + "'''xx'x'''" + HI_RESET + " y");
- testf("x '''xx''x''' y", "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");
+ testf("x '''xx''x''' y",
+ "x "s + HI_STRING + "'''xx''x'''" + HI_RESET + " y");
testf("x \"\" y", "x "s + HI_STRING + "\"\"" + HI_RESET + " y");
- testf("x \"\"\"\"\"\" y", "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");
- testf("x \"\"\"xxx\"\"\" y", "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");
+ testf("x \"\"\"\"\"\" y",
+ "x "s + HI_STRING + "\"\"\"\"\"\"" + HI_RESET + " y");
+ testf("x \"\"\"xxx\"\"\" y",
+ "x "s + HI_STRING + "\"\"\"xxx\"\"\"" + HI_RESET + " y");
testf("x 'c' y", "x "s + HI_STRING + "'c'" + HI_RESET + " y");
testf("x '\\'' y", "x "s + HI_STRING + "'\\''" + HI_RESET + " y");
testf("x \"c\" y", "x "s + HI_STRING + "\"c\"" + HI_RESET + " y");
diff --git a/llamafile/highlight_r.cpp b/llamafile/highlight/highlight_r.cpp
similarity index 97%
rename from llamafile/highlight_r.cpp
rename to llamafile/highlight/highlight_r.cpp
index 0b7c5a09..bf75dc81 100644
--- a/llamafile/highlight_r.cpp
+++ b/llamafile/highlight/highlight_r.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
COMMENT,
@@ -34,13 +35,17 @@ enum {
COLON,
};
-HighlightR::HighlightR() {
+HighlightR::HighlightR()
+{
}
-HighlightR::~HighlightR() {
+HighlightR::~HighlightR()
+{
}
-void HighlightR::feed(std::string *r, std::string_view input) {
+void
+HighlightR::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -214,7 +219,9 @@ void HighlightR::feed(std::string *r, std::string_view input) {
}
}
-void HighlightR::flush(std::string *r) {
+void
+HighlightR::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_r(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_ruby.cpp b/llamafile/highlight/highlight_ruby.cpp
similarity index 97%
rename from llamafile/highlight_ruby.cpp
rename to llamafile/highlight/highlight_ruby.cpp
index 928c1206..7500ebed 100644
--- a/llamafile/highlight_ruby.cpp
+++ b/llamafile/highlight/highlight_ruby.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
NUMBER,
WORD,
@@ -65,13 +66,16 @@ enum {
QUESTION_BACKSLASH,
};
-enum {
+enum
+{
EXPECT_EXPR,
EXPECT_VALUE,
EXPECT_OPERATOR,
};
-static int mirror(int c) {
+static int
+mirror(int c)
+{
switch (c) {
case '(':
return ')';
@@ -86,7 +90,9 @@ static int mirror(int c) {
}
}
-static bool ispunct_overridable(int c) {
+static bool
+ispunct_overridable(int c)
+{
switch (c) {
case '%':
case '&':
@@ -107,7 +113,9 @@ static bool ispunct_overridable(int c) {
}
}
-static bool is_dollar_one(int c) {
+static bool
+is_dollar_one(int c)
+{
switch (c) {
case '!':
case '"':
@@ -141,7 +149,9 @@ static bool is_dollar_one(int c) {
}
}
-static bool is_percent_literal(int c) {
+static bool
+is_percent_literal(int c)
+{
switch (c) {
case 'q':
case 'Q':
@@ -158,7 +168,9 @@ static bool is_percent_literal(int c) {
}
}
-static bool isident(int c) {
+static bool
+isident(int c)
+{
return !isascii(c) || //
isalnum(c) || //
c == '!' || //
@@ -174,13 +186,17 @@ static bool isident(int c) {
c == '_';
}
-HighlightRuby::HighlightRuby() {
+HighlightRuby::HighlightRuby()
+{
}
-HighlightRuby::~HighlightRuby() {
+HighlightRuby::~HighlightRuby()
+{
}
-void HighlightRuby::feed(std::string *r, std::string_view input) {
+void
+HighlightRuby::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
@@ -258,10 +274,12 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
*r += HI_COMMENT;
lf::append_wchar(r, c);
t_ = COMMENT;
- } else if (c == '<' && (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {
+ } else if (c == '<' &&
+ (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {
lf::append_wchar(r, c);
t_ = LT;
- } else if (c == '/' && (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {
+ } else if (c == '/' &&
+ (expect_ == EXPECT_EXPR || expect_ == EXPECT_VALUE)) {
t_ = REGEX;
*r += HI_STRING;
lf::append_wchar(r, c);
@@ -365,12 +383,14 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
expect_ = EXPECT_OPERATOR;
}
} else if (expect_ == EXPECT_EXPR &&
- is_keyword_ruby_builtin(word_.data(), word_.size())) {
+ is_keyword_ruby_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
expect_ = EXPECT_VALUE;
- } else if (is_keyword_ruby_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_ruby_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -909,7 +929,9 @@ void HighlightRuby::feed(std::string *r, std::string_view input) {
}
}
-void HighlightRuby::flush(std::string *r) {
+void
+HighlightRuby::flush(std::string* r)
+{
switch (t_) {
case EQUAL_WORD:
*r += '=';
diff --git a/llamafile/highlight_rust.cpp b/llamafile/highlight/highlight_rust.cpp
similarity index 94%
rename from llamafile/highlight_rust.cpp
rename to llamafile/highlight/highlight_rust.cpp
index 20bd8076..2763e111 100644
--- a/llamafile/highlight_rust.cpp
+++ b/llamafile/highlight/highlight_rust.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -36,13 +37,17 @@ enum {
ATTRIB,
};
-HighlightRust::HighlightRust() {
+HighlightRust::HighlightRust()
+{
}
-HighlightRust::~HighlightRust() {
+HighlightRust::~HighlightRust()
+{
}
-void HighlightRust::feed(std::string *r, std::string_view input) {
+void
+HighlightRust::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -79,7 +84,8 @@ void HighlightRust::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (word_.size() >= 2 && word_[word_.size() - 1] == '!') {
+ } else if (word_.size() >= 2 &&
+ word_[word_.size() - 1] == '!') {
*r += HI_MACRO;
*r += word_;
*r += HI_RESET;
@@ -87,7 +93,8 @@ void HighlightRust::feed(std::string *r, std::string_view input) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_rust_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_rust_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -230,7 +237,9 @@ void HighlightRust::feed(std::string *r, std::string_view input) {
}
}
-void HighlightRust::flush(std::string *r) {
+void
+HighlightRust::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_rust(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_scala.cpp b/llamafile/highlight/highlight_scala.cpp
similarity index 97%
rename from llamafile/highlight_scala.cpp
rename to llamafile/highlight/highlight_scala.cpp
index 226de140..142e56ba 100644
--- a/llamafile/highlight_scala.cpp
+++ b/llamafile/highlight/highlight_scala.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -46,13 +47,17 @@ enum {
DQUOTE32, // """...""
};
-HighlightScala::HighlightScala() {
+HighlightScala::HighlightScala()
+{
}
-HighlightScala::~HighlightScala() {
+HighlightScala::~HighlightScala()
+{
}
-void HighlightScala::feed(std::string *r, std::string_view input) {
+void
+HighlightScala::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -95,7 +100,8 @@ void HighlightScala::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_java_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_java_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -334,7 +340,9 @@ void HighlightScala::feed(std::string *r, std::string_view input) {
}
}
-void HighlightScala::flush(std::string *r) {
+void
+HighlightScala::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_scala(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_shell.cpp b/llamafile/highlight/highlight_shell.cpp
similarity index 98%
rename from llamafile/highlight_shell.cpp
rename to llamafile/highlight/highlight_shell.cpp
index 31bff95f..20669e8c 100644
--- a/llamafile/highlight_shell.cpp
+++ b/llamafile/highlight/highlight_shell.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -50,13 +51,17 @@ enum {
BACKSLASH,
};
-HighlightShell::HighlightShell() {
+HighlightShell::HighlightShell()
+{
}
-HighlightShell::~HighlightShell() {
+HighlightShell::~HighlightShell()
+{
}
-void HighlightShell::feed(std::string *r, std::string_view input) {
+void
+HighlightShell::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -139,7 +144,8 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
*r += HI_KEYWORD;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_shell_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_shell_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
@@ -507,7 +513,9 @@ void HighlightShell::feed(std::string *r, std::string_view input) {
}
}
-void HighlightShell::flush(std::string *r) {
+void
+HighlightShell::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_shell(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_sql.cpp b/llamafile/highlight/highlight_sql.cpp
similarity index 96%
rename from llamafile/highlight_sql.cpp
rename to llamafile/highlight/highlight_sql.cpp
index 8621b1cd..01ca9ca6 100644
--- a/llamafile/highlight_sql.cpp
+++ b/llamafile/highlight/highlight_sql.cpp
@@ -19,7 +19,8 @@
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -33,13 +34,17 @@ enum {
SLASH_STAR_STAR,
};
-HighlightSql::HighlightSql() {
+HighlightSql::HighlightSql()
+{
}
-HighlightSql::~HighlightSql() {
+HighlightSql::~HighlightSql()
+{
}
-void HighlightSql::feed(std::string *r, std::string_view input) {
+void
+HighlightSql::feed(std::string* r, std::string_view input)
+{
int c;
for (size_t i = 0; i < input.size(); ++i) {
c = input[i] & 255;
@@ -173,7 +178,9 @@ void HighlightSql::feed(std::string *r, std::string_view input) {
}
}
-void HighlightSql::flush(std::string *r) {
+void
+HighlightSql::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_sql(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_swift.cpp b/llamafile/highlight/highlight_swift.cpp
similarity index 97%
rename from llamafile/highlight_swift.cpp
rename to llamafile/highlight/highlight_swift.cpp
index 8af3d2d1..73b31a69 100644
--- a/llamafile/highlight_swift.cpp
+++ b/llamafile/highlight/highlight_swift.cpp
@@ -16,7 +16,7 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
@@ -34,7 +34,8 @@
// ##"""hi \##(2 * (2 + 2)) there"""##
//
-enum {
+enum
+{
NORMAL,
WORD,
SLASH,
@@ -58,18 +59,23 @@ enum {
REGEX_BACKSLASH,
};
-enum {
+enum
+{
EXPECT_VALUE,
EXPECT_OPERATOR,
};
-HighlightSwift::HighlightSwift() {
+HighlightSwift::HighlightSwift()
+{
}
-HighlightSwift::~HighlightSwift() {
+HighlightSwift::~HighlightSwift()
+{
}
-void HighlightSwift::feed(std::string *r, std::string_view input) {
+void
+HighlightSwift::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -148,11 +154,13 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_swift_builtin(word_.data(), word_.size())) {
+ } else if (is_keyword_swift_builtin(word_.data(),
+ word_.size())) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_swift_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_swift_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -415,7 +423,9 @@ void HighlightSwift::feed(std::string *r, std::string_view input) {
}
}
-void HighlightSwift::flush(std::string *r) {
+void
+HighlightSwift::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_swift(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_tcl.cpp b/llamafile/highlight/highlight_tcl.cpp
similarity index 96%
rename from llamafile/highlight_tcl.cpp
rename to llamafile/highlight/highlight_tcl.cpp
index 1da1f9e0..beecff7f 100644
--- a/llamafile/highlight_tcl.cpp
+++ b/llamafile/highlight/highlight_tcl.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
DQUOTE,
@@ -33,13 +34,17 @@ enum {
BACKSLASH,
};
-HighlightTcl::HighlightTcl() {
+HighlightTcl::HighlightTcl()
+{
}
-HighlightTcl::~HighlightTcl() {
+HighlightTcl::~HighlightTcl()
+{
}
-void HighlightTcl::feed(std::string *r, std::string_view input) {
+void
+HighlightTcl::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -187,7 +192,9 @@ void HighlightTcl::feed(std::string *r, std::string_view input) {
}
}
-void HighlightTcl::flush(std::string *r) {
+void
+HighlightTcl::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_tcl(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_test.cpp b/llamafile/highlight/highlight_test.cpp
similarity index 80%
rename from llamafile/highlight_test.cpp
rename to llamafile/highlight/highlight_test.cpp
index 7100585b..078e4ed7 100644
--- a/llamafile/highlight_test.cpp
+++ b/llamafile/highlight/highlight_test.cpp
@@ -16,7 +16,6 @@
// limitations under the License.
#include "highlight.h"
-#include "macros.h"
#include <ctype.h>
#include <stdio.h>
@@ -27,7 +26,7 @@
#define ITERATIONS 200000
#define CHARSET " raQq123{}[]!@#$%^*().\"'`\\/\n-_=&;:<>,"
-const char *const kLanguages[] = {
+const char* const kLanguages[] = {
"ada", //
"asm", //
"basic", //
@@ -75,7 +74,9 @@ const char *const kLanguages[] = {
"zig", //
};
-std::string generate_random_string(int n) {
+std::string
+generate_random_string(int n)
+{
std::string s;
s.reserve(n);
for (int i = 0; i < n; ++i)
@@ -83,7 +84,9 @@ std::string generate_random_string(int n) {
return s;
}
-std::string remove_ansi_sgr_codes(const std::string &input) {
+std::string
+remove_ansi_sgr_codes(const std::string& input)
+{
std::string result;
result.reserve(input.length());
bool in_escape_sequence = false;
@@ -100,7 +103,9 @@ std::string remove_ansi_sgr_codes(const std::string &input) {
return result;
}
-bool is_color_reset(const std::string &input) {
+bool
+is_color_reset(const std::string& input)
+{
int t = 0;
int number = 0;
bool has_color = false;
@@ -141,9 +146,11 @@ bool is_color_reset(const std::string &input) {
return !has_color;
}
-int main(int argc, char *argv[]) {
- for (int l = 0; l < ARRAYLEN(kLanguages); ++l) {
- Highlight *h = Highlight::create(kLanguages[l]);
+int
+main(int argc, char* argv[])
+{
+ for (int l = 0; l < sizeof(kLanguages) / sizeof(*kLanguages); ++l) {
+ Highlight* h = Highlight::create(kLanguages[l]);
if (!h) {
fprintf(stderr, "Highlight::create(%`'s) failed\n", kLanguages[l]);
exit(1);
@@ -154,14 +161,22 @@ int main(int argc, char *argv[]) {
h->feed(&colorized, sauce);
h->flush(&colorized);
if (!is_color_reset(colorized)) {
- fprintf(stderr, "%s highlight failed to reset color: %`'s -> %`'s\n", kLanguages[l],
- sauce.c_str(), colorized.c_str());
+ fprintf(stderr,
+ "%s highlight failed to reset color: %`'s -> %`'s\n",
+ kLanguages[l],
+ sauce.c_str(),
+ colorized.c_str());
exit(1);
}
std::string plain = remove_ansi_sgr_codes(colorized);
if (sauce != plain) {
- fprintf(stderr, "%s highlight failed to preserve code: %`'s -> %`'s -> %`'s\n",
- kLanguages[l], sauce.c_str(), colorized.c_str(), plain.c_str());
+ fprintf(stderr,
+ "%s highlight failed to preserve code: %`'s -> %`'s -> "
+ "%`'s\n",
+ kLanguages[l],
+ sauce.c_str(),
+ colorized.c_str(),
+ plain.c_str());
exit(1);
}
}
diff --git a/llamafile/highlight_tex.cpp b/llamafile/highlight/highlight_tex.cpp
similarity index 96%
rename from llamafile/highlight_tex.cpp
rename to llamafile/highlight/highlight_tex.cpp
index 55e6ae38..c41b2c22 100644
--- a/llamafile/highlight_tex.cpp
+++ b/llamafile/highlight/highlight_tex.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
BACKSLASH,
COMMAND,
@@ -33,13 +34,17 @@ enum {
STRING_QUOTE,
};
-HighlightTex::HighlightTex() {
+HighlightTex::HighlightTex()
+{
}
-HighlightTex::~HighlightTex() {
+HighlightTex::~HighlightTex()
+{
}
-void HighlightTex::feed(std::string *r, std::string_view input) {
+void
+HighlightTex::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -190,7 +195,9 @@ void HighlightTex::feed(std::string *r, std::string_view input) {
}
}
-void HighlightTex::flush(std::string *r) {
+void
+HighlightTex::flush(std::string* r)
+{
switch (t_) {
case BACKTICK:
*r += '`';
diff --git a/llamafile/highlight_txt.cpp b/llamafile/highlight/highlight_txt.cpp
similarity index 83%
rename from llamafile/highlight_txt.cpp
rename to llamafile/highlight/highlight_txt.cpp
index 18d5e390..10e85a7f 100644
--- a/llamafile/highlight_txt.cpp
+++ b/llamafile/highlight/highlight_txt.cpp
@@ -17,16 +17,22 @@
#include "highlight.h"
-HighlightTxt::HighlightTxt() {
+HighlightTxt::HighlightTxt()
+{
}
-HighlightTxt::~HighlightTxt() {
+HighlightTxt::~HighlightTxt()
+{
}
-void HighlightTxt::feed(std::string *r, std::string_view input) {
+void
+HighlightTxt::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i)
*r += input[i] & 255;
}
-void HighlightTxt::flush(std::string *r) {
+void
+HighlightTxt::flush(std::string* r)
+{
}
diff --git a/llamafile/highlight_typescript.cpp b/llamafile/highlight/highlight_typescript.cpp
similarity index 96%
rename from llamafile/highlight_typescript.cpp
rename to llamafile/highlight/highlight_typescript.cpp
index a6214306..9db3b35c 100644
--- a/llamafile/highlight_typescript.cpp
+++ b/llamafile/highlight/highlight_typescript.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -40,18 +41,23 @@ enum {
REGEX_SQUARE_BACKSLASH,
};
-enum {
+enum
+{
EXPECT_VALUE,
EXPECT_OPERATOR,
};
-HighlightTypescript::HighlightTypescript() {
+HighlightTypescript::HighlightTypescript()
+{
}
-HighlightTypescript::~HighlightTypescript() {
+HighlightTypescript::~HighlightTypescript()
+{
}
-void HighlightTypescript::feed(std::string *r, std::string_view input) {
+void
+HighlightTypescript::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -129,7 +135,8 @@ void HighlightTypescript::feed(std::string *r, std::string_view input) {
*r += word_;
*r += HI_RESET;
expect_ = EXPECT_VALUE;
- } else if (is_keyword_typescript_type(word_.data(), word_.size())) {
+ } else if (is_keyword_typescript_type(word_.data(),
+ word_.size())) {
*r += HI_TYPE;
*r += word_;
*r += HI_RESET;
@@ -316,7 +323,9 @@ void HighlightTypescript::feed(std::string *r, std::string_view input) {
}
}
-void HighlightTypescript::flush(std::string *r) {
+void
+HighlightTypescript::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_typescript(word_.data(), word_.size())) {
diff --git a/llamafile/highlight_zig.cpp b/llamafile/highlight/highlight_zig.cpp
similarity index 95%
rename from llamafile/highlight_zig.cpp
rename to llamafile/highlight/highlight_zig.cpp
index c251870a..4f2ef9ca 100644
--- a/llamafile/highlight_zig.cpp
+++ b/llamafile/highlight/highlight_zig.cpp
@@ -16,11 +16,12 @@
// limitations under the License.
#include "highlight.h"
-#include "string.h"
+#include "util.h"
#include <cosmo.h>
#include <ctype.h>
-enum {
+enum
+{
NORMAL,
WORD,
QUOTE,
@@ -33,13 +34,17 @@ enum {
BACKSLASH_BACKSLASH,
};
-HighlightZig::HighlightZig() {
+HighlightZig::HighlightZig()
+{
}
-HighlightZig::~HighlightZig() {
+HighlightZig::~HighlightZig()
+{
}
-void HighlightZig::feed(std::string *r, std::string_view input) {
+void
+HighlightZig::feed(std::string* r, std::string_view input)
+{
for (size_t i = 0; i < input.size(); ++i) {
wchar_t c;
int b = input[i] & 255;
@@ -101,7 +106,8 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
*r += HI_BUILTIN;
*r += word_;
*r += HI_RESET;
- } else if (is_keyword_zig_constant(word_.data(), word_.size())) {
+ } else if (is_keyword_zig_constant(word_.data(),
+ word_.size())) {
*r += HI_CONSTANT;
*r += word_;
*r += HI_RESET;
@@ -183,7 +189,9 @@ void HighlightZig::feed(std::string *r, std::string_view input) {
}
}
-void HighlightZig::flush(std::string *r) {
+void
+HighlightZig::flush(std::string* r)
+{
switch (t_) {
case WORD:
if (is_keyword_zig(word_.data(), word_.size())) {
diff --git a/llamafile/is_keyword_ada.gperf b/llamafile/highlight/is_keyword_ada.gperf
similarity index 100%
rename from llamafile/is_keyword_ada.gperf
rename to llamafile/highlight/is_keyword_ada.gperf
diff --git a/llamafile/is_keyword_ada_constant.gperf b/llamafile/highlight/is_keyword_ada_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_ada_constant.gperf
rename to llamafile/highlight/is_keyword_ada_constant.gperf
diff --git a/llamafile/is_keyword_asm_prefix.gperf b/llamafile/highlight/is_keyword_asm_prefix.gperf
similarity index 100%
rename from llamafile/is_keyword_asm_prefix.gperf
rename to llamafile/highlight/is_keyword_asm_prefix.gperf
diff --git a/llamafile/is_keyword_asm_qualifier.gperf b/llamafile/highlight/is_keyword_asm_qualifier.gperf
similarity index 100%
rename from llamafile/is_keyword_asm_qualifier.gperf
rename to llamafile/highlight/is_keyword_asm_qualifier.gperf
diff --git a/llamafile/is_keyword_basic.gperf b/llamafile/highlight/is_keyword_basic.gperf
similarity index 100%
rename from llamafile/is_keyword_basic.gperf
rename to llamafile/highlight/is_keyword_basic.gperf
diff --git a/llamafile/is_keyword_basic_builtin.gperf b/llamafile/highlight/is_keyword_basic_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_basic_builtin.gperf
rename to llamafile/highlight/is_keyword_basic_builtin.gperf
diff --git a/llamafile/is_keyword_basic_constant.gperf b/llamafile/highlight/is_keyword_basic_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_basic_constant.gperf
rename to llamafile/highlight/is_keyword_basic_constant.gperf
diff --git a/llamafile/is_keyword_basic_type.gperf b/llamafile/highlight/is_keyword_basic_type.gperf
similarity index 100%
rename from llamafile/is_keyword_basic_type.gperf
rename to llamafile/highlight/is_keyword_basic_type.gperf
diff --git a/llamafile/is_keyword_c.gperf b/llamafile/highlight/is_keyword_c.gperf
similarity index 100%
rename from llamafile/is_keyword_c.gperf
rename to llamafile/highlight/is_keyword_c.gperf
diff --git a/llamafile/is_keyword_c_builtin.gperf b/llamafile/highlight/is_keyword_c_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_c_builtin.gperf
rename to llamafile/highlight/is_keyword_c_builtin.gperf
diff --git a/llamafile/is_keyword_c_constant.gperf b/llamafile/highlight/is_keyword_c_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_c_constant.gperf
rename to llamafile/highlight/is_keyword_c_constant.gperf
diff --git a/llamafile/is_keyword_c_pod.gperf b/llamafile/highlight/is_keyword_c_pod.gperf
similarity index 100%
rename from llamafile/is_keyword_c_pod.gperf
rename to llamafile/highlight/is_keyword_c_pod.gperf
diff --git a/llamafile/is_keyword_c_type.gperf b/llamafile/highlight/is_keyword_c_type.gperf
similarity index 100%
rename from llamafile/is_keyword_c_type.gperf
rename to llamafile/highlight/is_keyword_c_type.gperf
diff --git a/llamafile/is_keyword_cmake.gperf b/llamafile/highlight/is_keyword_cmake.gperf
similarity index 100%
rename from llamafile/is_keyword_cmake.gperf
rename to llamafile/highlight/is_keyword_cmake.gperf
diff --git a/llamafile/is_keyword_cobol.gperf b/llamafile/highlight/is_keyword_cobol.gperf
similarity index 100%
rename from llamafile/is_keyword_cobol.gperf
rename to llamafile/highlight/is_keyword_cobol.gperf
diff --git a/llamafile/is_keyword_cpp.gperf b/llamafile/highlight/is_keyword_cpp.gperf
similarity index 100%
rename from llamafile/is_keyword_cpp.gperf
rename to llamafile/highlight/is_keyword_cpp.gperf
diff --git a/llamafile/is_keyword_csharp.gperf b/llamafile/highlight/is_keyword_csharp.gperf
similarity index 100%
rename from llamafile/is_keyword_csharp.gperf
rename to llamafile/highlight/is_keyword_csharp.gperf
diff --git a/llamafile/is_keyword_csharp_constant.gperf b/llamafile/highlight/is_keyword_csharp_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_csharp_constant.gperf
rename to llamafile/highlight/is_keyword_csharp_constant.gperf
diff --git a/llamafile/is_keyword_css_at.gperf b/llamafile/highlight/is_keyword_css_at.gperf
similarity index 100%
rename from llamafile/is_keyword_css_at.gperf
rename to llamafile/highlight/is_keyword_css_at.gperf
diff --git a/llamafile/is_keyword_css_bang.gperf b/llamafile/highlight/is_keyword_css_bang.gperf
similarity index 100%
rename from llamafile/is_keyword_css_bang.gperf
rename to llamafile/highlight/is_keyword_css_bang.gperf
diff --git a/llamafile/is_keyword_cxx.gperf b/llamafile/highlight/is_keyword_cxx.gperf
similarity index 100%
rename from llamafile/is_keyword_cxx.gperf
rename to llamafile/highlight/is_keyword_cxx.gperf
diff --git a/llamafile/is_keyword_d.gperf b/llamafile/highlight/is_keyword_d.gperf
similarity index 100%
rename from llamafile/is_keyword_d.gperf
rename to llamafile/highlight/is_keyword_d.gperf
diff --git a/llamafile/is_keyword_d_constant.gperf b/llamafile/highlight/is_keyword_d_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_d_constant.gperf
rename to llamafile/highlight/is_keyword_d_constant.gperf
diff --git a/llamafile/is_keyword_forth.gperf b/llamafile/highlight/is_keyword_forth.gperf
similarity index 100%
rename from llamafile/is_keyword_forth.gperf
rename to llamafile/highlight/is_keyword_forth.gperf
diff --git a/llamafile/is_keyword_forth_def.gperf b/llamafile/highlight/is_keyword_forth_def.gperf
similarity index 100%
rename from llamafile/is_keyword_forth_def.gperf
rename to llamafile/highlight/is_keyword_forth_def.gperf
diff --git a/llamafile/is_keyword_fortran.gperf b/llamafile/highlight/is_keyword_fortran.gperf
similarity index 100%
rename from llamafile/is_keyword_fortran.gperf
rename to llamafile/highlight/is_keyword_fortran.gperf
diff --git a/llamafile/is_keyword_fortran_builtin.gperf b/llamafile/highlight/is_keyword_fortran_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_fortran_builtin.gperf
rename to llamafile/highlight/is_keyword_fortran_builtin.gperf
diff --git a/llamafile/is_keyword_fortran_type.gperf b/llamafile/highlight/is_keyword_fortran_type.gperf
similarity index 100%
rename from llamafile/is_keyword_fortran_type.gperf
rename to llamafile/highlight/is_keyword_fortran_type.gperf
diff --git a/llamafile/is_keyword_go.gperf b/llamafile/highlight/is_keyword_go.gperf
similarity index 100%
rename from llamafile/is_keyword_go.gperf
rename to llamafile/highlight/is_keyword_go.gperf
diff --git a/llamafile/is_keyword_go_type.gperf b/llamafile/highlight/is_keyword_go_type.gperf
similarity index 100%
rename from llamafile/is_keyword_go_type.gperf
rename to llamafile/highlight/is_keyword_go_type.gperf
diff --git a/llamafile/is_keyword_haskell.gperf b/llamafile/highlight/is_keyword_haskell.gperf
similarity index 100%
rename from llamafile/is_keyword_haskell.gperf
rename to llamafile/highlight/is_keyword_haskell.gperf
diff --git a/llamafile/is_keyword_java.gperf b/llamafile/highlight/is_keyword_java.gperf
similarity index 100%
rename from llamafile/is_keyword_java.gperf
rename to llamafile/highlight/is_keyword_java.gperf
diff --git a/llamafile/is_keyword_java_constant.gperf b/llamafile/highlight/is_keyword_java_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_java_constant.gperf
rename to llamafile/highlight/is_keyword_java_constant.gperf
diff --git a/llamafile/is_keyword_js.gperf b/llamafile/highlight/is_keyword_js.gperf
similarity index 100%
rename from llamafile/is_keyword_js.gperf
rename to llamafile/highlight/is_keyword_js.gperf
diff --git a/llamafile/is_keyword_js_builtin.gperf b/llamafile/highlight/is_keyword_js_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_js_builtin.gperf
rename to llamafile/highlight/is_keyword_js_builtin.gperf
diff --git a/llamafile/is_keyword_js_constant.gperf b/llamafile/highlight/is_keyword_js_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_js_constant.gperf
rename to llamafile/highlight/is_keyword_js_constant.gperf
diff --git a/llamafile/is_keyword_julia.gperf b/llamafile/highlight/is_keyword_julia.gperf
similarity index 100%
rename from llamafile/is_keyword_julia.gperf
rename to llamafile/highlight/is_keyword_julia.gperf
diff --git a/llamafile/is_keyword_kotlin.gperf b/llamafile/highlight/is_keyword_kotlin.gperf
similarity index 100%
rename from llamafile/is_keyword_kotlin.gperf
rename to llamafile/highlight/is_keyword_kotlin.gperf
diff --git a/llamafile/is_keyword_ld.gperf b/llamafile/highlight/is_keyword_ld.gperf
similarity index 100%
rename from llamafile/is_keyword_ld.gperf
rename to llamafile/highlight/is_keyword_ld.gperf
diff --git a/llamafile/is_keyword_ld_builtin.gperf b/llamafile/highlight/is_keyword_ld_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_ld_builtin.gperf
rename to llamafile/highlight/is_keyword_ld_builtin.gperf
diff --git a/llamafile/is_keyword_ld_warning.gperf b/llamafile/highlight/is_keyword_ld_warning.gperf
similarity index 100%
rename from llamafile/is_keyword_ld_warning.gperf
rename to llamafile/highlight/is_keyword_ld_warning.gperf
diff --git a/llamafile/is_keyword_lisp.gperf b/llamafile/highlight/is_keyword_lisp.gperf
similarity index 100%
rename from llamafile/is_keyword_lisp.gperf
rename to llamafile/highlight/is_keyword_lisp.gperf
diff --git a/llamafile/is_keyword_lua.gperf b/llamafile/highlight/is_keyword_lua.gperf
similarity index 100%
rename from llamafile/is_keyword_lua.gperf
rename to llamafile/highlight/is_keyword_lua.gperf
diff --git a/llamafile/is_keyword_lua_builtin.gperf b/llamafile/highlight/is_keyword_lua_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_lua_builtin.gperf
rename to llamafile/highlight/is_keyword_lua_builtin.gperf
diff --git a/llamafile/is_keyword_lua_constant.gperf b/llamafile/highlight/is_keyword_lua_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_lua_constant.gperf
rename to llamafile/highlight/is_keyword_lua_constant.gperf
diff --git a/llamafile/is_keyword_m4.gperf b/llamafile/highlight/is_keyword_m4.gperf
similarity index 100%
rename from llamafile/is_keyword_m4.gperf
rename to llamafile/highlight/is_keyword_m4.gperf
diff --git a/llamafile/is_keyword_make.gperf b/llamafile/highlight/is_keyword_make.gperf
similarity index 100%
rename from llamafile/is_keyword_make.gperf
rename to llamafile/highlight/is_keyword_make.gperf
diff --git a/llamafile/is_keyword_make_builtin.gperf b/llamafile/highlight/is_keyword_make_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_make_builtin.gperf
rename to llamafile/highlight/is_keyword_make_builtin.gperf
diff --git a/llamafile/is_keyword_matlab.gperf b/llamafile/highlight/is_keyword_matlab.gperf
similarity index 100%
rename from llamafile/is_keyword_matlab.gperf
rename to llamafile/highlight/is_keyword_matlab.gperf
diff --git a/llamafile/is_keyword_matlab_builtin.gperf b/llamafile/highlight/is_keyword_matlab_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_matlab_builtin.gperf
rename to llamafile/highlight/is_keyword_matlab_builtin.gperf
diff --git a/llamafile/is_keyword_matlab_constant.gperf b/llamafile/highlight/is_keyword_matlab_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_matlab_constant.gperf
rename to llamafile/highlight/is_keyword_matlab_constant.gperf
diff --git a/llamafile/is_keyword_ocaml.gperf b/llamafile/highlight/is_keyword_ocaml.gperf
similarity index 100%
rename from llamafile/is_keyword_ocaml.gperf
rename to llamafile/highlight/is_keyword_ocaml.gperf
diff --git a/llamafile/is_keyword_ocaml_builtin.gperf b/llamafile/highlight/is_keyword_ocaml_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_ocaml_builtin.gperf
rename to llamafile/highlight/is_keyword_ocaml_builtin.gperf
diff --git a/llamafile/is_keyword_ocaml_constant.gperf b/llamafile/highlight/is_keyword_ocaml_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_ocaml_constant.gperf
rename to llamafile/highlight/is_keyword_ocaml_constant.gperf
diff --git a/llamafile/is_keyword_pascal.gperf b/llamafile/highlight/is_keyword_pascal.gperf
similarity index 100%
rename from llamafile/is_keyword_pascal.gperf
rename to llamafile/highlight/is_keyword_pascal.gperf
diff --git a/llamafile/is_keyword_pascal_builtin.gperf b/llamafile/highlight/is_keyword_pascal_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_pascal_builtin.gperf
rename to llamafile/highlight/is_keyword_pascal_builtin.gperf
diff --git a/llamafile/is_keyword_pascal_type.gperf b/llamafile/highlight/is_keyword_pascal_type.gperf
similarity index 100%
rename from llamafile/is_keyword_pascal_type.gperf
rename to llamafile/highlight/is_keyword_pascal_type.gperf
diff --git a/llamafile/is_keyword_perl.gperf b/llamafile/highlight/is_keyword_perl.gperf
similarity index 100%
rename from llamafile/is_keyword_perl.gperf
rename to llamafile/highlight/is_keyword_perl.gperf
diff --git a/llamafile/is_keyword_php.gperf b/llamafile/highlight/is_keyword_php.gperf
similarity index 100%
rename from llamafile/is_keyword_php.gperf
rename to llamafile/highlight/is_keyword_php.gperf
diff --git a/llamafile/is_keyword_php_constant.gperf b/llamafile/highlight/is_keyword_php_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_php_constant.gperf
rename to llamafile/highlight/is_keyword_php_constant.gperf
diff --git a/llamafile/is_keyword_python.gperf b/llamafile/highlight/is_keyword_python.gperf
similarity index 100%
rename from llamafile/is_keyword_python.gperf
rename to llamafile/highlight/is_keyword_python.gperf
diff --git a/llamafile/is_keyword_python_builtin.gperf b/llamafile/highlight/is_keyword_python_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_python_builtin.gperf
rename to llamafile/highlight/is_keyword_python_builtin.gperf
diff --git a/llamafile/is_keyword_python_constant.gperf b/llamafile/highlight/is_keyword_python_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_python_constant.gperf
rename to llamafile/highlight/is_keyword_python_constant.gperf
diff --git a/llamafile/is_keyword_r.gperf b/llamafile/highlight/is_keyword_r.gperf
similarity index 100%
rename from llamafile/is_keyword_r.gperf
rename to llamafile/highlight/is_keyword_r.gperf
diff --git a/llamafile/is_keyword_r_builtin.gperf b/llamafile/highlight/is_keyword_r_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_r_builtin.gperf
rename to llamafile/highlight/is_keyword_r_builtin.gperf
diff --git a/llamafile/is_keyword_r_constant.gperf b/llamafile/highlight/is_keyword_r_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_r_constant.gperf
rename to llamafile/highlight/is_keyword_r_constant.gperf
diff --git a/llamafile/is_keyword_ruby.gperf b/llamafile/highlight/is_keyword_ruby.gperf
similarity index 100%
rename from llamafile/is_keyword_ruby.gperf
rename to llamafile/highlight/is_keyword_ruby.gperf
diff --git a/llamafile/is_keyword_ruby_builtin.gperf b/llamafile/highlight/is_keyword_ruby_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_ruby_builtin.gperf
rename to llamafile/highlight/is_keyword_ruby_builtin.gperf
diff --git a/llamafile/is_keyword_ruby_constant.gperf b/llamafile/highlight/is_keyword_ruby_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_ruby_constant.gperf
rename to llamafile/highlight/is_keyword_ruby_constant.gperf
diff --git a/llamafile/is_keyword_rust.gperf b/llamafile/highlight/is_keyword_rust.gperf
similarity index 100%
rename from llamafile/is_keyword_rust.gperf
rename to llamafile/highlight/is_keyword_rust.gperf
diff --git a/llamafile/is_keyword_rust_constant.gperf b/llamafile/highlight/is_keyword_rust_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_rust_constant.gperf
rename to llamafile/highlight/is_keyword_rust_constant.gperf
diff --git a/llamafile/is_keyword_rust_type.gperf b/llamafile/highlight/is_keyword_rust_type.gperf
similarity index 100%
rename from llamafile/is_keyword_rust_type.gperf
rename to llamafile/highlight/is_keyword_rust_type.gperf
diff --git a/llamafile/is_keyword_scala.gperf b/llamafile/highlight/is_keyword_scala.gperf
similarity index 100%
rename from llamafile/is_keyword_scala.gperf
rename to llamafile/highlight/is_keyword_scala.gperf
diff --git a/llamafile/is_keyword_shell.gperf b/llamafile/highlight/is_keyword_shell.gperf
similarity index 100%
rename from llamafile/is_keyword_shell.gperf
rename to llamafile/highlight/is_keyword_shell.gperf
diff --git a/llamafile/is_keyword_shell_builtin.gperf b/llamafile/highlight/is_keyword_shell_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_shell_builtin.gperf
rename to llamafile/highlight/is_keyword_shell_builtin.gperf
diff --git a/llamafile/is_keyword_sql.gperf b/llamafile/highlight/is_keyword_sql.gperf
similarity index 100%
rename from llamafile/is_keyword_sql.gperf
rename to llamafile/highlight/is_keyword_sql.gperf
diff --git a/llamafile/is_keyword_sql_type.gperf b/llamafile/highlight/is_keyword_sql_type.gperf
similarity index 100%
rename from llamafile/is_keyword_sql_type.gperf
rename to llamafile/highlight/is_keyword_sql_type.gperf
diff --git a/llamafile/is_keyword_swift.gperf b/llamafile/highlight/is_keyword_swift.gperf
similarity index 100%
rename from llamafile/is_keyword_swift.gperf
rename to llamafile/highlight/is_keyword_swift.gperf
diff --git a/llamafile/is_keyword_swift_builtin.gperf b/llamafile/highlight/is_keyword_swift_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_swift_builtin.gperf
rename to llamafile/highlight/is_keyword_swift_builtin.gperf
diff --git a/llamafile/is_keyword_swift_constant.gperf b/llamafile/highlight/is_keyword_swift_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_swift_constant.gperf
rename to llamafile/highlight/is_keyword_swift_constant.gperf
diff --git a/llamafile/is_keyword_swift_type.gperf b/llamafile/highlight/is_keyword_swift_type.gperf
similarity index 100%
rename from llamafile/is_keyword_swift_type.gperf
rename to llamafile/highlight/is_keyword_swift_type.gperf
diff --git a/llamafile/is_keyword_tcl.gperf b/llamafile/highlight/is_keyword_tcl.gperf
similarity index 100%
rename from llamafile/is_keyword_tcl.gperf
rename to llamafile/highlight/is_keyword_tcl.gperf
diff --git a/llamafile/is_keyword_tcl_builtin.gperf b/llamafile/highlight/is_keyword_tcl_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_tcl_builtin.gperf
rename to llamafile/highlight/is_keyword_tcl_builtin.gperf
diff --git a/llamafile/is_keyword_tcl_type.gperf b/llamafile/highlight/is_keyword_tcl_type.gperf
similarity index 100%
rename from llamafile/is_keyword_tcl_type.gperf
rename to llamafile/highlight/is_keyword_tcl_type.gperf
diff --git a/llamafile/is_keyword_typescript.gperf b/llamafile/highlight/is_keyword_typescript.gperf
similarity index 100%
rename from llamafile/is_keyword_typescript.gperf
rename to llamafile/highlight/is_keyword_typescript.gperf
diff --git a/llamafile/is_keyword_typescript_type.gperf b/llamafile/highlight/is_keyword_typescript_type.gperf
similarity index 100%
rename from llamafile/is_keyword_typescript_type.gperf
rename to llamafile/highlight/is_keyword_typescript_type.gperf
diff --git a/llamafile/is_keyword_zig.gperf b/llamafile/highlight/is_keyword_zig.gperf
similarity index 100%
rename from llamafile/is_keyword_zig.gperf
rename to llamafile/highlight/is_keyword_zig.gperf
diff --git a/llamafile/is_keyword_zig_builtin.gperf b/llamafile/highlight/is_keyword_zig_builtin.gperf
similarity index 100%
rename from llamafile/is_keyword_zig_builtin.gperf
rename to llamafile/highlight/is_keyword_zig_builtin.gperf
diff --git a/llamafile/is_keyword_zig_constant.gperf b/llamafile/highlight/is_keyword_zig_constant.gperf
similarity index 100%
rename from llamafile/is_keyword_zig_constant.gperf
rename to llamafile/highlight/is_keyword_zig_constant.gperf
diff --git a/llamafile/is_keyword_zig_type.gperf b/llamafile/highlight/is_keyword_zig_type.gperf
similarity index 100%
rename from llamafile/is_keyword_zig_type.gperf
rename to llamafile/highlight/is_keyword_zig_type.gperf
diff --git a/llamafile/highlight/util.cpp b/llamafile/highlight/util.cpp
new file mode 100644
index 00000000..71072815
--- /dev/null
+++ b/llamafile/highlight/util.cpp
@@ -0,0 +1,38 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util.h"
+#include <cctype>
+#include <cosmo.h>
+#include <string>
+
+namespace lf {
+
+void
+append_wchar(std::string* r, wchar_t c)
+{
+ if (isascii(c)) {
+ *r += c;
+ } else {
+ char s[8];
+ uint64_t w = tpenc(c);
+ WRITE64LE(s, w);
+ *r += s;
+ }
+}
+
+} // namespace lf
diff --git a/llamafile/highlight/util.h b/llamafile/highlight/util.h
new file mode 100644
index 00000000..0b25c94e
--- /dev/null
+++ b/llamafile/highlight/util.h
@@ -0,0 +1,26 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <__fwd/string.h>
+
+namespace lf {
+
+void
+append_wchar(std::string*, wchar_t);
+
+} // namespace lf
diff --git a/llamafile/image.cpp b/llamafile/image.cpp
index 63cb14b2..2d9ca22b 100644
--- a/llamafile/image.cpp
+++ b/llamafile/image.cpp
@@ -25,8 +25,8 @@
#include "llama.cpp/base64.h"
#include "llamafile/macros.h"
#include "llamafile/xterm.h"
-#include "stb/stb_image.h"
-#include "stb/stb_image_resize2.h"
+#include "third_party/stb/stb_image.h"
+#include "third_party/stb/stb_image_resize2.h"
namespace lf {
diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mk
index 54aa5eb0..a7f76ce7 100644
--- a/llamafile/server/BUILD.mk
+++ b/llamafile/server/BUILD.mk
@@ -12,18 +12,18 @@ LLAMAFILE_SERVER_ASSETS = $(wildcard llamafile/server/www/*)
$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g
-o/$(MODE)/llamafile/server/server.a: \
+o/$(MODE)/llamafile/server/server.a: \
$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))
-o/$(MODE)/llamafile/server/main: \
- o/$(MODE)/llamafile/server/main.o \
- o/$(MODE)/llamafile/server/main.1.asc.zip.o \
- o/$(MODE)/llamafile/server/server.a \
- o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/llama.cpp/llava/llava.a \
- o/$(MODE)/double-conversion/double-conversion.a \
- o/$(MODE)/stb/stb.a \
- $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
+o/$(MODE)/llamafile/server/main: \
+ o/$(MODE)/llamafile/server/main.o \
+ o/$(MODE)/llamafile/server/main.1.asc.zip.o \
+ o/$(MODE)/llamafile/server/server.a \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+ o/$(MODE)/llama.cpp/llava/llava.a \
+ o/$(MODE)/third_party/double-conversion/double-conversion.a \
+ o/$(MODE)/third_party/stb/stb.a \
+ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o) \
# turn /zip/llamafile/server/www/...
# into /zip/www/...
@@ -31,37 +31,37 @@ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o): private ZIPOBJ_FLAGS += -C2
$(LLAMAFILE_SERVER_OBJS): llamafile/server/BUILD.mk
-o/$(MODE)/llamafile/server/atom_test: \
- o/$(MODE)/llamafile/server/atom_test.o \
- o/$(MODE)/llamafile/server/atom.o \
- o/$(MODE)/llamafile/server/image.o \
+o/$(MODE)/llamafile/server/atom_test: \
+ o/$(MODE)/llamafile/server/atom_test.o \
+ o/$(MODE)/llamafile/server/atom.o \
+ o/$(MODE)/llamafile/server/image.o \
-o/$(MODE)/llamafile/server/image_test: \
- o/$(MODE)/llamafile/server/image_test.o \
- o/$(MODE)/llamafile/server/image.o \
+o/$(MODE)/llamafile/server/image_test: \
+ o/$(MODE)/llamafile/server/image_test.o \
+ o/$(MODE)/llamafile/server/image.o \
-o/$(MODE)/llamafile/server/fastjson_test: \
- o/$(MODE)/llamafile/server/fastjson_test.o \
- o/$(MODE)/llamafile/server/fastjson.o \
- o/$(MODE)/double-conversion/double-conversion.a \
+o/$(MODE)/llamafile/server/fastjson_test: \
+ o/$(MODE)/llamafile/server/fastjson_test.o \
+ o/$(MODE)/llamafile/server/fastjson.o \
+ o/$(MODE)/double-conversion/double-conversion.a \
-o/$(MODE)/llamafile/server/json_test: \
- o/$(MODE)/llamafile/server/json_test.o \
- o/$(MODE)/llamafile/server/json.o \
- o/$(MODE)/llamafile/server/hextoint.o \
- o/$(MODE)/double-conversion/double-conversion.a \
+o/$(MODE)/llamafile/server/json_test: \
+ o/$(MODE)/llamafile/server/json_test.o \
+ o/$(MODE)/llamafile/server/json.o \
+ o/$(MODE)/llamafile/server/hextoint.o \
+ o/$(MODE)/double-conversion/double-conversion.a \
-o/$(MODE)/llamafile/server/tokenbucket_test: \
- o/$(MODE)/llamafile/server/tokenbucket_test.o \
- o/$(MODE)/llamafile/server/tokenbucket.o \
- o/$(MODE)/llamafile/server/log.o \
- o/$(MODE)/llama.cpp/llama.cpp.a \
+o/$(MODE)/llamafile/server/tokenbucket_test: \
+ o/$(MODE)/llamafile/server/tokenbucket_test.o \
+ o/$(MODE)/llamafile/server/tokenbucket.o \
+ o/$(MODE)/llamafile/server/log.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
.PHONY: o/$(MODE)/llamafile/server
-o/$(MODE)/llamafile/server: \
- o/$(MODE)/llamafile/server/main \
- o/$(MODE)/llamafile/server/atom_test.runs \
- o/$(MODE)/llamafile/server/fastjson_test.runs \
- o/$(MODE)/llamafile/server/image_test.runs \
- o/$(MODE)/llamafile/server/json_test.runs \
- o/$(MODE)/llamafile/server/tokenbucket_test.runs \
+o/$(MODE)/llamafile/server: \
+ o/$(MODE)/llamafile/server/main \
+ o/$(MODE)/llamafile/server/atom_test.runs \
+ o/$(MODE)/llamafile/server/fastjson_test.runs \
+ o/$(MODE)/llamafile/server/image_test.runs \
+ o/$(MODE)/llamafile/server/json_test.runs \
+ o/$(MODE)/llamafile/server/tokenbucket_test.runs \
diff --git a/llamafile/server/fastjson.cpp b/llamafile/server/fastjson.cpp
index 62d46ce8..efb76e47 100644
--- a/llamafile/server/fastjson.cpp
+++ b/llamafile/server/fastjson.cpp
@@ -16,8 +16,8 @@
// limitations under the License.
#include "fastjson.h"
-#include "double-conversion/double-to-string.h"
-#include "double-conversion/utils.h"
+#include "third_party/double-conversion/double-to-string.h"
+#include "third_party/double-conversion/utils.h"
#include <cosmo.h>
#include <net/http/escape.h>
#include <string>
diff --git a/llamafile/server/json.cpp b/llamafile/server/json.cpp
index a5b03c3b..127b6074 100644
--- a/llamafile/server/json.cpp
+++ b/llamafile/server/json.cpp
@@ -24,8 +24,8 @@
#include <cstdlib>
#include <stdckdint.h>
-#include "double-conversion/double-to-string.h"
-#include "double-conversion/string-to-double.h"
+#include "third_party/double-conversion/double-to-string.h"
+#include "third_party/double-conversion/string-to-double.h"
#define KEY 1
#define COMMA 2
diff --git a/llamafile/string.cpp b/llamafile/string.cpp
index ef67d581..a975d430 100644
--- a/llamafile/string.cpp
+++ b/llamafile/string.cpp
@@ -52,17 +52,6 @@ bool startscasewith(const std::string_view &str, const std::string_view &prefix)
return true;
}
-void append_wchar(std::string *r, wchar_t c) {
- if (isascii(c)) {
- *r += c;
- } else {
- char s[8];
- uint64_t w = tpenc(c);
- WRITE64LE(s, w);
- *r += s;
- }
-}
-
std::string format(const char *fmt, ...) {
va_list ap, ap2;
va_start(ap, fmt);
diff --git a/stable-diffusion.cpp/BUILD.mk b/stable-diffusion.cpp/BUILD.mk
index 2a6c95f1..6e5465cf 100644
--- a/stable-diffusion.cpp/BUILD.mk
+++ b/stable-diffusion.cpp/BUILD.mk
@@ -30,7 +30,7 @@ o/$(MODE)/stable-diffusion.cpp/main: \
o/$(MODE)/stable-diffusion.cpp/main.o \
o/$(MODE)/stable-diffusion.cpp/stable-diffusion.cpp.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a
+ o/$(MODE)/third_party/stb/stb.a
$(STABLE_DIFFUSION_CPP_OBJS): stable-diffusion.cpp/BUILD.mk
diff --git a/stable-diffusion.cpp/main.cpp b/stable-diffusion.cpp/main.cpp
index 2b4c75a9..1d466053 100644
--- a/stable-diffusion.cpp/main.cpp
+++ b/stable-diffusion.cpp/main.cpp
@@ -15,15 +15,15 @@
// #define STB_IMAGE_IMPLEMENTATION
// #define STB_IMAGE_STATIC
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
// #define STB_IMAGE_WRITE_IMPLEMENTATION
// #define STB_IMAGE_WRITE_STATIC
-#include "stb/stb_image_write.h"
+#include "third_party/stb/stb_image_write.h"
// #define STB_IMAGE_RESIZE_IMPLEMENTATION
// #define STB_IMAGE_RESIZE_STATIC
-#include "stb/stb_image_resize2.h"
+#include "third_party/stb/stb_image_resize2.h"
#include "llamafile/llamafile.h"
#include "llamafile/debug.h"
diff --git a/stable-diffusion.cpp/stable-diffusion.cpp b/stable-diffusion.cpp/stable-diffusion.cpp
index 29dfd739..352adf1a 100644
--- a/stable-diffusion.cpp/stable-diffusion.cpp
+++ b/stable-diffusion.cpp/stable-diffusion.cpp
@@ -18,11 +18,11 @@
// #define STB_IMAGE_IMPLEMENTATION
// #define STB_IMAGE_STATIC
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
// #define STB_IMAGE_WRITE_IMPLEMENTATION
// #define STB_IMAGE_WRITE_STATIC
-// #include "stb_image_write.h"
+// #include "third_party/stb/stb_image_write.h"
const char* model_version_to_str[] = {
"1.x",
diff --git a/stable-diffusion.cpp/util.cpp b/stable-diffusion.cpp/util.cpp
index 806b25ec..7aa48631 100644
--- a/stable-diffusion.cpp/util.cpp
+++ b/stable-diffusion.cpp/util.cpp
@@ -25,7 +25,7 @@
#include "llama.cpp/ggml.h"
#include "stable-diffusion.h"
-#include "stb/stb_image_resize2.h"
+#include "third_party/stb/stb_image_resize2.h"
bool ends_with(const std::string& str, const std::string& ending) {
if (str.length() >= ending.length()) {
diff --git a/third_party/BUILD.mk b/third_party/BUILD.mk
new file mode 100644
index 00000000..cb4edaef
--- /dev/null
+++ b/third_party/BUILD.mk
@@ -0,0 +1,7 @@
+include third_party/double-conversion/BUILD.mk
+include third_party/stb/BUILD.mk
+
+.PHONY: o/$(MODE)/third_party
+o/$(MODE)/third_party: \
+ o/$(MODE)/third_party/double-conversion \
+ o/$(MODE)/third_party/stb \
diff --git a/double-conversion/.gitignore b/third_party/double-conversion/.gitignore
similarity index 100%
rename from double-conversion/.gitignore
rename to third_party/double-conversion/.gitignore
diff --git a/double-conversion/AUTHORS b/third_party/double-conversion/AUTHORS
similarity index 100%
rename from double-conversion/AUTHORS
rename to third_party/double-conversion/AUTHORS
diff --git a/double-conversion/BUILD.mk b/third_party/double-conversion/BUILD.mk
similarity index 52%
rename from double-conversion/BUILD.mk
rename to third_party/double-conversion/BUILD.mk
index 2aca9e7f..689d04f4 100644
--- a/double-conversion/BUILD.mk
+++ b/third_party/double-conversion/BUILD.mk
@@ -3,14 +3,15 @@
PKGS += DOUBLE_CONVERSION
-DOUBLE_CONVERSION_FILES := $(wildcard double-conversion/*)
+DOUBLE_CONVERSION_FILES := $(wildcard third_party/double-conversion/*)
DOUBLE_CONVERSION_HDRS = $(filter %.h,$(DOUBLE_CONVERSION_FILES))
DOUBLE_CONVERSION_SRCS = $(filter %.cc,$(DOUBLE_CONVERSION_FILES))
DOUBLE_CONVERSION_OBJS = $(DOUBLE_CONVERSION_SRCS:%.cc=o/$(MODE)/%.o)
-o/$(MODE)/double-conversion/double-conversion.a: $(DOUBLE_CONVERSION_OBJS)
+o/$(MODE)/third_party/double-conversion/double-conversion.a: $(DOUBLE_CONVERSION_OBJS)
-$(DOUBLE_CONVERSION_OBJS): double-conversion/BUILD.mk
+$(DOUBLE_CONVERSION_OBJS): third_party/double-conversion/BUILD.mk
-.PHONY: o/$(MODE)/double-conversion
-o/$(MODE)/double-conversion: o/$(MODE)/double-conversion/double-conversion.a
+.PHONY: o/$(MODE)/third_party/double-conversion
+o/$(MODE)/third_party/double-conversion: \
+ o/$(MODE)/third_party/double-conversion/double-conversion.a
diff --git a/double-conversion/LICENSE b/third_party/double-conversion/LICENSE
similarity index 100%
rename from double-conversion/LICENSE
rename to third_party/double-conversion/LICENSE
diff --git a/double-conversion/README.llamafile b/third_party/double-conversion/README.llamafile
similarity index 100%
rename from double-conversion/README.llamafile
rename to third_party/double-conversion/README.llamafile
diff --git a/double-conversion/SConscript b/third_party/double-conversion/SConscript
similarity index 100%
rename from double-conversion/SConscript
rename to third_party/double-conversion/SConscript
diff --git a/double-conversion/bignum-dtoa.cc b/third_party/double-conversion/bignum-dtoa.cc
similarity index 100%
rename from double-conversion/bignum-dtoa.cc
rename to third_party/double-conversion/bignum-dtoa.cc
diff --git a/double-conversion/bignum-dtoa.h b/third_party/double-conversion/bignum-dtoa.h
similarity index 100%
rename from double-conversion/bignum-dtoa.h
rename to third_party/double-conversion/bignum-dtoa.h
diff --git a/double-conversion/bignum.cc b/third_party/double-conversion/bignum.cc
similarity index 100%
rename from double-conversion/bignum.cc
rename to third_party/double-conversion/bignum.cc
diff --git a/double-conversion/bignum.h b/third_party/double-conversion/bignum.h
similarity index 100%
rename from double-conversion/bignum.h
rename to third_party/double-conversion/bignum.h
diff --git a/double-conversion/cached-powers.cc b/third_party/double-conversion/cached-powers.cc
similarity index 100%
rename from double-conversion/cached-powers.cc
rename to third_party/double-conversion/cached-powers.cc
diff --git a/double-conversion/cached-powers.h b/third_party/double-conversion/cached-powers.h
similarity index 100%
rename from double-conversion/cached-powers.h
rename to third_party/double-conversion/cached-powers.h
diff --git a/double-conversion/diy-fp.h b/third_party/double-conversion/diy-fp.h
similarity index 100%
rename from double-conversion/diy-fp.h
rename to third_party/double-conversion/diy-fp.h
diff --git a/double-conversion/double-conversion.h b/third_party/double-conversion/double-conversion.h
similarity index 100%
rename from double-conversion/double-conversion.h
rename to third_party/double-conversion/double-conversion.h
diff --git a/double-conversion/double-to-string.cc b/third_party/double-conversion/double-to-string.cc
similarity index 100%
rename from double-conversion/double-to-string.cc
rename to third_party/double-conversion/double-to-string.cc
diff --git a/double-conversion/double-to-string.h b/third_party/double-conversion/double-to-string.h
similarity index 100%
rename from double-conversion/double-to-string.h
rename to third_party/double-conversion/double-to-string.h
diff --git a/double-conversion/fast-dtoa.cc b/third_party/double-conversion/fast-dtoa.cc
similarity index 100%
rename from double-conversion/fast-dtoa.cc
rename to third_party/double-conversion/fast-dtoa.cc
diff --git a/double-conversion/fast-dtoa.h b/third_party/double-conversion/fast-dtoa.h
similarity index 100%
rename from double-conversion/fast-dtoa.h
rename to third_party/double-conversion/fast-dtoa.h
diff --git a/double-conversion/fixed-dtoa.cc b/third_party/double-conversion/fixed-dtoa.cc
similarity index 100%
rename from double-conversion/fixed-dtoa.cc
rename to third_party/double-conversion/fixed-dtoa.cc
diff --git a/double-conversion/fixed-dtoa.h b/third_party/double-conversion/fixed-dtoa.h
similarity index 100%
rename from double-conversion/fixed-dtoa.h
rename to third_party/double-conversion/fixed-dtoa.h
diff --git a/double-conversion/ieee.h b/third_party/double-conversion/ieee.h
similarity index 100%
rename from double-conversion/ieee.h
rename to third_party/double-conversion/ieee.h
diff --git a/double-conversion/string-to-double.cc b/third_party/double-conversion/string-to-double.cc
similarity index 100%
rename from double-conversion/string-to-double.cc
rename to third_party/double-conversion/string-to-double.cc
diff --git a/double-conversion/string-to-double.h b/third_party/double-conversion/string-to-double.h
similarity index 100%
rename from double-conversion/string-to-double.h
rename to third_party/double-conversion/string-to-double.h
diff --git a/double-conversion/strtod.cc b/third_party/double-conversion/strtod.cc
similarity index 100%
rename from double-conversion/strtod.cc
rename to third_party/double-conversion/strtod.cc
diff --git a/double-conversion/strtod.h b/third_party/double-conversion/strtod.h
similarity index 100%
rename from double-conversion/strtod.h
rename to third_party/double-conversion/strtod.h
diff --git a/double-conversion/utils.h b/third_party/double-conversion/utils.h
similarity index 100%
rename from double-conversion/utils.h
rename to third_party/double-conversion/utils.h
diff --git a/stb/BUILD.mk b/third_party/stb/BUILD.mk
similarity index 62%
rename from stb/BUILD.mk
rename to third_party/stb/BUILD.mk
index aa8c5be7..af276a6a 100644
--- a/stb/BUILD.mk
+++ b/third_party/stb/BUILD.mk
@@ -3,15 +3,15 @@
PKGS += STB
-STB_FILES := $(wildcard stb/*)
+STB_FILES := $(wildcard third_party/stb/*)
STB_HDRS = $(filter %.h,$(STB_FILES))
STB_INCS = $(filter %.inc,$(STB_FILES))
STB_SRCS = $(filter %.c,$(STB_FILES))
STB_OBJS = $(STB_SRCS:%.c=o/$(MODE)/%.o)
-o/$(MODE)/stb/stb.a: $(STB_OBJS)
+o/$(MODE)/third_party/stb/stb.a: $(STB_OBJS)
-$(STB_OBJS): stb/BUILD.mk
+$(STB_OBJS): third_party/stb/BUILD.mk
-.PHONY: o/$(MODE)/stb
-o/$(MODE)/stb: o/$(MODE)/stb/stb.a
+.PHONY: o/$(MODE)/third_party/stb
+o/$(MODE)/third_party/stb: o/$(MODE)/third_party/stb/stb.a
diff --git a/stb/README.llamafile b/third_party/stb/README.llamafile
similarity index 100%
rename from stb/README.llamafile
rename to third_party/stb/README.llamafile
diff --git a/stb/stb_image.c b/third_party/stb/stb_image.c
similarity index 100%
rename from stb/stb_image.c
rename to third_party/stb/stb_image.c
diff --git a/stb/stb_image.h b/third_party/stb/stb_image.h
similarity index 100%
rename from stb/stb_image.h
rename to third_party/stb/stb_image.h
diff --git a/stb/stb_image_resize2.c b/third_party/stb/stb_image_resize2.c
similarity index 100%
rename from stb/stb_image_resize2.c
rename to third_party/stb/stb_image_resize2.c
diff --git a/stb/stb_image_resize2.h b/third_party/stb/stb_image_resize2.h
similarity index 100%
rename from stb/stb_image_resize2.h
rename to third_party/stb/stb_image_resize2.h
diff --git a/stb/stb_image_write.c b/third_party/stb/stb_image_write.c
similarity index 100%
rename from stb/stb_image_write.c
rename to third_party/stb/stb_image_write.c
diff --git a/stb/stb_image_write.h b/third_party/stb/stb_image_write.h
similarity index 100%
rename from stb/stb_image_write.h
rename to third_party/stb/stb_image_write.h
diff --git a/stb/stb_vorbis.c b/third_party/stb/stb_vorbis.c
similarity index 100%
rename from stb/stb_vorbis.c
rename to third_party/stb/stb_vorbis.c
diff --git a/stb/stb_vorbis.h b/third_party/stb/stb_vorbis.h
similarity index 100%
rename from stb/stb_vorbis.h
rename to third_party/stb/stb_vorbis.h
diff --git a/whisper.cpp/BUILD.mk b/whisper.cpp/BUILD.mk
index 6e692c2a..af6aebc6 100644
--- a/whisper.cpp/BUILD.mk
+++ b/whisper.cpp/BUILD.mk
@@ -33,12 +33,12 @@ o/$(MODE)/whisper.cpp/main: \
o/$(MODE)/whisper.cpp/main.1.asc.zip.o \
o/$(MODE)/whisper.cpp/whisper.cpp.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a \
+ o/$(MODE)/third_party/stb/stb.a \
o/$(MODE)/whisper.cpp/stream: \
o/$(MODE)/whisper.cpp/whisper.cpp.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/stb/stb.a \
+ o/$(MODE)/third_party/stb/stb.a \
o/$(MODE)/whisper.cpp/mic2txt: \
o/$(MODE)/whisper.cpp/whisper.cpp.a \
diff --git a/whisper.cpp/common.cpp b/whisper.cpp/common.cpp
index 1492c148..06e9ed70 100644
--- a/whisper.cpp/common.cpp
+++ b/whisper.cpp/common.cpp
@@ -24,7 +24,7 @@
#include <stdlib.h>
#include <unistd.h>
-#include "stb/stb_vorbis.h"
+#include "third_party/stb/stb_vorbis.h"
#include "miniaudio.h"
void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
diff --git a/whisper.cpp/miniaudio.c b/whisper.cpp/miniaudio.c
index a5e285b3..5629fc99 100644
--- a/whisper.cpp/miniaudio.c
+++ b/whisper.cpp/miniaudio.c
@@ -1,4 +1,4 @@
-#include "stb/stb_vorbis.h"
+#include "third_party/stb/stb_vorbis.h"
#define MA_NO_DEVICE_IO
#define MA_NO_RUNTIME_LINKING