Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add PHP syntax highlighting
Justine Tunney2158c031dee0
diff --git a/llamafile/highlight.cpp b/llamafile/highlight.cppindex 91f6b0b5..cc257365 100644--- a/llamafile/highlight.cpp+++ b/llamafile/highlight.cpp@@ -29,23 +29,25 @@ Highlight *Highlight::create(const std::string_view &lang) {} else if (lang == "java") {return new HighlightC(is_keyword_java);} else if (lang == "py" || lang == "python") {- return new HighlightPython();+ return new HighlightPython;} else if (lang == "rs" || lang == "rust") {- return new HighlightRust();+ return new HighlightRust;} else if (lang == "f" || lang == "fortran") {- return new HighlightFortran();+ return new HighlightFortran;} else if (lang == "cob" || lang == "cbl" || lang == "cobol") {- return new HighlightCobol();+ return new HighlightCobol;} else if (lang == "pas" || lang == "pascal") {- return new HighlightPascal();+ return new HighlightPascal;} else if (lang == "go") {return new HighlightC(is_keyword_go, is_keyword_go_type);} else if (lang == "sql") {- return new HighlightSql();+ return new HighlightSql;} else if (lang == "css") {- return new HighlightCss();+ return new HighlightCss;} else if (lang == "html") {- return new HighlightHtml();+ return new HighlightHtml;+ } else if (lang == "php") {+ return new HighlightHtml; // sic} else {return new HighlightPlain;}diff --git a/llamafile/highlight.h b/llamafile/highlight.hindex b212942f..185a75bb 100644--- a/llamafile/highlight.h+++ b/llamafile/highlight.h@@ -24,6 +24,7 @@#define HI_KEYWORD "\033[1;34m" // bold blue#define HI_STRING "\033[32m" // green#define HI_COMMENT "\033[31m" // red+#define HI_VAR "\033[1;35m" // magenta#define HI_MACRO "\033[35m" // magenta#define HI_ATTRIB "\033[35m" // magenta#define HI_LINENO "\033[2m" // fade@@ -34,6 +35,7 @@#define HI_PROPERTY "\033[36m" // cyan#define HI_TAG "\033[33m" // yellow#define HI_INCODE "\033[1;35m" // magenta+#define HI_ENTITY "\033[36m" // cyantypedef const char *is_keyword_f(const char *, size_t);@@ -52,6 +54,7 @@ is_keyword_f is_keyword_pascal_type;is_keyword_f is_keyword_go;is_keyword_f is_keyword_go_type;is_keyword_f is_keyword_sql;+is_keyword_f is_keyword_php;}class Highlight {@@ -198,3 +201,15 @@ class HighlightHtml : public Highlight {std::string pending_;Highlight *highlighter_ = nullptr;};++class HighlightPhp : public Highlight {+ public:+ HighlightPhp();+ ~HighlightPhp() override;+ void feed(std::string *result, std::string_view input) override;+ void flush(std::string *result) override;++ private:+ int t_ = 0;+ std::string word_;+};diff --git a/llamafile/highlight_html.cpp b/llamafile/highlight_html.cppindex 7d209582..f0bed187 100644--- a/llamafile/highlight_html.cpp+++ b/llamafile/highlight_html.cpp@@ -30,6 +30,10 @@#define COMMENT_HYPHEN 10#define COMMENT_HYPHEN_HYPHEN 11#define RELAY 12+#define TAG_QUESTION 13+#define TAG_QUESTION_P 14+#define TAG_QUESTION_P_H 15+#define ENTITY 16HighlightHtml::HighlightHtml() {}@@ -47,14 +51,28 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {if (c == '<') {t_ = TAG;name_.clear();+ } else if (c == '&') {+ t_ = ENTITY;+ *r += HI_ENTITY;+ *r += c;} else {*r += c;}break;+ case ENTITY:+ *r += c;+ if (c == ';') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;+case TAG:if (c == '!') {t_ = TAG_EXCLAIM;+ } else if (c == '?') {+ t_ = TAG_QUESTION;} else if (c == '>' || isspace(c)) {*r += '<';*r += c;@@ -186,13 +204,13 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {if (name_ == "script") {pending_.clear();closer_ = "</script>";- highlighter_ = Highlight::create("js");+ highlighter_ = new HighlightC(is_keyword_js);t_ = RELAY;i_ = 0;} else if (name_ == "style") {pending_.clear();closer_ = "</style>";- highlighter_ = Highlight::create("css");+ highlighter_ = new HighlightCss;t_ = RELAY;i_ = 0;}@@ -221,6 +239,53 @@ void HighlightHtml::feed(std::string *r, std::string_view input) {}break;+ case TAG_QUESTION:+ if (c == 'p') {+ t_ = TAG_QUESTION_P;+ } else if (c == '=') {+ *r += HI_TAG;+ *r += "<?=";+ *r += HI_RESET;+ pending_.clear();+ closer_ = "?>";+ highlighter_ = new HighlightPhp;+ t_ = RELAY;+ i_ = 0;+ } else {+ *r += "<?";+ *r += c;+ t_ = NORMAL;+ }+ break;++ case TAG_QUESTION_P:+ if (c == 'h') {+ t_ = TAG_QUESTION_P_H;+ } else {+ *r += "<?p";+ *r += c;+ t_ = NORMAL;+ }+ break;++ case TAG_QUESTION_P_H:+ if (c == 'p') {+ *r += "<";+ *r += HI_TAG;+ *r += "?php";+ *r += HI_RESET;+ pending_.clear();+ closer_ = "?>";+ highlighter_ = new HighlightPhp;+ t_ = RELAY;+ i_ = 0;+ } else {+ *r += "<?ph";+ *r += c;+ t_ = NORMAL;+ }+ break;+default:__builtin_unreachable();}@@ -238,6 +303,15 @@ void HighlightHtml::flush(std::string *r) {case TAG_EXCLAIM_HYPHEN:*r += "<!-";break;+ case TAG_QUESTION:+ *r += "<?";+ break;+ case TAG_QUESTION_P:+ *r += "<?p";+ break;+ case TAG_QUESTION_P_H:+ *r += "<?ph";+ break;case COMMENT_HYPHEN_HYPHEN:case COMMENT_HYPHEN:case COMMENT:diff --git a/llamafile/highlight_markdown.cpp b/llamafile/highlight_markdown.cppindex 362a04be..965fc149 100644--- a/llamafile/highlight_markdown.cpp+++ b/llamafile/highlight_markdown.cpp@@ -122,7 +122,7 @@ void HighlightMarkdown::feed(std::string *r, std::string_view input) {case LANG:if (!isascii(c) || !isspace(c)) {*r += c;- lang_ += c;+ lang_ += tolower(c);} else {highlighter_ = Highlight::create(lang_);lang_.clear();diff --git a/llamafile/highlight_php.cpp b/llamafile/highlight_php.cppnew file mode 100644index 00000000..51b9d6be--- /dev/null+++ b/llamafile/highlight_php.cpp@@ -0,0 +1,213 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "highlight.h"++#include <ctype.h>++#define NORMAL 0+#define WORD 1+#define QUOTE 2+#define DQUOTE 3+#define SLASH 4+#define SLASH_SLASH 5+#define SLASH_STAR 6+#define SLASH_STAR_STAR 7+#define TICK 8+#define VAR 9+#define BACKSLASH 64++HighlightPhp::HighlightPhp() {+}++HighlightPhp::~HighlightPhp() {+}++void HighlightPhp::feed(std::string *r, std::string_view input) {+ int c;+ for (size_t i = 0; i < input.size(); ++i) {+ c = input[i] & 255;++ if (t_ & BACKSLASH) {+ t_ &= ~BACKSLASH;+ *r += c;+ continue;+ } else if (c == '\\') {+ *r += c;+ t_ |= BACKSLASH;+ continue;+ }++ switch (t_) {++ Normal:+ case NORMAL:+ if (!isascii(c) || isalpha(c) || c == '_' || c == '#') {+ t_ = WORD;+ goto Word;+ } else if (c == '/') {+ t_ = SLASH;+ } else if (c == '\'') {+ t_ = QUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '"') {+ t_ = DQUOTE;+ *r += HI_STRING;+ *r += c;+ } else if (c == '`') {+ t_ = TICK;+ *r += HI_STRING;+ *r += c;+ } else if (c == '$') {+ *r += c;+ t_ = VAR;+ *r += HI_VAR;+ } else {+ *r += c;+ }+ break;++ Word:+ case WORD:+ if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {+ word_ += c;+ } else {+ if (is_keyword_php(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case VAR:+ if (!isascii(c) || isalpha(c) || isdigit(c) || c == '_') {+ *r += c;+ } else {+ *r += HI_RESET;+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH:+ if (c == '/') {+ *r += HI_COMMENT;+ *r += "//";+ t_ = SLASH_SLASH;+ } else if (c == '*') {+ *r += HI_COMMENT;+ *r += "/*";+ t_ = SLASH_STAR;+ } else {+ *r += '/';+ t_ = NORMAL;+ goto Normal;+ }+ break;++ case SLASH_SLASH:+ if (c == '\n') {+ *r += HI_RESET;+ *r += c;+ t_ = NORMAL;+ } else {+ *r += c;+ }+ break;++ case SLASH_STAR:+ *r += c;+ if (c == '*')+ t_ = SLASH_STAR_STAR;+ break;++ case SLASH_STAR_STAR:+ *r += c;+ if (c == '/') {+ *r += HI_RESET;+ t_ = NORMAL;+ } else {+ t_ = SLASH_STAR;+ }+ break;++ case QUOTE:+ *r += c;+ if (c == '\'') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case DQUOTE:+ *r += c;+ if (c == '"') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ case TICK:+ *r += c;+ if (c == '`') {+ *r += HI_RESET;+ t_ = NORMAL;+ }+ break;++ default:+ __builtin_unreachable();+ }+ }+}++void HighlightPhp::flush(std::string *r) {+ t_ &= ~BACKSLASH;+ switch (t_) {+ case WORD:+ if (is_keyword_php(word_.data(), word_.size())) {+ *r += HI_KEYWORD;+ *r += word_;+ *r += HI_RESET;+ } else {+ *r += word_;+ }+ word_.clear();+ break;+ case SLASH:+ *r += '/';+ break;+ case VAR:+ case QUOTE:+ case DQUOTE:+ case SLASH_SLASH:+ case SLASH_STAR:+ case SLASH_STAR_STAR:+ *r += HI_RESET;+ break;+ default:+ break;+ }+ t_ = NORMAL;+}diff --git a/llamafile/is_keyword_php.c b/llamafile/is_keyword_php.cnew file mode 100644index 00000000..88a6e5ec--- /dev/null+++ b/llamafile/is_keyword_php.c@@ -0,0 +1,289 @@+/* ANSI-C code produced by gperf version 3.1 */+/* Command-line: gperf llamafile/is_keyword_php.gperf */+/* Computed positions: -k'1,3-4' */++#if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \+ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \+ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \+ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \+ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \+ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \+ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \+ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \+ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \+ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \+ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \+ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \+ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \+ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \+ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \+ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \+ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \+ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \+ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \+ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \+ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \+ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \+ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126))+/* The character set is not based on ISO-646. */+#error "gperf generated tables don't work with this execution character set. Please report a bug to <[email protected]>."+#endif++#line 1 "llamafile/is_keyword_php.gperf"++#include <string.h>+#include <libc/str/tab.h>+#define GPERF_DOWNCASE++#define TOTAL_KEYWORDS 80+#define MIN_WORD_LENGTH 2+#define MAX_WORD_LENGTH 51+#define MIN_HASH_VALUE 2+#define MAX_HASH_VALUE 182+/* maximum key range = 181, duplicates = 0 */++#ifndef GPERF_DOWNCASE+#define GPERF_DOWNCASE 1+static unsigned char gperf_downcase[256] =+ {+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,+ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,+ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,+ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,+ 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106,+ 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,+ 122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104,+ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134,+ 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,+ 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164,+ 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,+ 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,+ 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209,+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224,+ 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,+ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254,+ 255+ };+#endif++#ifndef GPERF_CASE_STRNCMP+#define GPERF_CASE_STRNCMP 1+static int+gperf_case_strncmp (register const char *s1, register const char *s2, register size_t n)+{+ for (; n > 0;)+ {+ unsigned char c1 = gperf_downcase[(unsigned char)*s1++];+ unsigned char c2 = gperf_downcase[(unsigned char)*s2++];+ if (c1 != 0 && c1 == c2)+ {+ n--;+ continue;+ }+ return (int)c1 - (int)c2;+ }+ return 0;+}+#endif++#ifdef __GNUC__+__inline+#else+#ifdef __cplusplus+inline+#endif+#endif+static unsigned int+hash (register const char *str, register size_t len)+{+ static const unsigned char asso_values[] =+ {+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 30, 5, 20, 10, 10,+ 40, 5, 0, 0, 183, 183, 35, 50, 35, 75,+ 50, 0, 40, 0, 0, 100, 25, 90, 5, 90,+ 183, 183, 183, 183, 183, 30, 183, 30, 5, 20,+ 10, 10, 40, 5, 0, 0, 183, 183, 35, 50,+ 35, 75, 50, 0, 40, 0, 0, 100, 25, 90,+ 5, 90, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 5, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 0, 183, 183, 183, 183, 183, 183, 183, 183, 183,+ 183, 183, 183, 183, 183, 183+ };+ register unsigned int hval = len;++ switch (hval)+ {+ default:+ hval += asso_values[(unsigned char)str[3]];+ /*FALLTHROUGH*/+ case 3:+ hval += asso_values[(unsigned char)str[2]];+ /*FALLTHROUGH*/+ case 2:+ case 1:+ hval += asso_values[(unsigned char)str[0]];+ break;+ }+ return hval;+}++const char *+is_keyword_php (register const char *str, register size_t len)+{+ static const char * const wordlist[] =+ {+ "", "",+ "if",+ "", "", "",+ "switch",+ "", "",+ "insteadof",+ "instanceof",+ "",+ "do",+ "",+ "exit",+ "isset",+ "", "", "",+ "interface",+ "", "", "",+ "die",+ "else",+ "endif",+ "elseif",+ "extends",+ "",+ "endswitch",+ "", "",+ "as",+ "",+ "case",+ "trait",+ "static",+ "",+ "abstract",+ "list",+ "enddeclare",+ "",+ "fn",+ "and",+ "",+ "catch",+ "",+ "__DIR__",+ "xor",+ "",+ "break",+ "", "", "", "",+ "class",+ "", "", "", "",+ "const",+ "",+ "include",+ "continue",+ "",+ "empty",+ "endfor",+ "include_once",+ "var",+ "",+ "endforeach",+ "",+ "declare",+ "__LINE__",+ "",+ "match",+ "",+ "or",+ "__FILE__",+ "eval",+ "false",+ "__TRAIT__\011\011",+ "private",+ "for",+ "goto",+ "",+ "__\360\235\224\245\360\235\224\236\360\235\224\251\360\235\224\261_\360\235\224\240\360\235\224\254\360\235\224\252\360\235\224\255\360\235\224\246\360\235\224\251\360\235\224\242\360\235\224\257",+ "default",+ "readonly",+ "echo",+ "print",+ "global",+ "",+ "try",+ "__CLASS__",+ "implements",+ "public",+ "foreach",+ "callable",+ "",+ "__METHOD__",+ "", "",+ "function",+ "namespace",+ "array",+ "", "",+ "__NAMESPACE__",+ "",+ "final",+ "",+ "finally",+ "use",+ "true",+ "unset",+ "", "",+ "endwhile",+ "",+ "throw",+ "", "", "", "", "", "", "",+ "new",+ "",+ "while",+ "", "", "",+ "protected",+ "clone",+ "", "", "", "",+ "yield",+ "", "", "", "", "",+ "return",+ "require",+ "", "", "", "",+ "require_once",+ "", "", "", "", "", "", "", "", "",+ "", "", "", "", "", "", "", "", "",+ "", "", "", "", "", "", "", "", "",+ "", "",+ "__FUNCTION__"+ };++ if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)+ {+ register unsigned int key = hash (str, len);++ if (key <= MAX_HASH_VALUE)+ {+ register const char *s = wordlist[key];++ if ((((unsigned char)*str ^ (unsigned char)*s) & ~32) == 0 && !gperf_case_strncmp (str, s, len) && s[len] == '\0')+ return s;+ }+ }+ return 0;+}diff --git a/llamafile/is_keyword_php.gperf b/llamafile/is_keyword_php.gperfnew file mode 100644index 00000000..198a1df2--- /dev/null+++ b/llamafile/is_keyword_php.gperf@@ -0,0 +1,91 @@+%{+#include <string.h>+#include <libc/str/tab.h>+#define GPERF_DOWNCASE+%}+%ignore-case+%compare-strncmp+%language=ANSI-C+%readonly-tables+%define lookup-function-name is_keyword_php+%%+__CLASS__+__DIR__+__FILE__+__FUNCTION__+__LINE__+__METHOD__+__NAMESPACE__+__TRAIT__+__π₯ππ©π±_π π¬πͺππ¦π©π’π―+abstract+and+array+as+break+callable+case+catch+class+clone+const+continue+declare+default+die+do+echo+else+elseif+empty+enddeclare+endfor+endforeach+endif+endswitch+endwhile+eval+exit+extends+false+final+finally+fn+for+foreach+function+global+goto+if+implements+include+include_once+instanceof+insteadof+interface+isset+list+match+namespace+new+or+private+protected+public+readonly+require+require_once+return+static+switch+throw+trait+true+try+unset+use+var+while+xor+yield