Skip to content

Distribute and run LLMs with a single file.

Fix emoji editing in chatbot

Justine Tunneyf51e5358d09f
diff --git a/llamafile/bestline.c b/llamafile/bestline.c
index b5509066..5c009622 100644
--- a/llamafile/bestline.c
+++ b/llamafile/bestline.c
@@ -271,14 +271,263 @@ static char IsControl(unsigned c) {
return c <= 0x1F || (0x7F <= c && c <= 0x9F);
}
-static int GetMonospaceCharacterWidth(unsigned c) {
- return !IsControl(c) +
- (c >= 0x1100 && (c <= 0x115f || c == 0x2329 || c == 0x232a ||
- (c >= 0x2e80 && c <= 0xa4cf && c != 0x303f) ||
- (c >= 0xac00 && c <= 0xd7a3) || (c >= 0xf900 && c <= 0xfaff) ||
- (c >= 0xfe10 && c <= 0xfe19) || (c >= 0xfe30 && c <= 0xfe6f) ||
- (c >= 0xff00 && c <= 0xff60) || (c >= 0xffe0 && c <= 0xffe6) ||
- (c >= 0x20000 && c <= 0x2fffd) || (c >= 0x30000 && c <= 0x3fffd)));
+/**
+ * Returns monospace character width.
+ *
+ * This will be zero for control characters, combining marks, etc.
+ * Chinese, Korean, Japanese, Emoji, etc. will have a width of 2, and
+ * all other characters will be 1.
+ *
+ * This implementation is consistent with wcwidth() on Linux, except
+ * that this won't return -1 for various character ranges.
+ */
+int bestlineCharacterWidth(int c) {
+ if ((0x000 <= c && c <= 0x01F) || (0x07F <= c && c <= 0x09F) ||
+ (0x300 <= c && c <= 0x36f) || (0x483 <= c && c <= 0x489) ||
+ (0x591 <= c && c <= 0x5bd) || (0x5bf <= c && c <= 0x5bf) ||
+ (0x5c1 <= c && c <= 0x5c2) || (0x5c4 <= c && c <= 0x5c5) ||
+ (0x5c7 <= c && c <= 0x5c7) || (0x610 <= c && c <= 0x61a) ||
+ (0x61c <= c && c <= 0x61c) || (0x64b <= c && c <= 0x65f) ||
+ (0x670 <= c && c <= 0x670) || (0x6d6 <= c && c <= 0x6dc) ||
+ (0x6df <= c && c <= 0x6e4) || (0x6e7 <= c && c <= 0x6e8) ||
+ (0x6ea <= c && c <= 0x6ed) || (0x711 <= c && c <= 0x711) ||
+ (0x730 <= c && c <= 0x74a) || (0x7a6 <= c && c <= 0x7b0) ||
+ (0x7eb <= c && c <= 0x7f3) || (0x7fd <= c && c <= 0x7fd) ||
+ (0x816 <= c && c <= 0x819) || (0x81b <= c && c <= 0x823) ||
+ (0x825 <= c && c <= 0x827) || (0x829 <= c && c <= 0x82d) ||
+ (0x859 <= c && c <= 0x85b) || (0x898 <= c && c <= 0x89f) ||
+ (0x8ca <= c && c <= 0x8e1) || (0x8e3 <= c && c <= 0x902) ||
+ (0x93a <= c && c <= 0x93a) || (0x93c <= c && c <= 0x93c) ||
+ (0x941 <= c && c <= 0x948) || (0x94d <= c && c <= 0x94d) ||
+ (0x951 <= c && c <= 0x957) || (0x962 <= c && c <= 0x963) ||
+ (0x981 <= c && c <= 0x981) || (0x9bc <= c && c <= 0x9bc) ||
+ (0x9c1 <= c && c <= 0x9c4) || (0x9cd <= c && c <= 0x9cd) ||
+ (0x9e2 <= c && c <= 0x9e3) || (0x9fe <= c && c <= 0x9fe) ||
+ (0xa01 <= c && c <= 0xa02) || (0xa3c <= c && c <= 0xa3c) ||
+ (0xa41 <= c && c <= 0xa42) || (0xa47 <= c && c <= 0xa48) ||
+ (0xa4b <= c && c <= 0xa4d) || (0xa51 <= c && c <= 0xa51) ||
+ (0xa70 <= c && c <= 0xa71) || (0xa75 <= c && c <= 0xa75) ||
+ (0xa81 <= c && c <= 0xa82) || (0xabc <= c && c <= 0xabc) ||
+ (0xac1 <= c && c <= 0xac5) || (0xac7 <= c && c <= 0xac8) ||
+ (0xacd <= c && c <= 0xacd) || (0xae2 <= c && c <= 0xae3) ||
+ (0xafa <= c && c <= 0xaff) || (0xb01 <= c && c <= 0xb01) ||
+ (0xb3c <= c && c <= 0xb3c) || (0xb3f <= c && c <= 0xb3f) ||
+ (0xb41 <= c && c <= 0xb44) || (0xb4d <= c && c <= 0xb4d) ||
+ (0xb55 <= c && c <= 0xb56) || (0xb62 <= c && c <= 0xb63) ||
+ (0xb82 <= c && c <= 0xb82) || (0xbc0 <= c && c <= 0xbc0) ||
+ (0xbcd <= c && c <= 0xbcd) || (0xc00 <= c && c <= 0xc00) ||
+ (0xc04 <= c && c <= 0xc04) || (0xc3c <= c && c <= 0xc3c) ||
+ (0xc3e <= c && c <= 0xc40) || (0xc46 <= c && c <= 0xc48) ||
+ (0xc4a <= c && c <= 0xc4d) || (0xc55 <= c && c <= 0xc56) ||
+ (0xc62 <= c && c <= 0xc63) || (0xc81 <= c && c <= 0xc81) ||
+ (0xcbc <= c && c <= 0xcbc) || (0xcbf <= c && c <= 0xcbf) ||
+ (0xcc6 <= c && c <= 0xcc6) || (0xccc <= c && c <= 0xccd) ||
+ (0xce2 <= c && c <= 0xce3) || (0xd00 <= c && c <= 0xd01) ||
+ (0xd3b <= c && c <= 0xd3c) || (0xd41 <= c && c <= 0xd44) ||
+ (0xd4d <= c && c <= 0xd4d) || (0xd62 <= c && c <= 0xd63) ||
+ (0xd81 <= c && c <= 0xd81) || (0xdca <= c && c <= 0xdca) ||
+ (0xdd2 <= c && c <= 0xdd4) || (0xdd6 <= c && c <= 0xdd6) ||
+ (0xe31 <= c && c <= 0xe31) || (0xe34 <= c && c <= 0xe3a) ||
+ (0xe47 <= c && c <= 0xe4e) || (0xeb1 <= c && c <= 0xeb1) ||
+ (0xeb4 <= c && c <= 0xebc) || (0xec8 <= c && c <= 0xece) ||
+ (0xf18 <= c && c <= 0xf19) || (0xf35 <= c && c <= 0xf35) ||
+ (0xf37 <= c && c <= 0xf37) || (0xf39 <= c && c <= 0xf39) ||
+ (0xf71 <= c && c <= 0xf7e) || (0xf80 <= c && c <= 0xf84) ||
+ (0xf86 <= c && c <= 0xf87) || (0xf8d <= c && c <= 0xf97) ||
+ (0xf99 <= c && c <= 0xfbc) || (0xfc6 <= c && c <= 0xfc6) ||
+ (0x102d <= c && c <= 0x1030) || (0x1032 <= c && c <= 0x1037) ||
+ (0x1039 <= c && c <= 0x103a) || (0x103d <= c && c <= 0x103e) ||
+ (0x1058 <= c && c <= 0x1059) || (0x105e <= c && c <= 0x1060) ||
+ (0x1071 <= c && c <= 0x1074) || (0x1082 <= c && c <= 0x1082) ||
+ (0x1085 <= c && c <= 0x1086) || (0x108d <= c && c <= 0x108d) ||
+ (0x109d <= c && c <= 0x109d) || (0x1160 <= c && c <= 0x11ff) ||
+ (0x135d <= c && c <= 0x135f) || (0x1712 <= c && c <= 0x1714) ||
+ (0x1732 <= c && c <= 0x1733) || (0x1752 <= c && c <= 0x1753) ||
+ (0x1772 <= c && c <= 0x1773) || (0x17b4 <= c && c <= 0x17b5) ||
+ (0x17b7 <= c && c <= 0x17bd) || (0x17c6 <= c && c <= 0x17c6) ||
+ (0x17c9 <= c && c <= 0x17d3) || (0x17dd <= c && c <= 0x17dd) ||
+ (0x180b <= c && c <= 0x180f) || (0x1885 <= c && c <= 0x1886) ||
+ (0x18a9 <= c && c <= 0x18a9) || (0x1920 <= c && c <= 0x1922) ||
+ (0x1927 <= c && c <= 0x1928) || (0x1932 <= c && c <= 0x1932) ||
+ (0x1939 <= c && c <= 0x193b) || (0x1a17 <= c && c <= 0x1a18) ||
+ (0x1a1b <= c && c <= 0x1a1b) || (0x1a56 <= c && c <= 0x1a56) ||
+ (0x1a58 <= c && c <= 0x1a5e) || (0x1a60 <= c && c <= 0x1a60) ||
+ (0x1a62 <= c && c <= 0x1a62) || (0x1a65 <= c && c <= 0x1a6c) ||
+ (0x1a73 <= c && c <= 0x1a7c) || (0x1a7f <= c && c <= 0x1a7f) ||
+ (0x1ab0 <= c && c <= 0x1ace) || (0x1b00 <= c && c <= 0x1b03) ||
+ (0x1b34 <= c && c <= 0x1b34) || (0x1b36 <= c && c <= 0x1b3a) ||
+ (0x1b3c <= c && c <= 0x1b3c) || (0x1b42 <= c && c <= 0x1b42) ||
+ (0x1b6b <= c && c <= 0x1b73) || (0x1b80 <= c && c <= 0x1b81) ||
+ (0x1ba2 <= c && c <= 0x1ba5) || (0x1ba8 <= c && c <= 0x1ba9) ||
+ (0x1bab <= c && c <= 0x1bad) || (0x1be6 <= c && c <= 0x1be6) ||
+ (0x1be8 <= c && c <= 0x1be9) || (0x1bed <= c && c <= 0x1bed) ||
+ (0x1bef <= c && c <= 0x1bf1) || (0x1c2c <= c && c <= 0x1c33) ||
+ (0x1c36 <= c && c <= 0x1c37) || (0x1cd0 <= c && c <= 0x1cd2) ||
+ (0x1cd4 <= c && c <= 0x1ce0) || (0x1ce2 <= c && c <= 0x1ce8) ||
+ (0x1ced <= c && c <= 0x1ced) || (0x1cf4 <= c && c <= 0x1cf4) ||
+ (0x1cf8 <= c && c <= 0x1cf9) || (0x1dc0 <= c && c <= 0x1dff) ||
+ (0x200b <= c && c <= 0x200f) || (0x202a <= c && c <= 0x202e) ||
+ (0x2060 <= c && c <= 0x2064) || (0x2066 <= c && c <= 0x206f) ||
+ (0x20d0 <= c && c <= 0x20f0) || (0x2cef <= c && c <= 0x2cf1) ||
+ (0x2d7f <= c && c <= 0x2d7f) || (0x2de0 <= c && c <= 0x2dff) ||
+ (0x302a <= c && c <= 0x302d) || (0x3099 <= c && c <= 0x309a) ||
+ (0xa66f <= c && c <= 0xa672) || (0xa674 <= c && c <= 0xa67d) ||
+ (0xa69e <= c && c <= 0xa69f) || (0xa6f0 <= c && c <= 0xa6f1) ||
+ (0xa802 <= c && c <= 0xa802) || (0xa806 <= c && c <= 0xa806) ||
+ (0xa80b <= c && c <= 0xa80b) || (0xa825 <= c && c <= 0xa826) ||
+ (0xa82c <= c && c <= 0xa82c) || (0xa8c4 <= c && c <= 0xa8c5) ||
+ (0xa8e0 <= c && c <= 0xa8f1) || (0xa8ff <= c && c <= 0xa8ff) ||
+ (0xa926 <= c && c <= 0xa92d) || (0xa947 <= c && c <= 0xa951) ||
+ (0xa980 <= c && c <= 0xa982) || (0xa9b3 <= c && c <= 0xa9b3) ||
+ (0xa9b6 <= c && c <= 0xa9b9) || (0xa9bc <= c && c <= 0xa9bd) ||
+ (0xa9e5 <= c && c <= 0xa9e5) || (0xaa29 <= c && c <= 0xaa2e) ||
+ (0xaa31 <= c && c <= 0xaa32) || (0xaa35 <= c && c <= 0xaa36) ||
+ (0xaa43 <= c && c <= 0xaa43) || (0xaa4c <= c && c <= 0xaa4c) ||
+ (0xaa7c <= c && c <= 0xaa7c) || (0xaab0 <= c && c <= 0xaab0) ||
+ (0xaab2 <= c && c <= 0xaab4) || (0xaab7 <= c && c <= 0xaab8) ||
+ (0xaabe <= c && c <= 0xaabf) || (0xaac1 <= c && c <= 0xaac1) ||
+ (0xaaec <= c && c <= 0xaaed) || (0xaaf6 <= c && c <= 0xaaf6) ||
+ (0xabe5 <= c && c <= 0xabe5) || (0xabe8 <= c && c <= 0xabe8) ||
+ (0xabed <= c && c <= 0xabed) || (0xd7b0 <= c && c <= 0xd7c6) ||
+ (0xd7cb <= c && c <= 0xd7fb) || (0xfb1e <= c && c <= 0xfb1e) ||
+ (0xfe00 <= c && c <= 0xfe0f) || (0xfe20 <= c && c <= 0xfe2f) ||
+ (0xfeff <= c && c <= 0xfeff) || (0xfff9 <= c && c <= 0xfffb) ||
+ (0x101fd <= c && c <= 0x101fd) || (0x102e0 <= c && c <= 0x102e0) ||
+ (0x10376 <= c && c <= 0x1037a) || (0x10a01 <= c && c <= 0x10a03) ||
+ (0x10a05 <= c && c <= 0x10a06) || (0x10a0c <= c && c <= 0x10a0f) ||
+ (0x10a38 <= c && c <= 0x10a3a) || (0x10a3f <= c && c <= 0x10a3f) ||
+ (0x10ae5 <= c && c <= 0x10ae6) || (0x10d24 <= c && c <= 0x10d27) ||
+ (0x10eab <= c && c <= 0x10eac) || (0x10efd <= c && c <= 0x10eff) ||
+ (0x10f46 <= c && c <= 0x10f50) || (0x10f82 <= c && c <= 0x10f85) ||
+ (0x11001 <= c && c <= 0x11001) || (0x11038 <= c && c <= 0x11046) ||
+ (0x11070 <= c && c <= 0x11070) || (0x11073 <= c && c <= 0x11074) ||
+ (0x1107f <= c && c <= 0x11081) || (0x110b3 <= c && c <= 0x110b6) ||
+ (0x110b9 <= c && c <= 0x110ba) || (0x110c2 <= c && c <= 0x110c2) ||
+ (0x11100 <= c && c <= 0x11102) || (0x11127 <= c && c <= 0x1112b) ||
+ (0x1112d <= c && c <= 0x11134) || (0x11173 <= c && c <= 0x11173) ||
+ (0x11180 <= c && c <= 0x11181) || (0x111b6 <= c && c <= 0x111be) ||
+ (0x111c9 <= c && c <= 0x111cc) || (0x111cf <= c && c <= 0x111cf) ||
+ (0x1122f <= c && c <= 0x11231) || (0x11234 <= c && c <= 0x11234) ||
+ (0x11236 <= c && c <= 0x11237) || (0x1123e <= c && c <= 0x1123e) ||
+ (0x11241 <= c && c <= 0x11241) || (0x112df <= c && c <= 0x112df) ||
+ (0x112e3 <= c && c <= 0x112ea) || (0x11300 <= c && c <= 0x11301) ||
+ (0x1133b <= c && c <= 0x1133c) || (0x11340 <= c && c <= 0x11340) ||
+ (0x11366 <= c && c <= 0x1136c) || (0x11370 <= c && c <= 0x11374) ||
+ (0x11438 <= c && c <= 0x1143f) || (0x11442 <= c && c <= 0x11444) ||
+ (0x11446 <= c && c <= 0x11446) || (0x1145e <= c && c <= 0x1145e) ||
+ (0x114b3 <= c && c <= 0x114b8) || (0x114ba <= c && c <= 0x114ba) ||
+ (0x114bf <= c && c <= 0x114c0) || (0x114c2 <= c && c <= 0x114c3) ||
+ (0x115b2 <= c && c <= 0x115b5) || (0x115bc <= c && c <= 0x115bd) ||
+ (0x115bf <= c && c <= 0x115c0) || (0x115dc <= c && c <= 0x115dd) ||
+ (0x11633 <= c && c <= 0x1163a) || (0x1163d <= c && c <= 0x1163d) ||
+ (0x1163f <= c && c <= 0x11640) || (0x116ab <= c && c <= 0x116ab) ||
+ (0x116ad <= c && c <= 0x116ad) || (0x116b0 <= c && c <= 0x116b5) ||
+ (0x116b7 <= c && c <= 0x116b7) || (0x1171d <= c && c <= 0x1171f) ||
+ (0x11722 <= c && c <= 0x11725) || (0x11727 <= c && c <= 0x1172b) ||
+ (0x1182f <= c && c <= 0x11837) || (0x11839 <= c && c <= 0x1183a) ||
+ (0x1193b <= c && c <= 0x1193c) || (0x1193e <= c && c <= 0x1193e) ||
+ (0x11943 <= c && c <= 0x11943) || (0x119d4 <= c && c <= 0x119d7) ||
+ (0x119da <= c && c <= 0x119db) || (0x119e0 <= c && c <= 0x119e0) ||
+ (0x11a01 <= c && c <= 0x11a0a) || (0x11a33 <= c && c <= 0x11a38) ||
+ (0x11a3b <= c && c <= 0x11a3e) || (0x11a47 <= c && c <= 0x11a47) ||
+ (0x11a51 <= c && c <= 0x11a56) || (0x11a59 <= c && c <= 0x11a5b) ||
+ (0x11a8a <= c && c <= 0x11a96) || (0x11a98 <= c && c <= 0x11a99) ||
+ (0x11c30 <= c && c <= 0x11c36) || (0x11c38 <= c && c <= 0x11c3d) ||
+ (0x11c3f <= c && c <= 0x11c3f) || (0x11c92 <= c && c <= 0x11ca7) ||
+ (0x11caa <= c && c <= 0x11cb0) || (0x11cb2 <= c && c <= 0x11cb3) ||
+ (0x11cb5 <= c && c <= 0x11cb6) || (0x11d31 <= c && c <= 0x11d36) ||
+ (0x11d3a <= c && c <= 0x11d3a) || (0x11d3c <= c && c <= 0x11d3d) ||
+ (0x11d3f <= c && c <= 0x11d45) || (0x11d47 <= c && c <= 0x11d47) ||
+ (0x11d90 <= c && c <= 0x11d91) || (0x11d95 <= c && c <= 0x11d95) ||
+ (0x11d97 <= c && c <= 0x11d97) || (0x11ef3 <= c && c <= 0x11ef4) ||
+ (0x11f00 <= c && c <= 0x11f01) || (0x11f36 <= c && c <= 0x11f3a) ||
+ (0x11f40 <= c && c <= 0x11f40) || (0x11f42 <= c && c <= 0x11f42) ||
+ (0x13430 <= c && c <= 0x13440) || (0x13447 <= c && c <= 0x13455) ||
+ (0x16af0 <= c && c <= 0x16af4) || (0x16b30 <= c && c <= 0x16b36) ||
+ (0x16f4f <= c && c <= 0x16f4f) || (0x16f8f <= c && c <= 0x16f92) ||
+ (0x16fe4 <= c && c <= 0x16fe4) || (0x1bc9d <= c && c <= 0x1bc9e) ||
+ (0x1bca0 <= c && c <= 0x1bca3) || (0x1cf00 <= c && c <= 0x1cf2d) ||
+ (0x1cf30 <= c && c <= 0x1cf46) || (0x1d167 <= c && c <= 0x1d169) ||
+ (0x1d173 <= c && c <= 0x1d182) || (0x1d185 <= c && c <= 0x1d18b) ||
+ (0x1d1aa <= c && c <= 0x1d1ad) || (0x1d242 <= c && c <= 0x1d244) ||
+ (0x1da00 <= c && c <= 0x1da36) || (0x1da3b <= c && c <= 0x1da6c) ||
+ (0x1da75 <= c && c <= 0x1da75) || (0x1da84 <= c && c <= 0x1da84) ||
+ (0x1da9b <= c && c <= 0x1da9f) || (0x1daa1 <= c && c <= 0x1daaf) ||
+ (0x1e000 <= c && c <= 0x1e006) || (0x1e008 <= c && c <= 0x1e018) ||
+ (0x1e01b <= c && c <= 0x1e021) || (0x1e023 <= c && c <= 0x1e024) ||
+ (0x1e026 <= c && c <= 0x1e02a) || (0x1e08f <= c && c <= 0x1e08f) ||
+ (0x1e130 <= c && c <= 0x1e136) || (0x1e2ae <= c && c <= 0x1e2ae) ||
+ (0x1e2ec <= c && c <= 0x1e2ef) || (0x1e4ec <= c && c <= 0x1e4ef) ||
+ (0x1e8d0 <= c && c <= 0x1e8d6) || (0x1e944 <= c && c <= 0x1e94a) ||
+ (0xe0001 <= c && c <= 0xe0001) || (0xe0020 <= c && c <= 0xe007f) ||
+ (0xe0100 <= c && c <= 0xe01ef))
+ return 0;
+ if ((0x1100 <= c && c <= 0x115f) || (0x231a <= c && c <= 0x231b) ||
+ (0x2329 <= c && c <= 0x232a) || (0x23e9 <= c && c <= 0x23ec) ||
+ (0x23f0 <= c && c <= 0x23f0) || (0x23f3 <= c && c <= 0x23f3) ||
+ (0x25fd <= c && c <= 0x25fe) || (0x2614 <= c && c <= 0x2615) ||
+ (0x2648 <= c && c <= 0x2653) || (0x267f <= c && c <= 0x267f) ||
+ (0x2693 <= c && c <= 0x2693) || (0x26a1 <= c && c <= 0x26a1) ||
+ (0x26aa <= c && c <= 0x26ab) || (0x26bd <= c && c <= 0x26be) ||
+ (0x26c4 <= c && c <= 0x26c5) || (0x26ce <= c && c <= 0x26ce) ||
+ (0x26d4 <= c && c <= 0x26d4) || (0x26ea <= c && c <= 0x26ea) ||
+ (0x26f2 <= c && c <= 0x26f3) || (0x26f5 <= c && c <= 0x26f5) ||
+ (0x26fa <= c && c <= 0x26fa) || (0x26fd <= c && c <= 0x26fd) ||
+ (0x2705 <= c && c <= 0x2705) || (0x270a <= c && c <= 0x270b) ||
+ (0x2728 <= c && c <= 0x2728) || (0x274c <= c && c <= 0x274c) ||
+ (0x274e <= c && c <= 0x274e) || (0x2753 <= c && c <= 0x2755) ||
+ (0x2757 <= c && c <= 0x2757) || (0x2795 <= c && c <= 0x2797) ||
+ (0x27b0 <= c && c <= 0x27b0) || (0x27bf <= c && c <= 0x27bf) ||
+ (0x2b1b <= c && c <= 0x2b1c) || (0x2b50 <= c && c <= 0x2b50) ||
+ (0x2b55 <= c && c <= 0x2b55) || (0x2e80 <= c && c <= 0x2e99) ||
+ (0x2e9b <= c && c <= 0x2ef3) || (0x2f00 <= c && c <= 0x2fd5) ||
+ (0x2ff0 <= c && c <= 0x3029) || (0x302e <= c && c <= 0x303e) ||
+ (0x3041 <= c && c <= 0x3096) || (0x309b <= c && c <= 0x30ff) ||
+ (0x3105 <= c && c <= 0x312f) || (0x3131 <= c && c <= 0x318e) ||
+ (0x3190 <= c && c <= 0x31e3) || (0x31ef <= c && c <= 0x321e) ||
+ (0x3220 <= c && c <= 0xa48c) || (0xa490 <= c && c <= 0xa4c6) ||
+ (0xa960 <= c && c <= 0xa97c) || (0xac00 <= c && c <= 0xd7a3) ||
+ (0xf900 <= c && c <= 0xfa6d) || (0xfa70 <= c && c <= 0xfad9) ||
+ (0xfe10 <= c && c <= 0xfe19) || (0xfe30 <= c && c <= 0xfe52) ||
+ (0xfe54 <= c && c <= 0xfe66) || (0xfe68 <= c && c <= 0xfe6b) ||
+ (0xff01 <= c && c <= 0xff60) || (0xffe0 <= c && c <= 0xffe6) ||
+ (0x16fe0 <= c && c <= 0x16fe3) || (0x16ff0 <= c && c <= 0x16ff1) ||
+ (0x17000 <= c && c <= 0x187f7) || (0x18800 <= c && c <= 0x18cd5) ||
+ (0x18d00 <= c && c <= 0x18d08) || (0x1aff0 <= c && c <= 0x1aff3) ||
+ (0x1aff5 <= c && c <= 0x1affb) || (0x1affd <= c && c <= 0x1affe) ||
+ (0x1b000 <= c && c <= 0x1b122) || (0x1b132 <= c && c <= 0x1b132) ||
+ (0x1b150 <= c && c <= 0x1b152) || (0x1b155 <= c && c <= 0x1b155) ||
+ (0x1b164 <= c && c <= 0x1b167) || (0x1b170 <= c && c <= 0x1b2fb) ||
+ (0x1f004 <= c && c <= 0x1f004) || (0x1f0cf <= c && c <= 0x1f0cf) ||
+ (0x1f18e <= c && c <= 0x1f18e) || (0x1f191 <= c && c <= 0x1f19a) ||
+ (0x1f200 <= c && c <= 0x1f202) || (0x1f210 <= c && c <= 0x1f23b) ||
+ (0x1f240 <= c && c <= 0x1f248) || (0x1f250 <= c && c <= 0x1f251) ||
+ (0x1f260 <= c && c <= 0x1f265) || (0x1f300 <= c && c <= 0x1f320) ||
+ (0x1f32d <= c && c <= 0x1f335) || (0x1f337 <= c && c <= 0x1f37c) ||
+ (0x1f37e <= c && c <= 0x1f393) || (0x1f3a0 <= c && c <= 0x1f3ca) ||
+ (0x1f3cf <= c && c <= 0x1f3d3) || (0x1f3e0 <= c && c <= 0x1f3f0) ||
+ (0x1f3f4 <= c && c <= 0x1f3f4) || (0x1f3f8 <= c && c <= 0x1f43e) ||
+ (0x1f440 <= c && c <= 0x1f440) || (0x1f442 <= c && c <= 0x1f4fc) ||
+ (0x1f4ff <= c && c <= 0x1f53d) || (0x1f54b <= c && c <= 0x1f54e) ||
+ (0x1f550 <= c && c <= 0x1f567) || (0x1f57a <= c && c <= 0x1f57a) ||
+ (0x1f595 <= c && c <= 0x1f596) || (0x1f5a4 <= c && c <= 0x1f5a4) ||
+ (0x1f5fb <= c && c <= 0x1f64f) || (0x1f680 <= c && c <= 0x1f6c5) ||
+ (0x1f6cc <= c && c <= 0x1f6cc) || (0x1f6d0 <= c && c <= 0x1f6d2) ||
+ (0x1f6d5 <= c && c <= 0x1f6d7) || (0x1f6dc <= c && c <= 0x1f6df) ||
+ (0x1f6eb <= c && c <= 0x1f6ec) || (0x1f6f4 <= c && c <= 0x1f6fc) ||
+ (0x1f7e0 <= c && c <= 0x1f7eb) || (0x1f7f0 <= c && c <= 0x1f7f0) ||
+ (0x1f90c <= c && c <= 0x1f93a) || (0x1f93c <= c && c <= 0x1f945) ||
+ (0x1f947 <= c && c <= 0x1f9ff) || (0x1fa70 <= c && c <= 0x1fa7c) ||
+ (0x1fa80 <= c && c <= 0x1fa88) || (0x1fa90 <= c && c <= 0x1fabd) ||
+ (0x1fabf <= c && c <= 0x1fac5) || (0x1face <= c && c <= 0x1fadb) ||
+ (0x1fae0 <= c && c <= 0x1fae8) || (0x1faf0 <= c && c <= 0x1faf8) ||
+ (0x20000 <= c && c <= 0x2a6df) || (0x2a700 <= c && c <= 0x2b739) ||
+ (0x2b740 <= c && c <= 0x2b81d) || (0x2b820 <= c && c <= 0x2cea1) ||
+ (0x2ceb0 <= c && c <= 0x2ebe0) || (0x2ebf0 <= c && c <= 0x2ee5d) ||
+ (0x2f800 <= c && c <= 0x2fa1d) || (0x30000 <= c && c <= 0x3134a) ||
+ (0x31350 <= c && c <= 0x323af))
+ return 2;
+ return 1;
}
/**
@@ -451,14 +700,16 @@ char bestlineIsSeparator(unsigned c) {
l = 0;
r = n = sizeof(kGlyphs) / sizeof(kGlyphs[0]);
while (l < r) {
- m = (l + r) >> 1;
- if (kGlyphs[m][1] < c) {
+ unsigned m = (l & r) + ((l ^ r) >> 1);
+ if (c < kGlyphs[m][0]) {
+ r = m;
+ } else if (c > kGlyphs[m][1]) {
l = m + 1;
} else {
- r = m;
+ return 0;
}
}
- return !(l < n && kGlyphs[l][0] <= c && c <= kGlyphs[l][1]);
+ return 1;
} else {
static const unsigned kAstralGlyphs[][2] = {
{0x10107, 0x10133}, /* 45x Aegean */
@@ -671,14 +922,16 @@ char bestlineIsSeparator(unsigned c) {
l = 0;
r = n = sizeof(kAstralGlyphs) / sizeof(kAstralGlyphs[0]);
while (l < r) {
- m = (l + r) >> 1;
- if (kAstralGlyphs[m][1] < c) {
+ unsigned m = (l & r) + ((l ^ r) >> 1);
+ if (c < kAstralGlyphs[m][0]) {
+ r = m;
+ } else if (c > kAstralGlyphs[m][1]) {
l = m + 1;
} else {
- r = m;
+ return 0;
}
}
- return !(l < n && kAstralGlyphs[l][0] <= c && c <= kAstralGlyphs[l][1]);
+ return 1;
}
}
@@ -1746,7 +1999,7 @@ static size_t GetMonospaceWidth(const char *p, size_t n, char *out_haswides) {
r.c <<= 6;
r.c |= c & 077;
if (!--r.n) {
- d = GetMonospaceCharacterWidth(r.c);
+ d = bestlineCharacterWidth(r.c);
d = Max(0, d);
w += d;
haswides |= d > 1;
@@ -2379,7 +2632,7 @@ StartOver:
--len;
rune = GetUtf8(buf + len, t - len);
}
- if ((t = GetMonospaceCharacterWidth(rune.c)) > 0) {
+ if ((t = bestlineCharacterWidth(rune.c)) > 0) {
width -= t;
}
}
@@ -2440,7 +2693,7 @@ StartOver:
if (flipit)
abAppends(&ab, "\033[22m");
}
- t = GetMonospaceCharacterWidth(rune.c);
+ t = bestlineCharacterWidth(rune.c);
t = Max(0, t);
x += t;
}
@@ -3756,6 +4009,8 @@ void bestlineMaskModeEnable(void) {
/**
* Disables "mask mode".
+ *
+ * @see bestlineMaskModeEnable()
*/
void bestlineMaskModeDisable(void) {
maskmode = 0;
diff --git a/llamafile/bestline.h b/llamafile/bestline.h
index 0107bdb1..deb56fd4 100644
--- a/llamafile/bestline.h
+++ b/llamafile/bestline.h
@@ -42,6 +42,7 @@ void bestlineMaskModeEnable(void);
void bestlineUserIO(int (*)(int, void *, int), int (*)(int, const void *, int),
int (*)(int, int, int));
+int bestlineCharacterWidth(int);
char bestlineIsSeparator(unsigned);
char bestlineNotSeparator(unsigned);
char bestlineIsXeparator(unsigned);