Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(java): the remaining character tables, and laya.lang's word lists

Adeshina Lasisi8fa95393bd74
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 6e1869ae..d41f77a5 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -84,6 +84,9 @@ jobs:
- name: Unicode tables match CPython
if: ${{ !cancelled() }}
run: python laya-java/scripts/gen_unicode_tables.py --check
+ - name: Language tables match laya/lang.py
+ if: ${{ !cancelled() }}
+ run: python laya-java/scripts/gen_language_tables.py --check
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ failure() }}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java
new file mode 100644
index 00000000..4f1b28da
--- /dev/null
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java
@@ -0,0 +1,148 @@
+// GENERATED by laya-java/scripts/gen_language_tables.py -- do not edit.
+//
+// Regenerate with:
+// python laya-java/scripts/gen_language_tables.py
+// and `--check` fails when the committed copy no longer matches laya/lang.py.
+package com.convaiinnovations.laya.lang;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The word lists, script ranges and thresholds {@code laya/lang.py} detects with.
+ *
+ * <p>Compiled in rather than read from {@code fixtures/lang_tables.json}: a runtime library must
+ * not need a test fixture on disk. The fixture keeps its job -- a test asserts this copy equals it,
+ * so a table that drifts from Python fails a test rather than changing a routing decision quietly.
+ *
+ * <p><b>Two orderings are load-bearing.</b> {@link #SCRIPT_RANGES} is scanned in order and the
+ * first matching range wins, so a reordering changes which script a code point counts under. And
+ * {@link #STOP_WORDS} is iterated to score languages, where the winner is the first of equal
+ * scores, so its order decides ties.
+ */
+public final class LanguageTables {
+
+ private LanguageTables() {
+ }
+
+ /** One named non-Latin script and the code-point ranges that belong to it. */
+ public record Script(String name, int[] ranges) {
+ }
+
+ /**
+ * Named scripts, in scan order. The first range that contains a code point wins, so
+ * this is a sequence and not a set.
+ */
+ public static final List<Script> SCRIPT_RANGES = List.of(
+ new Script("greek", new int[] {880, 1023, 7936, 8191}),
+ new Script("cyrillic", new int[] {1024, 1327, 11744, 11775, 42560, 42655}),
+ new Script("armenian", new int[] {1328, 1423}),
+ new Script("hebrew", new int[] {1424, 1535}),
+ new Script("arabic", new int[] {1536, 1791, 1872, 1919, 2208, 2303, 64336, 65023, 65136, 65279}),
+ new Script("devanagari", new int[] {2304, 2431, 43232, 43263}),
+ new Script("bengali", new int[] {2432, 2559}),
+ new Script("gurmukhi", new int[] {2560, 2687}),
+ new Script("gujarati", new int[] {2688, 2815}),
+ new Script("oriya", new int[] {2816, 2943}),
+ new Script("tamil", new int[] {2944, 3071}),
+ new Script("telugu", new int[] {3072, 3199}),
+ new Script("kannada", new int[] {3200, 3327}),
+ new Script("malayalam", new int[] {3328, 3455}),
+ new Script("sinhala", new int[] {3456, 3583}),
+ new Script("thai", new int[] {3584, 3711}),
+ new Script("lao", new int[] {3712, 3839}),
+ new Script("tibetan", new int[] {3840, 4095}),
+ new Script("myanmar", new int[] {4096, 4255}),
+ new Script("georgian", new int[] {4256, 4351}),
+ new Script("ethiopic", new int[] {4608, 4991}),
+ new Script("khmer", new int[] {6016, 6143}),
+ new Script("hangul", new int[] {4352, 4607, 12592, 12687, 44032, 55215}),
+ new Script("kana", new int[] {12352, 12447, 12448, 12543, 12784, 12799}),
+ new Script("han", new int[] {13312, 19903, 19968, 40959, 63744, 64255}));
+
+ /**
+ * Function words per language, in the order Python declares them.
+ *
+ * <p>A {@link LinkedHashMap}, because the language that wins a tied score is the
+ * first one iterated.
+ */
+ public static final Map<String, Set<String>> STOP_WORDS;
+
+ static {
+ Map<String, Set<String>> STOP = new LinkedHashMap<>();
+ STOP.put("en", Set.of("and", "are", "as", "at", "be", "but", "can", "for", "from", "has", "have", "i", "in", "is", "it", "not", "of", "on", "please", "that", "the", "their", "there", "this", "to", "was", "we", "were", "what", "which", "will", "with", "would", "you"));
+ STOP.put("fr", Set.of("alors", "au", "aux", "avec", "bien", "bonjour", "ce", "ces", "cette", "comment", "dans", "des", "deux", "dois", "doit", "donc", "du", "elle", "elles", "est", "et", "fait", "fois", "il", "ils", "je", "jour", "jours", "la", "le", "les", "ma", "mais", "merci", "mes", "mois", "mon", "nous", "ont", "ou", "pas", "peut", "peux", "plus", "pour", "pourquoi", "quand", "que", "qui", "sa", "ses", "sont", "sur", "ta", "tes", "ton", "tous", "tout", "toute", "trois", "tr\u00E8s", "tu", "une", "veut", "veux", "vous", "\u00EAtre"));
+ STOP.put("de", Set.of("aber", "auch", "auf", "aus", "bei", "bitte", "das", "dem", "den", "der", "dich", "die", "diese", "diesen", "dieser", "dieses", "dir", "ein", "eine", "einem", "einen", "einer", "f\u00FCr", "gibt", "habe", "haben", "heute", "ich", "im", "in", "ist", "jetzt", "kann", "kannst", "mein", "meine", "meinem", "meinen", "meiner", "mich", "mir", "mit", "nach", "nicht", "noch", "oder", "sich", "sind", "und", "uns", "von", "wann", "was", "welche", "werden", "wie", "wir", "wird", "wo", "wurde", "zu", "zum", "zur"));
+ STOP.put("es", Set.of("al", "algo", "aqu\u00ED", "aunque", "como", "con", "cuando", "del", "donde", "dos", "el", "entre", "es", "esa", "ese", "eso", "esta", "este", "esto", "est\u00E1", "fue", "fueron", "gracias", "han", "hay", "hemos", "hoy", "la", "las", "le", "les", "lo", "los", "mi", "muy", "m\u00E1s", "nada", "necesito", "ni", "nos", "para", "pero", "por", "porque", "puede", "pueden", "que", "quiero", "se", "ser", "sobre", "son", "su", "sus", "tambi\u00E9n", "tengo", "tiene", "tienen", "todo", "tres", "tu", "un", "una", "y", "ya"));
+ STOP.put("pt", Set.of("agora", "ainda", "alguem", "algu\u00E9m", "ali", "antes", "ao", "aos", "aqui", "as", "at\u00E9", "boa", "cad\u00EA", "com", "como", "consigo", "da", "das", "depois", "deu", "do", "dois", "dos", "e", "em", "entao", "ent\u00E3o", "era", "esta", "estamos", "estava", "este", "estou", "est\u00E1", "eu", "ficou", "fiz", "foi", "gostaria", "hoje", "isso", "isto", "ja", "j\u00E1", "mais", "mas", "meu", "meus", "minha", "minhas", "muito", "na", "nada", "nao", "nas", "nenhum", "nenhuma", "ninguem", "ningu\u00E9m", "noite", "nos", "nossa", "nosso", "n\u00E3o", "o", "obrigada", "obrigado", "ol\u00E1", "onde", "ontem", "os", "para", "pela", "pelo", "pode", "podem", "por", "porque", "pra", "preciso", "quando", "que", "quero", "sao", "se", "ser", "seu", "sou", "sua", "s\u00E3o", "tambem", "tamb\u00E9m", "tarde", "tem", "tenho", "tr\u00EAs", "tudo", "t\u00E1", "um", "uma", "vc", "vcs", "voce", "voces", "voc\u00EA", "voc\u00EAs", "\u00E9"));
+ STOP.put("it", Set.of("abbiamo", "adesso", "agli", "alla", "alle", "anche", "ancora", "avete", "che", "ci", "ciao", "col", "come", "con", "da", "dagli", "dal", "dalla", "dallo", "degli", "dei", "del", "della", "delle", "dello", "deve", "devo", "devono", "di", "dove", "e", "ed", "era", "fra", "gi\u00E0", "gli", "grazie", "ha", "hai", "hanno", "ho", "ieri", "il", "la", "le", "lo", "mai", "mi", "mia", "mio", "molto", "ne", "negli", "nel", "nell", "nella", "non", "o", "oggi", "per", "perche", "pi\u00F9", "poco", "quando", "questa", "questo", "scusa", "sempre", "si", "sono", "stata", "stato", "su", "sua", "sul", "sulla", "sulle", "tra", "tuo", "un", "una", "uno", "voglio", "vorrei", "\u00E8"));
+ STOP.put("nl", Set.of("aan", "dat", "deze", "door", "een", "het", "is", "maar", "met", "naar", "niet", "ook", "op", "te", "van", "voor", "worden", "wordt", "zijn"));
+ STOP.put("sv", Set.of("appen", "aterbetala", "aterbetalning", "att", "behover", "beh\u00F6ver", "blivit", "debiterade", "dessa", "detta", "faktura", "fel", "fr\u00E5n", "f\u00E5r", "ganger", "g\u00E5ng", "g\u00E5nger", "g\u00F6r", "g\u00F6ra", "hittar", "hjalp", "hjalpa", "hj\u00E4lp", "hj\u00E4lpa", "h\u00E4r", "installningarna", "installningen", "inst\u00E4llningarna", "inst\u00E4llningen", "inte", "jag", "kommer", "kraschar", "kvittot", "losenord", "l\u00F6senord", "mig", "min", "mitt", "m\u00E5ste", "nar", "n\u00E4r", "n\u00E4sta", "och", "ocks\u00E5", "om", "oppnar", "paket", "ska", "skicka", "skulle", "sparningen", "sp\u00E5rningen", "s\u00E4ger", "till", "tva", "tv\u00E5", "upp", "uppdaterats", "utan", "vi", "vill", "\u00E4r", "\u00E5terbetala", "\u00E5terbetalning", "\u00F6ppnar"));
+ STOP.put("ro", Set.of("aceasta", "aceast\u0103", "acest", "acesta", "acum", "ale", "care", "dar", "din", "dup\u0103", "este", "foarte", "fost", "f\u0103r\u0103", "lui", "mi", "nu", "pentru", "p\u00E2n\u0103", "sunt", "s\u0103", "trebuie", "vreau", "v\u0103", "\u00EEn", "\u0219i", "\u021Bi"));
+ STOP.put("bn", Set.of("abar", "ajke", "akhon", "amader", "amake", "amar", "ami", "amra", "apnake", "apnar", "apnara", "apni", "ar", "asbe", "bhai", "bhalo", "bolte", "bolun", "chai", "chaina", "dhonnobad", "dilam", "dite", "diye", "diyechi", "dorkar", "duibar", "ei", "eita", "ekbar", "ekhon", "ekhono", "ekta", "ferot", "geche", "gese", "hobe", "hocche", "hoise", "hoye", "hoyeche", "hoyni", "jabe", "jodi", "jonno", "kalke", "keno", "keu", "kharap", "khub", "ki", "kibhabe", "kichu", "kintu", "kivabe", "kobe", "kokhon", "korbo", "korchi", "kore", "koreche", "korechi", "koren", "korlam", "korsi", "korte", "korun", "kothay", "koto", "lagbe", "moddhe", "nai", "niye", "oi", "oita", "onek", "ota", "pabo", "paini", "parben", "parbo", "parchi", "parchina", "peyechi", "sathe", "shathe", "shob", "shomossa", "somossa", "tader", "tahole", "taka", "theke", "tomake", "tomar", "tomra", "tumi", "valo"));
+ STOP.put("az", Set.of("amma", "ancaq", "artiq", "art\u0131q", "bir", "biz", "bu", "cox", "daha", "deyil", "d\u0259", "eger", "g\u00F6r\u0259", "h\u0259m", "h\u0259r", "ile", "il\u0259", "is\u0259", "kimi", "lakin", "m\u0259n", "n\u0259", "olan", "olmasa", "olub", "onlar", "siz", "sonra", "s\u0259n", "ucun", "var", "ve", "v\u0259", "yalniz", "yaln\u0131z", "yox", "yoxdur", "\u00E7ox", "\u00FC\u00E7\u00FCn", "\u0259g\u0259r"));
+ STOP_WORDS = Collections.unmodifiableMap(STOP);
+ }
+
+ /** Distinctive Swedish words that name the language from a short input. */
+ public static final Set<String> SHORT_SWEDISH_WORDS = Set.of(
+ "abonnemanget", "aterbetalning", "atkomst", "behover", "beh\u00F6ver", "betalningen",
+ "fakturan", "felmeddelande", "glomt", "gl\u00F6mt", "inloggningen", "installningen",
+ "inst\u00E4llningen", "kvitto", "kvittot", "losenord", "l\u00F6senord", "sparningen",
+ "sp\u00E5rningen", "\u00E5terbetalning", "\u00E5tkomst");
+
+ /** Letters whose presence is evidence the text is not English. */
+ public static final Set<String> NON_EN_DIACRITICS = Set.of(
+ "\u00DF", "\u00E0", "\u00E1", "\u00E2", "\u00E3", "\u00E4", "\u00E5", "\u00E6", "\u00E7",
+ "\u00E8", "\u00E9", "\u00EA", "\u00EB", "\u00EC", "\u00ED", "\u00EE", "\u00EF", "\u00F1",
+ "\u00F2", "\u00F3", "\u00F4", "\u00F5", "\u00F6", "\u00F8", "\u00F9", "\u00FA", "\u00FB",
+ "\u00FC", "\u00FD", "\u00FF", "\u0101", "\u0103", "\u0105", "\u0107", "\u010D", "\u010F",
+ "\u0111", "\u0113", "\u0119", "\u011B", "\u011F", "\u0123", "\u012B", "\u0131", "\u0137",
+ "\u013C", "\u0142", "\u0144", "\u0146", "\u0148", "\u0151", "\u0153", "\u0159", "\u015B",
+ "\u015F", "\u0161", "\u0163", "\u0165", "\u016B", "\u016F", "\u0171", "\u017A", "\u017C",
+ "\u017E", "\u0219", "\u021B", "\u0259");
+
+ /** Function words several lists hold, which therefore name no language. */
+ public static final Set<String> SHARED_WORDS = Set.of(
+ "as", "como", "con", "da", "das", "del", "dois", "dos", "e", "era", "esta", "este",
+ "est\u00E1", "f\u00E5r", "hej", "il", "in", "is", "ja", "jo", "kommer", "la", "le", "les",
+ "lo", "mais", "mi", "mig", "min", "nada", "nej", "nos", "o", "om", "para", "por", "porque",
+ "quando", "que", "se", "ser", "skulle", "su", "sua", "tack", "tu", "un", "una", "vi",
+ "was");
+
+ /** Markers shared by Swedish and Danish: evidence of non-English, not of which. */
+ public static final Set<String> NORDIC_OVERLAP_WORDS = Set.of(
+ "f\u00E5r", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi");
+
+ /** English function words no other list holds. */
+ public static final Set<String> EN_ONLY_WORDS = Set.of(
+ "and", "are", "at", "be", "but", "can", "for", "from", "has", "have", "i", "it", "not",
+ "of", "on", "please", "that", "the", "their", "there", "this", "to", "we", "were", "what",
+ "which", "will", "with", "would", "you");
+
+ /** Words that collide with English, counted once however often they repeat. */
+ public static final Set<String> EN_COLLISION_WORDS = Set.of(
+ "care", "come", "do", "im", "per", "plus", "son", "todo");
+
+ /**
+ * A diacritic rate at or above this is evidence the text is not English, even with no stopword
+ * match. Measured over every character, so one accented loanword in a short sentence clears it
+ * -- which is what the English rescue exists to answer.
+ */
+ public static final double NON_EN_DIACRITIC_RATE = 0.02;
+
+ /** Above this rate, English function words no longer rescue the text. */
+ public static final double ENGLISH_RESCUE_DIACRITIC_RATE = 0.06;
+
+ /** Non-Latin share at which non-Latin text is not for the English checkpoint. */
+ public static final double NON_LATIN_FRACTION = 0.2;
+
+ /** The lower share that counts when there are enough letters to dilute it. */
+ public static final double NON_LATIN_MIN_FRACTION = 0.1;
+
+ /** How many non-Latin letters make the lower share count. */
+ public static final int NON_LATIN_MIN_LETTERS = 10;
+}
diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
index b9af452c..2da980c0 100644
--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java
@@ -304,6 +304,237 @@ public final class UnicodeTables {
123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258
};
+ /**
+ * Python's {@code \\d}: the Nd category. Needed because Python's {@code \\w} is
+ * letters plus every numeric category, and the regexes subtract {@code \\d} from it.
+ *
+ * <p>64 ranges covering 680 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] DIGIT = {
+ 48, 57, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534, 2543, 2662, 2671, 2790,
+ 2799, 2918, 2927, 3046, 3055, 3174, 3183, 3302, 3311, 3430, 3439, 3558, 3567, 3664,
+ 3673, 3792, 3801, 3872, 3881, 4160, 4169, 4240, 4249, 6112, 6121, 6160, 6169, 6470,
+ 6479, 6608, 6617, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248,
+ 7257, 42528, 42537, 43216, 43225, 43264, 43273, 43472, 43481, 43504, 43513, 43600,
+ 43609, 44016, 44025, 65296, 65305, 66720, 66729, 68912, 68921, 69734, 69743, 69872,
+ 69881, 69942, 69951, 70096, 70105, 70384, 70393, 70736, 70745, 70864, 70873, 71248,
+ 71257, 71360, 71369, 71472, 71481, 71904, 71913, 72016, 72025, 72784, 72793, 73040,
+ 73049, 73120, 73129, 73552, 73561, 92768, 92777, 92864, 92873, 93008, 93017, 120782,
+ 120831, 123200, 123209, 123632, 123641, 124144, 124153, 125264, 125273, 130032, 130041
+ };
+
+ /**
+ * Python's {@code str.isupper()} for one character.
+ *
+ * <p>651 ranges covering 1951 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] UPPER = {
+ 65, 90, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264, 266, 266,
+ 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282, 284,
+ 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300,
+ 302, 302, 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319,
+ 319, 321, 321, 323, 323, 325, 325, 327, 327, 330, 330, 332, 332, 334, 334, 336, 336,
+ 338, 338, 340, 340, 342, 342, 344, 344, 346, 346, 348, 348, 350, 350, 352, 352, 354,
+ 354, 356, 356, 358, 358, 360, 360, 362, 362, 364, 364, 366, 366, 368, 368, 370, 370,
+ 372, 372, 374, 374, 376, 377, 379, 379, 381, 381, 385, 386, 388, 388, 390, 391, 393,
+ 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416, 418, 418, 420, 420, 422, 423,
+ 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 444, 444, 452, 452, 455,
+ 455, 458, 458, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473,
+ 475, 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492,
+ 492, 494, 494, 497, 497, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512,
+ 514, 514, 516, 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530,
+ 530, 532, 532, 534, 534, 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546,
+ 548, 548, 550, 550, 552, 552, 554, 554, 556, 556, 558, 558, 560, 560, 562, 562, 570,
+ 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588, 588, 590, 590, 880, 880,
+ 882, 882, 886, 886, 895, 895, 902, 902, 904, 906, 908, 908, 910, 911, 913, 929, 931,
+ 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988, 990, 990, 992, 992, 994, 994,
+ 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006, 1012, 1012, 1015,
+ 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, 1126, 1126, 1128,
+ 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142,
+ 1142, 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164,
+ 1164, 1166, 1166, 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178,
+ 1178, 1180, 1180, 1182, 1182, 1184, 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192,
+ 1192, 1194, 1194, 1196, 1196, 1198, 1198, 1200, 1200, 1202, 1202, 1204, 1204, 1206,
+ 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214, 1216, 1217, 1219, 1219, 1221,
+ 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232, 1234, 1234, 1236,
+ 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, 1248, 1248, 1250,
+ 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264,
+ 1264, 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278,
+ 1278, 1280, 1280, 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292,
+ 1292, 1294, 1294, 1296, 1296, 1298, 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306,
+ 1306, 1308, 1308, 1310, 1310, 1312, 1312, 1314, 1314, 1316, 1316, 1318, 1318, 1320,
+ 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366, 4256, 4293, 4295, 4295, 4301,
+ 4301, 5024, 5109, 7312, 7354, 7357, 7359, 7680, 7680, 7682, 7682, 7684, 7684, 7686,
+ 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700,
+ 7700, 7702, 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714,
+ 7714, 7716, 7716, 7718, 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728,
+ 7728, 7730, 7730, 7732, 7732, 7734, 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742,
+ 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750, 7750, 7752, 7752, 7754, 7754, 7756,
+ 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766, 7766, 7768, 7768, 7770,
+ 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782, 7782, 7784,
+ 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798,
+ 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812,
+ 7812, 7814, 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826,
+ 7826, 7828, 7828, 7838, 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848,
+ 7848, 7850, 7850, 7852, 7852, 7854, 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862,
+ 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870, 7870, 7872, 7872, 7874, 7874, 7876,
+ 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886, 7886, 7888, 7888, 7890,
+ 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902, 7902, 7904,
+ 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918,
+ 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932,
+ 7932, 7934, 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025,
+ 8025, 8027, 8027, 8029, 8029, 8031, 8031, 8040, 8047, 8120, 8123, 8136, 8139, 8152,
+ 8155, 8168, 8172, 8184, 8187, 8450, 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469,
+ 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496, 8499, 8510,
+ 8511, 8517, 8517, 8544, 8559, 8579, 8579, 9398, 9423, 11264, 11311, 11360, 11360,
+ 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373, 11376, 11378, 11378,
+ 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398, 11400, 11400,
+ 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412, 11412,
+ 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424,
+ 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436,
+ 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448,
+ 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460,
+ 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472,
+ 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484,
+ 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506,
+ 42560, 42560, 42562, 42562, 42564, 42564, 42566, 42566, 42568, 42568, 42570, 42570,
+ 42572, 42572, 42574, 42574, 42576, 42576, 42578, 42578, 42580, 42580, 42582, 42582,
+ 42584, 42584, 42586, 42586, 42588, 42588, 42590, 42590, 42592, 42592, 42594, 42594,
+ 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602, 42604, 42604, 42624, 42624,
+ 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632, 42634, 42634, 42636, 42636,
+ 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, 42646, 42648, 42648,
+ 42650, 42650, 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794,
+ 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808,
+ 42810, 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820,
+ 42822, 42822, 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832,
+ 42834, 42834, 42836, 42836, 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844,
+ 42846, 42846, 42848, 42848, 42850, 42850, 42852, 42852, 42854, 42854, 42856, 42856,
+ 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873, 42875, 42875, 42877, 42878,
+ 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891, 42893, 42893,
+ 42896, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908, 42908,
+ 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920,
+ 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940,
+ 42942, 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960,
+ 42966, 42966, 42968, 42968, 42997, 42997, 65313, 65338, 66560, 66599, 66736, 66771,
+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 68736, 68786, 71840, 71871,
+ 93760, 93791, 119808, 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966,
+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119989, 120016, 120041,
+ 120068, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120120, 120121, 120123,
+ 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120172, 120197, 120224, 120249,
+ 120276, 120301, 120328, 120353, 120380, 120405, 120432, 120457, 120488, 120512, 120546,
+ 120570, 120604, 120628, 120662, 120686, 120720, 120744, 120778, 120778, 125184, 125217,
+ 127280, 127305, 127312, 127337, 127344, 127369
+ };
+
+ /**
+ * Python's {@code str.islower()} for one character.
+ *
+ * <p>671 ranges covering 2544 code points, recorded
+ * from CPython 15.0.0.
+ */
+ private static final int[] LOWER = {
+ 97, 122, 170, 170, 181, 181, 186, 186, 223, 246, 248, 255, 257, 257, 259, 259, 261,
+ 261, 263, 263, 265, 265, 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277,
+ 279, 279, 281, 281, 283, 283, 285, 285, 287, 287, 289, 289, 291, 291, 293, 293, 295,
+ 295, 297, 297, 299, 299, 301, 301, 303, 303, 305, 305, 307, 307, 309, 309, 311, 312,
+ 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, 324, 324, 326, 326, 328, 329, 331,
+ 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, 343, 343, 345, 345, 347, 347,
+ 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361, 361, 363, 363, 365,
+ 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 378, 378, 380, 380, 382, 384,
+ 387, 387, 389, 389, 392, 392, 396, 397, 402, 402, 405, 405, 409, 411, 414, 414, 417,
+ 417, 419, 419, 421, 421, 424, 424, 426, 427, 429, 429, 432, 432, 436, 436, 438, 438,
+ 441, 442, 445, 447, 454, 454, 457, 457, 460, 460, 462, 462, 464, 464, 466, 466, 468,
+ 468, 470, 470, 472, 472, 474, 474, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485,
+ 487, 487, 489, 489, 491, 491, 493, 493, 495, 496, 499, 499, 501, 501, 505, 505, 507,
+ 507, 509, 509, 511, 511, 513, 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523,
+ 525, 525, 527, 527, 529, 529, 531, 531, 533, 533, 535, 535, 537, 537, 539, 539, 541,
+ 541, 543, 543, 545, 545, 547, 547, 549, 549, 551, 551, 553, 553, 555, 555, 557, 557,
+ 559, 559, 561, 561, 563, 569, 572, 572, 575, 576, 578, 578, 583, 583, 585, 585, 587,
+ 587, 589, 589, 591, 659, 661, 696, 704, 705, 736, 740, 837, 837, 881, 881, 883, 883,
+ 887, 887, 890, 893, 912, 912, 940, 974, 976, 977, 981, 983, 985, 985, 987, 987, 989,
+ 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005,
+ 1005, 1007, 1011, 1013, 1013, 1016, 1016, 1019, 1020, 1072, 1119, 1121, 1121, 1123,
+ 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137,
+ 1137, 1139, 1139, 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151,
+ 1151, 1153, 1153, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173,
+ 1173, 1175, 1175, 1177, 1177, 1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187,
+ 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, 1197, 1199, 1199, 1201,
+ 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215,
+ 1215, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230,
+ 1231, 1233, 1233, 1235, 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245,
+ 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, 1255, 1255, 1257, 1257, 1259,
+ 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273,
+ 1273, 1275, 1275, 1277, 1277, 1279, 1279, 1281, 1281, 1283, 1283, 1285, 1285, 1287,
+ 1287, 1289, 1289, 1291, 1291, 1293, 1293, 1295, 1295, 1297, 1297, 1299, 1299, 1301,
+ 1301, 1303, 1303, 1305, 1305, 1307, 1307, 1309, 1309, 1311, 1311, 1313, 1313, 1315,
+ 1315, 1317, 1317, 1319, 1319, 1321, 1321, 1323, 1323, 1325, 1325, 1327, 1327, 1376,
+ 1416, 4304, 4346, 4348, 4351, 5112, 5117, 7296, 7304, 7424, 7615, 7681, 7681, 7683,
+ 7683, 7685, 7685, 7687, 7687, 7689, 7689, 7691, 7691, 7693, 7693, 7695, 7695, 7697,
+ 7697, 7699, 7699, 7701, 7701, 7703, 7703, 7705, 7705, 7707, 7707, 7709, 7709, 7711,
+ 7711, 7713, 7713, 7715, 7715, 7717, 7717, 7719, 7719, 7721, 7721, 7723, 7723, 7725,
+ 7725, 7727, 7727, 7729, 7729, 7731, 7731, 7733, 7733, 7735, 7735, 7737, 7737, 7739,
+ 7739, 7741, 7741, 7743, 7743, 7745, 7745, 7747, 7747, 7749, 7749, 7751, 7751, 7753,
+ 7753, 7755, 7755, 7757, 7757, 7759, 7759, 7761, 7761, 7763, 7763, 7765, 7765, 7767,
+ 7767, 7769, 7769, 7771, 7771, 7773, 7773, 7775, 7775, 7777, 7777, 7779, 7779, 7781,
+ 7781, 7783, 7783, 7785, 7785, 7787, 7787, 7789, 7789, 7791, 7791, 7793, 7793, 7795,
+ 7795, 7797, 7797, 7799, 7799, 7801, 7801, 7803, 7803, 7805, 7805, 7807, 7807, 7809,
+ 7809, 7811, 7811, 7813, 7813, 7815, 7815, 7817, 7817, 7819, 7819, 7821, 7821, 7823,
+ 7823, 7825, 7825, 7827, 7827, 7829, 7837, 7839, 7839, 7841, 7841, 7843, 7843, 7845,
+ 7845, 7847, 7847, 7849, 7849, 7851, 7851, 7853, 7853, 7855, 7855, 7857, 7857, 7859,
+ 7859, 7861, 7861, 7863, 7863, 7865, 7865, 7867, 7867, 7869, 7869, 7871, 7871, 7873,
+ 7873, 7875, 7875, 7877, 7877, 7879, 7879, 7881, 7881, 7883, 7883, 7885, 7885, 7887,
+ 7887, 7889, 7889, 7891, 7891, 7893, 7893, 7895, 7895, 7897, 7897, 7899, 7899, 7901,
+ 7901, 7903, 7903, 7905, 7905, 7907, 7907, 7909, 7909, 7911, 7911, 7913, 7913, 7915,
+ 7915, 7917, 7917, 7919, 7919, 7921, 7921, 7923, 7923, 7925, 7925, 7927, 7927, 7929,
+ 7929, 7931, 7931, 7933, 7933, 7935, 7943, 7952, 7957, 7968, 7975, 7984, 7991, 8000,
+ 8005, 8016, 8023, 8032, 8039, 8048, 8061, 8064, 8071, 8080, 8087, 8096, 8103, 8112,
+ 8116, 8118, 8119, 8126, 8126, 8130, 8132, 8134, 8135, 8144, 8147, 8150, 8151, 8160,
+ 8167, 8178, 8180, 8182, 8183, 8305, 8305, 8319, 8319, 8336, 8348, 8458, 8458, 8462,
+ 8463, 8467, 8467, 8495, 8495, 8500, 8500, 8505, 8505, 8508, 8509, 8518, 8521, 8526,
+ 8526, 8560, 8575, 8580, 8580, 9424, 9449, 11312, 11359, 11361, 11361, 11365, 11366,
+ 11368, 11368, 11370, 11370, 11372, 11372, 11377, 11377, 11379, 11380, 11382, 11389,
+ 11393, 11393, 11395, 11395, 11397, 11397, 11399, 11399, 11401, 11401, 11403, 11403,
+ 11405, 11405, 11407, 11407, 11409, 11409, 11411, 11411, 11413, 11413, 11415, 11415,
+ 11417, 11417, 11419, 11419, 11421, 11421, 11423, 11423, 11425, 11425, 11427, 11427,
+ 11429, 11429, 11431, 11431, 11433, 11433, 11435, 11435, 11437, 11437, 11439, 11439,
+ 11441, 11441, 11443, 11443, 11445, 11445, 11447, 11447, 11449, 11449, 11451, 11451,
+ 11453, 11453, 11455, 11455, 11457, 11457, 11459, 11459, 11461, 11461, 11463, 11463,
+ 11465, 11465, 11467, 11467, 11469, 11469, 11471, 11471, 11473, 11473, 11475, 11475,
+ 11477, 11477, 11479, 11479, 11481, 11481, 11483, 11483, 11485, 11485, 11487, 11487,
+ 11489, 11489, 11491, 11492, 11500, 11500, 11502, 11502, 11507, 11507, 11520, 11557,
+ 11559, 11559, 11565, 11565, 42561, 42561, 42563, 42563, 42565, 42565, 42567, 42567,
+ 42569, 42569, 42571, 42571, 42573, 42573, 42575, 42575, 42577, 42577, 42579, 42579,
+ 42581, 42581, 42583, 42583, 42585, 42585, 42587, 42587, 42589, 42589, 42591, 42591,
+ 42593, 42593, 42595, 42595, 42597, 42597, 42599, 42599, 42601, 42601, 42603, 42603,
+ 42605, 42605, 42625, 42625, 42627, 42627, 42629, 42629, 42631, 42631, 42633, 42633,
+ 42635, 42635, 42637, 42637, 42639, 42639, 42641, 42641, 42643, 42643, 42645, 42645,
+ 42647, 42647, 42649, 42649, 42651, 42653, 42787, 42787, 42789, 42789, 42791, 42791,
+ 42793, 42793, 42795, 42795, 42797, 42797, 42799, 42801, 42803, 42803, 42805, 42805,
+ 42807, 42807, 42809, 42809, 42811, 42811, 42813, 42813, 42815, 42815, 42817, 42817,
+ 42819, 42819, 42821, 42821, 42823, 42823, 42825, 42825, 42827, 42827, 42829, 42829,
+ 42831, 42831, 42833, 42833, 42835, 42835, 42837, 42837, 42839, 42839, 42841, 42841,
+ 42843, 42843, 42845, 42845, 42847, 42847, 42849, 42849, 42851, 42851, 42853, 42853,
+ 42855, 42855, 42857, 42857, 42859, 42859, 42861, 42861, 42863, 42872, 42874, 42874,
+ 42876, 42876, 42879, 42879, 42881, 42881, 42883, 42883, 42885, 42885, 42887, 42887,
+ 42892, 42892, 42894, 42894, 42897, 42897, 42899, 42901, 42903, 42903, 42905, 42905,
+ 42907, 42907, 42909, 42909, 42911, 42911, 42913, 42913, 42915, 42915, 42917, 42917,
+ 42919, 42919, 42921, 42921, 42927, 42927, 42933, 42933, 42935, 42935, 42937, 42937,
+ 42939, 42939, 42941, 42941, 42943, 42943, 42945, 42945, 42947, 42947, 42952, 42952,
+ 42954, 42954, 42961, 42961, 42963, 42963, 42965, 42965, 42967, 42967, 42969, 42969,
+ 42994, 42996, 42998, 42998, 43000, 43002, 43824, 43866, 43868, 43881, 43888, 43967,
+ 64256, 64262, 64275, 64279, 65345, 65370, 66600, 66639, 66776, 66811, 66967, 66977,
+ 66979, 66993, 66995, 67001, 67003, 67004, 67456, 67456, 67459, 67461, 67463, 67504,
+ 67506, 67514, 68800, 68850, 71872, 71903, 93792, 93823, 119834, 119859, 119886, 119892,
+ 119894, 119911, 119938, 119963, 119990, 119993, 119995, 119995, 119997, 120003, 120005,
+ 120015, 120042, 120067, 120094, 120119, 120146, 120171, 120198, 120223, 120250, 120275,
+ 120302, 120327, 120354, 120379, 120406, 120431, 120458, 120485, 120514, 120538, 120540,
+ 120545, 120572, 120596, 120598, 120603, 120630, 120654, 120656, 120661, 120688, 120712,
+ 120714, 120719, 120746, 120770, 120772, 120777, 120779, 120779, 122624, 122633, 122635,
+ 122654, 122661, 122666, 122928, 122989, 125218, 125251
+ };
+
/** Python's {@code str.isalpha()} for one code point. */
public static boolean isAlpha(int codePoint) {
return contains(ALPHA, codePoint);
@@ -319,6 +550,86 @@ public final class UnicodeTables {
return contains(COMBINING, codePoint);
}
+ /** Python's {@code \d}: the Nd category. The JDK misses 30 of these at Unicode 13. */
+ public static boolean isDigit(int codePoint) {
+ return contains(DIGIT, codePoint);
+ }
+
+ /**
+ * Python's {@code \w} for a str: alphanumeric or underscore.
+ *
+ * <p>Spelled out because the regexes in {@code laya/lang.py} subtract from it --
+ * {@code [^\W\d_]} is this minus digits and underscore, {@code [^\W_]} is this minus
+ * underscore -- and Java's own {@code \w} is a different set again (it admits combining
+ * marks and join controls, which Python's does not).
+ */
+ public static boolean isPythonWordChar(int codePoint) {
+ return codePoint == '_' || isWordChar(codePoint) || isDigit(codePoint);
+ }
+
+ /** Python's {@code [^\W_]}: a word character that is not the underscore. */
+ public static boolean isWordOrDigit(int codePoint) {
+ return isWordChar(codePoint) || isDigit(codePoint);
+ }
+
+ /**
+ * Python's {@code str.isupper()} for one code point.
+ *
+ * <p>A table, because {@code Character.isUpperCase} misses 40 of these and
+ * {@code Character.isLowerCase} misses 200 at Unicode 13 -- U+10FC, a Georgian modifier
+ * letter Python calls lowercase through Other_Lowercase, among them. Detection reads the case
+ * of the first letter of a non-Latin run to tell a proper noun from a request, so a letter
+ * whose case the JDK does not know would change that decision.
+ */
+ public static boolean isUpper(int codePoint) {
+ return contains(UPPER, codePoint);
+ }
+
+ /** Python's {@code str.islower()} for one code point. */
+ public static boolean isLower(int codePoint) {
+ return contains(LOWER, codePoint);
+ }
+
+ /** Whether a code point has case at all, which is what {@code str.isupper()} requires one of. */
+ public static boolean isCased(int codePoint) {
+ return isUpper(codePoint) || isLower(codePoint)
+ || Character.getType(codePoint) == Character.TITLECASE_LETTER;
+ }
+
+ /**
+ * Python's {@code str.isupper()} for a whole string: at least one cased code point, and every
+ * cased code point uppercase.
+ */
+ public static boolean isUpperString(String text) {
+ boolean sawCased = false;
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (!isCased(codePoint)) {
+ continue;
+ }
+ if (!isUpper(codePoint)) {
+ return false;
+ }
+ sawCased = true;
+ }
+ return sawCased;
+ }
+
+ /** Whether any code point of {@code text} is lowercase, as Python's {@code islower} sees it. */
+ public static boolean hasLower(String text) {
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (isLower(codePoint)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Python's {@code str.lower()}, which is not Java's.
*
diff --git a/laya-java/scripts/gen_language_tables.py b/laya-java/scripts/gen_language_tables.py
new file mode 100755
index 00000000..8062a0b1
--- /dev/null
+++ b/laya-java/scripts/gen_language_tables.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Generate `laya.lang`'s word lists and script ranges as Java source.
+
+Compiled in rather than read from `fixtures/lang_tables.json`, because a runtime library must not
+depend on a test fixture being on disk. The fixture keeps its job: `LanguageTablesTest` asserts the
+compiled copy equals it, so a table that drifts from Python fails a test rather than changing a
+routing decision quietly.
+
+TWO ORDERINGS ARE LOAD-BEARING and are preserved deliberately:
+
+ * `_SCRIPT_RANGES` is scanned in order and the FIRST matching range wins, so a reordering
+ changes which script a code point is counted under.
+ * `_STOP` is iterated to build the per-language scores, and `max` returns the FIRST of equal
+ values, so its insertion order decides ties between languages with the same score.
+
+ python laya-java/scripts/gen_language_tables.py # write
+ python laya-java/scripts/gen_language_tables.py --check # fail if the committed copy differs
+"""
+import argparse
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+REPO = os.path.dirname(os.path.dirname(HERE))
+TARGET = os.path.join(
+ HERE, "..", "laya-java", "src", "main", "java", "com", "convaiinnovations", "laya", "lang",
+ "LanguageTables.java")
+
+
+def quote(text):
+ """A Java string literal, with every non-ASCII character escaped so the file stays ASCII."""
+ out = ['"']
+ for ch in text:
+ if ch == '"':
+ out.append('\\"')
+ elif ch == "\\":
+ out.append("\\\\")
+ elif " " <= ch <= "~":
+ out.append(ch)
+ else:
+ encoded = ch.encode("utf-16-be")
+ for i in range(0, len(encoded), 2):
+ out.append("\\u%02X%02X" % (encoded[i], encoded[i + 1]))
+ out.append('"')
+ return "".join(out)
+
+
+def words_block(name, words, doc, sort=True):
+ items = sorted(words) if sort else list(words)
+ lines, current = [], " "
+ for word in items:
+ piece = quote(word) + ", "
+ if len(current) + len(piece) > 104:
+ lines.append(current.rstrip())
+ current = " "
+ current += piece
+ if current.strip():
+ lines.append(current.rstrip().rstrip(","))
+ return (' /** %s */\n public static final Set<String> %s = Set.of(\n%s);\n'
+ % (doc, name, "\n".join(lines)))
+
+
+def render():
+ sys.path.insert(0, REPO)
+ from laya import lang
+
+ script_rows = []
+ for name, ranges in lang._SCRIPT_RANGES:
+ pairs = ", ".join("%d, %d" % (lo, hi) for lo, hi in ranges)
+ script_rows.append(' new Script(%s, new int[] {%s})' % (quote(name), pairs))
+
+ stop_rows = []
+ for code, words in lang._STOP.items():
+ items = ", ".join(quote(w) for w in sorted(words))
+ stop_rows.append(' STOP.put(%s, Set.of(%s));' % (quote(code), items))
+
+ header = '''// GENERATED by laya-java/scripts/gen_language_tables.py -- do not edit.
+//
+// Regenerate with:
+// python laya-java/scripts/gen_language_tables.py
+// and `--check` fails when the committed copy no longer matches laya/lang.py.
+package com.convaiinnovations.laya.lang;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The word lists, script ranges and thresholds {@code laya/lang.py} detects with.
+ *
+ * <p>Compiled in rather than read from {@code fixtures/lang_tables.json}: a runtime library must
+ * not need a test fixture on disk. The fixture keeps its job -- a test asserts this copy equals it,
+ * so a table that drifts from Python fails a test rather than changing a routing decision quietly.
+ *
+ * <p><b>Two orderings are load-bearing.</b> {@link #SCRIPT_RANGES} is scanned in order and the
+ * first matching range wins, so a reordering changes which script a code point counts under. And
+ * {@link #STOP_WORDS} is iterated to score languages, where the winner is the first of equal
+ * scores, so its order decides ties.
+ */
+public final class LanguageTables {
+
+ private LanguageTables() {
+ }
+
+ /** One named non-Latin script and the code-point ranges that belong to it. */
+ public record Script(String name, int[] ranges) {
+ }
+
+'''
+ body = (
+ ' /**\n'
+ ' * Named scripts, in scan order. The first range that contains a code point wins, so\n'
+ ' * this is a sequence and not a set.\n'
+ ' */\n'
+ ' public static final List<Script> SCRIPT_RANGES = List.of(\n'
+ + ",\n".join(script_rows) + ");\n\n"
+ ' /**\n'
+ ' * Function words per language, in the order Python declares them.\n'
+ ' *\n'
+ ' * <p>A {@link LinkedHashMap}, because the language that wins a tied score is the\n'
+ ' * first one iterated.\n'
+ ' */\n'
+ ' public static final Map<String, Set<String>> STOP_WORDS;\n\n'
+ ' static {\n'
+ ' Map<String, Set<String>> STOP = new LinkedHashMap<>();\n'
+ + "\n".join(stop_rows) + "\n"
+ ' STOP_WORDS = Collections.unmodifiableMap(STOP);\n'
+ ' }\n\n'
+ + words_block("SHORT_SWEDISH_WORDS", lang._SHORT_SWEDISH_WORDS,
+ "Distinctive Swedish words that name the language from a short input.")
+ + "\n"
+ + words_block("NON_EN_DIACRITICS", lang._NON_EN_DIACRITICS,
+ "Letters whose presence is evidence the text is not English.")
+ + "\n"
+ + words_block("SHARED_WORDS", lang._SHARED_WORDS,
+ "Function words several lists hold, which therefore name no language.")
+ + "\n"
+ + words_block("NORDIC_OVERLAP_WORDS", lang._NORDIC_OVERLAP_WORDS,
+ "Markers shared by Swedish and Danish: evidence of non-English, not of which.")
+ + "\n"
+ + words_block("EN_ONLY_WORDS", lang._EN_ONLY_WORDS,
+ "English function words no other list holds.")
+ + "\n"
+ + words_block("EN_COLLISION_WORDS", lang._EN_COLLISION_WORDS,
+ "Words that collide with English, counted once however often they repeat.")
+ + '''
+ /**
+ * A diacritic rate at or above this is evidence the text is not English, even with no stopword
+ * match. Measured over every character, so one accented loanword in a short sentence clears it
+ * -- which is what the English rescue exists to answer.
+ */
+ public static final double NON_EN_DIACRITIC_RATE = %r;
+
+ /** Above this rate, English function words no longer rescue the text. */
+ public static final double ENGLISH_RESCUE_DIACRITIC_RATE = %r;
+
+ /** Non-Latin share at which non-Latin text is not for the English checkpoint. */
+ public static final double NON_LATIN_FRACTION = %r;
+
+ /** The lower share that counts when there are enough letters to dilute it. */
+ public static final double NON_LATIN_MIN_FRACTION = %r;
+
+ /** How many non-Latin letters make the lower share count. */
+ public static final int NON_LATIN_MIN_LETTERS = %d;
+}
+''' % (lang.NON_EN_DIACRITIC_RATE, lang.ENGLISH_RESCUE_DIACRITIC_RATE,
+ lang.NON_LATIN_FRACTION, lang.NON_LATIN_MIN_FRACTION, lang.NON_LATIN_MIN_LETTERS)
+ )
+ return header + body
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--check", action="store_true",
+ help="do not write; exit 1 if the committed copy would change")
+ args = parser.parse_args(argv)
+ fresh = render()
+ target = os.path.normpath(TARGET)
+ if args.check:
+ try:
+ with open(target, encoding="utf-8") as handle:
+ current = handle.read()
+ except OSError:
+ print("gen_language_tables: %s is missing" % target, file=sys.stderr)
+ return 1
+ if current != fresh:
+ print("gen_language_tables: %s is stale; run "
+ "laya-java/scripts/gen_language_tables.py and commit the result" % target,
+ file=sys.stderr)
+ return 1
+ print("gen_language_tables: LanguageTables.java matches laya/lang.py")
+ return 0
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ with open(target, "w", encoding="utf-8") as handle:
+ handle.write(fresh)
+ print("wrote %s (%d bytes)" % (os.path.relpath(target, REPO), len(fresh)))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.py
index f0c892cc..3e8090dd 100755
--- a/laya-java/scripts/gen_unicode_tables.py
+++ b/laya-java/scripts/gen_unicode_tables.py
@@ -35,6 +35,7 @@ TARGET = os.path.join(
"UnicodeTables.java")
WORD = re.compile(r"[^\W\d_]", re.UNICODE)
+DIGIT = re.compile(r"\d", re.UNICODE)
MAX = 0x110000
SURROGATES = range(0xD800, 0xE000)
@@ -78,6 +79,9 @@ def render():
alpha = ranges(lambda cp: chr(cp).isalpha())
word = ranges(lambda cp: WORD.fullmatch(chr(cp)) is not None)
combining = ranges(lambda cp: unicodedata.combining(chr(cp)) != 0)
+ digit = ranges(lambda cp: DIGIT.fullmatch(chr(cp)) is not None)
+ upper = ranges(lambda cp: chr(cp).isupper())
+ lower = ranges(lambda cp: chr(cp).islower())
# Every code point whose lowercase is more than one code point. In Unicode 15 there is
# exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because
@@ -132,6 +136,14 @@ public final class UnicodeTables {
+ "\n"
+ table("COMBINING", combining,
"Code points with a non-zero canonical combining class.")
+ + "\n"
+ + table("DIGIT", digit,
+ "Python's {@code \\\\d}: the Nd category. Needed because Python's {@code \\\\w} is\n"
+ " * letters plus every numeric category, and the regexes subtract {@code \\\\d} from it.")
+ + "\n"
+ + table("UPPER", upper, "Python's {@code str.isupper()} for one character.")
+ + "\n"
+ + table("LOWER", lower, "Python's {@code str.islower()} for one character.")
)
multi_cases = "\n".join(
@@ -153,6 +165,86 @@ public final class UnicodeTables {
return contains(COMBINING, codePoint);
}
+ /** Python's {@code \\d}: the Nd category. The JDK misses 30 of these at Unicode 13. */
+ public static boolean isDigit(int codePoint) {
+ return contains(DIGIT, codePoint);
+ }
+
+ /**
+ * Python's {@code \\w} for a str: alphanumeric or underscore.
+ *
+ * <p>Spelled out because the regexes in {@code laya/lang.py} subtract from it --
+ * {@code [^\\W\\d_]} is this minus digits and underscore, {@code [^\\W_]} is this minus
+ * underscore -- and Java's own {@code \\w} is a different set again (it admits combining
+ * marks and join controls, which Python's does not).
+ */
+ public static boolean isPythonWordChar(int codePoint) {
+ return codePoint == '_' || isWordChar(codePoint) || isDigit(codePoint);
+ }
+
+ /** Python's {@code [^\\W_]}: a word character that is not the underscore. */
+ public static boolean isWordOrDigit(int codePoint) {
+ return isWordChar(codePoint) || isDigit(codePoint);
+ }
+
+ /**
+ * Python's {@code str.isupper()} for one code point.
+ *
+ * <p>A table, because {@code Character.isUpperCase} misses 40 of these and
+ * {@code Character.isLowerCase} misses 200 at Unicode 13 -- U+10FC, a Georgian modifier
+ * letter Python calls lowercase through Other_Lowercase, among them. Detection reads the case
+ * of the first letter of a non-Latin run to tell a proper noun from a request, so a letter
+ * whose case the JDK does not know would change that decision.
+ */
+ public static boolean isUpper(int codePoint) {
+ return contains(UPPER, codePoint);
+ }
+
+ /** Python's {@code str.islower()} for one code point. */
+ public static boolean isLower(int codePoint) {
+ return contains(LOWER, codePoint);
+ }
+
+ /** Whether a code point has case at all, which is what {@code str.isupper()} requires one of. */
+ public static boolean isCased(int codePoint) {
+ return isUpper(codePoint) || isLower(codePoint)
+ || Character.getType(codePoint) == Character.TITLECASE_LETTER;
+ }
+
+ /**
+ * Python's {@code str.isupper()} for a whole string: at least one cased code point, and every
+ * cased code point uppercase.
+ */
+ public static boolean isUpperString(String text) {
+ boolean sawCased = false;
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (!isCased(codePoint)) {
+ continue;
+ }
+ if (!isUpper(codePoint)) {
+ return false;
+ }
+ sawCased = true;
+ }
+ return sawCased;
+ }
+
+ /** Whether any code point of {@code text} is lowercase, as Python's {@code islower} sees it. */
+ public static boolean hasLower(String text) {
+ int i = 0;
+ while (i < text.length()) {
+ int codePoint = text.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (isLower(codePoint)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Python's {@code str.lower()}, which is not Java's.
*