Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(java): the remaining character tables, and laya.lang's word lists
Adeshina Lasisi8fa95393bd74
diff --git a/.github/workflows/java.yml b/.github/workflows/java.ymlindex 6e1869ae..d41f77a5 100644--- a/.github/workflows/java.yml+++ b/.github/workflows/java.yml@@ -84,6 +84,9 @@ jobs:- name: Unicode tables match CPythonif: ${{ !cancelled() }}run: python laya-java/scripts/gen_unicode_tables.py --check+ - name: Language tables match laya/lang.py+ if: ${{ !cancelled() }}+ run: python laya-java/scripts/gen_language_tables.py --check- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2if: ${{ failure() }}diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.javanew file mode 100644index 00000000..4f1b28da--- /dev/null+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/LanguageTables.java@@ -0,0 +1,148 @@+// GENERATED by laya-java/scripts/gen_language_tables.py -- do not edit.+//+// Regenerate with:+// python laya-java/scripts/gen_language_tables.py+// and `--check` fails when the committed copy no longer matches laya/lang.py.+package com.convaiinnovations.laya.lang;++import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;++/**+ * The word lists, script ranges and thresholds {@code laya/lang.py} detects with.+ *+ * <p>Compiled in rather than read from {@code fixtures/lang_tables.json}: a runtime library must+ * not need a test fixture on disk. The fixture keeps its job -- a test asserts this copy equals it,+ * so a table that drifts from Python fails a test rather than changing a routing decision quietly.+ *+ * <p><b>Two orderings are load-bearing.</b> {@link #SCRIPT_RANGES} is scanned in order and the+ * first matching range wins, so a reordering changes which script a code point counts under. And+ * {@link #STOP_WORDS} is iterated to score languages, where the winner is the first of equal+ * scores, so its order decides ties.+ */+public final class LanguageTables {++ private LanguageTables() {+ }++ /** One named non-Latin script and the code-point ranges that belong to it. */+ public record Script(String name, int[] ranges) {+ }++ /**+ * Named scripts, in scan order. The first range that contains a code point wins, so+ * this is a sequence and not a set.+ */+ public static final List<Script> SCRIPT_RANGES = List.of(+ new Script("greek", new int[] {880, 1023, 7936, 8191}),+ new Script("cyrillic", new int[] {1024, 1327, 11744, 11775, 42560, 42655}),+ new Script("armenian", new int[] {1328, 1423}),+ new Script("hebrew", new int[] {1424, 1535}),+ new Script("arabic", new int[] {1536, 1791, 1872, 1919, 2208, 2303, 64336, 65023, 65136, 65279}),+ new Script("devanagari", new int[] {2304, 2431, 43232, 43263}),+ new Script("bengali", new int[] {2432, 2559}),+ new Script("gurmukhi", new int[] {2560, 2687}),+ new Script("gujarati", new int[] {2688, 2815}),+ new Script("oriya", new int[] {2816, 2943}),+ new Script("tamil", new int[] {2944, 3071}),+ new Script("telugu", new int[] {3072, 3199}),+ new Script("kannada", new int[] {3200, 3327}),+ new Script("malayalam", new int[] {3328, 3455}),+ new Script("sinhala", new int[] {3456, 3583}),+ new Script("thai", new int[] {3584, 3711}),+ new Script("lao", new int[] {3712, 3839}),+ new Script("tibetan", new int[] {3840, 4095}),+ new Script("myanmar", new int[] {4096, 4255}),+ new Script("georgian", new int[] {4256, 4351}),+ new Script("ethiopic", new int[] {4608, 4991}),+ new Script("khmer", new int[] {6016, 6143}),+ new Script("hangul", new int[] {4352, 4607, 12592, 12687, 44032, 55215}),+ new Script("kana", new int[] {12352, 12447, 12448, 12543, 12784, 12799}),+ new Script("han", new int[] {13312, 19903, 19968, 40959, 63744, 64255}));++ /**+ * Function words per language, in the order Python declares them.+ *+ * <p>A {@link LinkedHashMap}, because the language that wins a tied score is the+ * first one iterated.+ */+ public static final Map<String, Set<String>> STOP_WORDS;++ static {+ Map<String, Set<String>> STOP = new LinkedHashMap<>();+ STOP.put("en", Set.of("and", "are", "as", "at", "be", "but", "can", "for", "from", "has", "have", "i", "in", "is", "it", "not", "of", "on", "please", "that", "the", "their", "there", "this", "to", "was", "we", "were", "what", "which", "will", "with", "would", "you"));+ STOP.put("fr", Set.of("alors", "au", "aux", "avec", "bien", "bonjour", "ce", "ces", "cette", "comment", "dans", "des", "deux", "dois", "doit", "donc", "du", "elle", "elles", "est", "et", "fait", "fois", "il", "ils", "je", "jour", "jours", "la", "le", "les", "ma", "mais", "merci", "mes", "mois", "mon", "nous", "ont", "ou", "pas", "peut", "peux", "plus", "pour", "pourquoi", "quand", "que", "qui", "sa", "ses", "sont", "sur", "ta", "tes", "ton", "tous", "tout", "toute", "trois", "tr\u00E8s", "tu", "une", "veut", "veux", "vous", "\u00EAtre"));+ STOP.put("de", Set.of("aber", "auch", "auf", "aus", "bei", "bitte", "das", "dem", "den", "der", "dich", "die", "diese", "diesen", "dieser", "dieses", "dir", "ein", "eine", "einem", "einen", "einer", "f\u00FCr", "gibt", "habe", "haben", "heute", "ich", "im", "in", "ist", "jetzt", "kann", "kannst", "mein", "meine", "meinem", "meinen", "meiner", "mich", "mir", "mit", "nach", "nicht", "noch", "oder", "sich", "sind", "und", "uns", "von", "wann", "was", "welche", "werden", "wie", "wir", "wird", "wo", "wurde", "zu", "zum", "zur"));+ STOP.put("es", Set.of("al", "algo", "aqu\u00ED", "aunque", "como", "con", "cuando", "del", "donde", "dos", "el", "entre", "es", "esa", "ese", "eso", "esta", "este", "esto", "est\u00E1", "fue", "fueron", "gracias", "han", "hay", "hemos", "hoy", "la", "las", "le", "les", "lo", "los", "mi", "muy", "m\u00E1s", "nada", "necesito", "ni", "nos", "para", "pero", "por", "porque", "puede", "pueden", "que", "quiero", "se", "ser", "sobre", "son", "su", "sus", "tambi\u00E9n", "tengo", "tiene", "tienen", "todo", "tres", "tu", "un", "una", "y", "ya"));+ STOP.put("pt", Set.of("agora", "ainda", "alguem", "algu\u00E9m", "ali", "antes", "ao", "aos", "aqui", "as", "at\u00E9", "boa", "cad\u00EA", "com", "como", "consigo", "da", "das", "depois", "deu", "do", "dois", "dos", "e", "em", "entao", "ent\u00E3o", "era", "esta", "estamos", "estava", "este", "estou", "est\u00E1", "eu", "ficou", "fiz", "foi", "gostaria", "hoje", "isso", "isto", "ja", "j\u00E1", "mais", "mas", "meu", "meus", "minha", "minhas", "muito", "na", "nada", "nao", "nas", "nenhum", "nenhuma", "ninguem", "ningu\u00E9m", "noite", "nos", "nossa", "nosso", "n\u00E3o", "o", "obrigada", "obrigado", "ol\u00E1", "onde", "ontem", "os", "para", "pela", "pelo", "pode", "podem", "por", "porque", "pra", "preciso", "quando", "que", "quero", "sao", "se", "ser", "seu", "sou", "sua", "s\u00E3o", "tambem", "tamb\u00E9m", "tarde", "tem", "tenho", "tr\u00EAs", "tudo", "t\u00E1", "um", "uma", "vc", "vcs", "voce", "voces", "voc\u00EA", "voc\u00EAs", "\u00E9"));+ STOP.put("it", Set.of("abbiamo", "adesso", "agli", "alla", "alle", "anche", "ancora", "avete", "che", "ci", "ciao", "col", "come", "con", "da", "dagli", "dal", "dalla", "dallo", "degli", "dei", "del", "della", "delle", "dello", "deve", "devo", "devono", "di", "dove", "e", "ed", "era", "fra", "gi\u00E0", "gli", "grazie", "ha", "hai", "hanno", "ho", "ieri", "il", "la", "le", "lo", "mai", "mi", "mia", "mio", "molto", "ne", "negli", "nel", "nell", "nella", "non", "o", "oggi", "per", "perche", "pi\u00F9", "poco", "quando", "questa", "questo", "scusa", "sempre", "si", "sono", "stata", "stato", "su", "sua", "sul", "sulla", "sulle", "tra", "tuo", "un", "una", "uno", "voglio", "vorrei", "\u00E8"));+ STOP.put("nl", Set.of("aan", "dat", "deze", "door", "een", "het", "is", "maar", "met", "naar", "niet", "ook", "op", "te", "van", "voor", "worden", "wordt", "zijn"));+ STOP.put("sv", Set.of("appen", "aterbetala", "aterbetalning", "att", "behover", "beh\u00F6ver", "blivit", "debiterade", "dessa", "detta", "faktura", "fel", "fr\u00E5n", "f\u00E5r", "ganger", "g\u00E5ng", "g\u00E5nger", "g\u00F6r", "g\u00F6ra", "hittar", "hjalp", "hjalpa", "hj\u00E4lp", "hj\u00E4lpa", "h\u00E4r", "installningarna", "installningen", "inst\u00E4llningarna", "inst\u00E4llningen", "inte", "jag", "kommer", "kraschar", "kvittot", "losenord", "l\u00F6senord", "mig", "min", "mitt", "m\u00E5ste", "nar", "n\u00E4r", "n\u00E4sta", "och", "ocks\u00E5", "om", "oppnar", "paket", "ska", "skicka", "skulle", "sparningen", "sp\u00E5rningen", "s\u00E4ger", "till", "tva", "tv\u00E5", "upp", "uppdaterats", "utan", "vi", "vill", "\u00E4r", "\u00E5terbetala", "\u00E5terbetalning", "\u00F6ppnar"));+ STOP.put("ro", Set.of("aceasta", "aceast\u0103", "acest", "acesta", "acum", "ale", "care", "dar", "din", "dup\u0103", "este", "foarte", "fost", "f\u0103r\u0103", "lui", "mi", "nu", "pentru", "p\u00E2n\u0103", "sunt", "s\u0103", "trebuie", "vreau", "v\u0103", "\u00EEn", "\u0219i", "\u021Bi"));+ STOP.put("bn", Set.of("abar", "ajke", "akhon", "amader", "amake", "amar", "ami", "amra", "apnake", "apnar", "apnara", "apni", "ar", "asbe", "bhai", "bhalo", "bolte", "bolun", "chai", "chaina", "dhonnobad", "dilam", "dite", "diye", "diyechi", "dorkar", "duibar", "ei", "eita", "ekbar", "ekhon", "ekhono", "ekta", "ferot", "geche", "gese", "hobe", "hocche", "hoise", "hoye", "hoyeche", "hoyni", "jabe", "jodi", "jonno", "kalke", "keno", "keu", "kharap", "khub", "ki", "kibhabe", "kichu", "kintu", "kivabe", "kobe", "kokhon", "korbo", "korchi", "kore", "koreche", "korechi", "koren", "korlam", "korsi", "korte", "korun", "kothay", "koto", "lagbe", "moddhe", "nai", "niye", "oi", "oita", "onek", "ota", "pabo", "paini", "parben", "parbo", "parchi", "parchina", "peyechi", "sathe", "shathe", "shob", "shomossa", "somossa", "tader", "tahole", "taka", "theke", "tomake", "tomar", "tomra", "tumi", "valo"));+ STOP.put("az", Set.of("amma", "ancaq", "artiq", "art\u0131q", "bir", "biz", "bu", "cox", "daha", "deyil", "d\u0259", "eger", "g\u00F6r\u0259", "h\u0259m", "h\u0259r", "ile", "il\u0259", "is\u0259", "kimi", "lakin", "m\u0259n", "n\u0259", "olan", "olmasa", "olub", "onlar", "siz", "sonra", "s\u0259n", "ucun", "var", "ve", "v\u0259", "yalniz", "yaln\u0131z", "yox", "yoxdur", "\u00E7ox", "\u00FC\u00E7\u00FCn", "\u0259g\u0259r"));+ STOP_WORDS = Collections.unmodifiableMap(STOP);+ }++ /** Distinctive Swedish words that name the language from a short input. */+ public static final Set<String> SHORT_SWEDISH_WORDS = Set.of(+ "abonnemanget", "aterbetalning", "atkomst", "behover", "beh\u00F6ver", "betalningen",+ "fakturan", "felmeddelande", "glomt", "gl\u00F6mt", "inloggningen", "installningen",+ "inst\u00E4llningen", "kvitto", "kvittot", "losenord", "l\u00F6senord", "sparningen",+ "sp\u00E5rningen", "\u00E5terbetalning", "\u00E5tkomst");++ /** Letters whose presence is evidence the text is not English. */+ public static final Set<String> NON_EN_DIACRITICS = Set.of(+ "\u00DF", "\u00E0", "\u00E1", "\u00E2", "\u00E3", "\u00E4", "\u00E5", "\u00E6", "\u00E7",+ "\u00E8", "\u00E9", "\u00EA", "\u00EB", "\u00EC", "\u00ED", "\u00EE", "\u00EF", "\u00F1",+ "\u00F2", "\u00F3", "\u00F4", "\u00F5", "\u00F6", "\u00F8", "\u00F9", "\u00FA", "\u00FB",+ "\u00FC", "\u00FD", "\u00FF", "\u0101", "\u0103", "\u0105", "\u0107", "\u010D", "\u010F",+ "\u0111", "\u0113", "\u0119", "\u011B", "\u011F", "\u0123", "\u012B", "\u0131", "\u0137",+ "\u013C", "\u0142", "\u0144", "\u0146", "\u0148", "\u0151", "\u0153", "\u0159", "\u015B",+ "\u015F", "\u0161", "\u0163", "\u0165", "\u016B", "\u016F", "\u0171", "\u017A", "\u017C",+ "\u017E", "\u0219", "\u021B", "\u0259");++ /** Function words several lists hold, which therefore name no language. */+ public static final Set<String> SHARED_WORDS = Set.of(+ "as", "como", "con", "da", "das", "del", "dois", "dos", "e", "era", "esta", "este",+ "est\u00E1", "f\u00E5r", "hej", "il", "in", "is", "ja", "jo", "kommer", "la", "le", "les",+ "lo", "mais", "mi", "mig", "min", "nada", "nej", "nos", "o", "om", "para", "por", "porque",+ "quando", "que", "se", "ser", "skulle", "su", "sua", "tack", "tu", "un", "una", "vi",+ "was");++ /** Markers shared by Swedish and Danish: evidence of non-English, not of which. */+ public static final Set<String> NORDIC_OVERLAP_WORDS = Set.of(+ "f\u00E5r", "hej", "ja", "jo", "kommer", "mig", "min", "nej", "om", "skulle", "tack", "vi");++ /** English function words no other list holds. */+ public static final Set<String> EN_ONLY_WORDS = Set.of(+ "and", "are", "at", "be", "but", "can", "for", "from", "has", "have", "i", "it", "not",+ "of", "on", "please", "that", "the", "their", "there", "this", "to", "we", "were", "what",+ "which", "will", "with", "would", "you");++ /** Words that collide with English, counted once however often they repeat. */+ public static final Set<String> EN_COLLISION_WORDS = Set.of(+ "care", "come", "do", "im", "per", "plus", "son", "todo");++ /**+ * A diacritic rate at or above this is evidence the text is not English, even with no stopword+ * match. Measured over every character, so one accented loanword in a short sentence clears it+ * -- which is what the English rescue exists to answer.+ */+ public static final double NON_EN_DIACRITIC_RATE = 0.02;++ /** Above this rate, English function words no longer rescue the text. */+ public static final double ENGLISH_RESCUE_DIACRITIC_RATE = 0.06;++ /** Non-Latin share at which non-Latin text is not for the English checkpoint. */+ public static final double NON_LATIN_FRACTION = 0.2;++ /** The lower share that counts when there are enough letters to dilute it. */+ public static final double NON_LATIN_MIN_FRACTION = 0.1;++ /** How many non-Latin letters make the lower share count. */+ public static final int NON_LATIN_MIN_LETTERS = 10;+}diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.javaindex b9af452c..2da980c0 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/lang/UnicodeTables.java@@ -304,6 +304,237 @@ public final class UnicodeTables {123190, 123566, 123566, 123628, 123631, 124140, 124143, 125136, 125142, 125252, 125258};+ /**+ * Python's {@code \\d}: the Nd category. Needed because Python's {@code \\w} is+ * letters plus every numeric category, and the regexes subtract {@code \\d} from it.+ *+ * <p>64 ranges covering 680 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] DIGIT = {+ 48, 57, 1632, 1641, 1776, 1785, 1984, 1993, 2406, 2415, 2534, 2543, 2662, 2671, 2790,+ 2799, 2918, 2927, 3046, 3055, 3174, 3183, 3302, 3311, 3430, 3439, 3558, 3567, 3664,+ 3673, 3792, 3801, 3872, 3881, 4160, 4169, 4240, 4249, 6112, 6121, 6160, 6169, 6470,+ 6479, 6608, 6617, 6784, 6793, 6800, 6809, 6992, 7001, 7088, 7097, 7232, 7241, 7248,+ 7257, 42528, 42537, 43216, 43225, 43264, 43273, 43472, 43481, 43504, 43513, 43600,+ 43609, 44016, 44025, 65296, 65305, 66720, 66729, 68912, 68921, 69734, 69743, 69872,+ 69881, 69942, 69951, 70096, 70105, 70384, 70393, 70736, 70745, 70864, 70873, 71248,+ 71257, 71360, 71369, 71472, 71481, 71904, 71913, 72016, 72025, 72784, 72793, 73040,+ 73049, 73120, 73129, 73552, 73561, 92768, 92777, 92864, 92873, 93008, 93017, 120782,+ 120831, 123200, 123209, 123632, 123641, 124144, 124153, 125264, 125273, 130032, 130041+ };++ /**+ * Python's {@code str.isupper()} for one character.+ *+ * <p>651 ranges covering 1951 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] UPPER = {+ 65, 90, 192, 214, 216, 222, 256, 256, 258, 258, 260, 260, 262, 262, 264, 264, 266, 266,+ 268, 268, 270, 270, 272, 272, 274, 274, 276, 276, 278, 278, 280, 280, 282, 282, 284,+ 284, 286, 286, 288, 288, 290, 290, 292, 292, 294, 294, 296, 296, 298, 298, 300, 300,+ 302, 302, 304, 304, 306, 306, 308, 308, 310, 310, 313, 313, 315, 315, 317, 317, 319,+ 319, 321, 321, 323, 323, 325, 325, 327, 327, 330, 330, 332, 332, 334, 334, 336, 336,+ 338, 338, 340, 340, 342, 342, 344, 344, 346, 346, 348, 348, 350, 350, 352, 352, 354,+ 354, 356, 356, 358, 358, 360, 360, 362, 362, 364, 364, 366, 366, 368, 368, 370, 370,+ 372, 372, 374, 374, 376, 377, 379, 379, 381, 381, 385, 386, 388, 388, 390, 391, 393,+ 395, 398, 401, 403, 404, 406, 408, 412, 413, 415, 416, 418, 418, 420, 420, 422, 423,+ 425, 425, 428, 428, 430, 431, 433, 435, 437, 437, 439, 440, 444, 444, 452, 452, 455,+ 455, 458, 458, 461, 461, 463, 463, 465, 465, 467, 467, 469, 469, 471, 471, 473, 473,+ 475, 475, 478, 478, 480, 480, 482, 482, 484, 484, 486, 486, 488, 488, 490, 490, 492,+ 492, 494, 494, 497, 497, 500, 500, 502, 504, 506, 506, 508, 508, 510, 510, 512, 512,+ 514, 514, 516, 516, 518, 518, 520, 520, 522, 522, 524, 524, 526, 526, 528, 528, 530,+ 530, 532, 532, 534, 534, 536, 536, 538, 538, 540, 540, 542, 542, 544, 544, 546, 546,+ 548, 548, 550, 550, 552, 552, 554, 554, 556, 556, 558, 558, 560, 560, 562, 562, 570,+ 571, 573, 574, 577, 577, 579, 582, 584, 584, 586, 586, 588, 588, 590, 590, 880, 880,+ 882, 882, 886, 886, 895, 895, 902, 902, 904, 906, 908, 908, 910, 911, 913, 929, 931,+ 939, 975, 975, 978, 980, 984, 984, 986, 986, 988, 988, 990, 990, 992, 992, 994, 994,+ 996, 996, 998, 998, 1000, 1000, 1002, 1002, 1004, 1004, 1006, 1006, 1012, 1012, 1015,+ 1015, 1017, 1018, 1021, 1071, 1120, 1120, 1122, 1122, 1124, 1124, 1126, 1126, 1128,+ 1128, 1130, 1130, 1132, 1132, 1134, 1134, 1136, 1136, 1138, 1138, 1140, 1140, 1142,+ 1142, 1144, 1144, 1146, 1146, 1148, 1148, 1150, 1150, 1152, 1152, 1162, 1162, 1164,+ 1164, 1166, 1166, 1168, 1168, 1170, 1170, 1172, 1172, 1174, 1174, 1176, 1176, 1178,+ 1178, 1180, 1180, 1182, 1182, 1184, 1184, 1186, 1186, 1188, 1188, 1190, 1190, 1192,+ 1192, 1194, 1194, 1196, 1196, 1198, 1198, 1200, 1200, 1202, 1202, 1204, 1204, 1206,+ 1206, 1208, 1208, 1210, 1210, 1212, 1212, 1214, 1214, 1216, 1217, 1219, 1219, 1221,+ 1221, 1223, 1223, 1225, 1225, 1227, 1227, 1229, 1229, 1232, 1232, 1234, 1234, 1236,+ 1236, 1238, 1238, 1240, 1240, 1242, 1242, 1244, 1244, 1246, 1246, 1248, 1248, 1250,+ 1250, 1252, 1252, 1254, 1254, 1256, 1256, 1258, 1258, 1260, 1260, 1262, 1262, 1264,+ 1264, 1266, 1266, 1268, 1268, 1270, 1270, 1272, 1272, 1274, 1274, 1276, 1276, 1278,+ 1278, 1280, 1280, 1282, 1282, 1284, 1284, 1286, 1286, 1288, 1288, 1290, 1290, 1292,+ 1292, 1294, 1294, 1296, 1296, 1298, 1298, 1300, 1300, 1302, 1302, 1304, 1304, 1306,+ 1306, 1308, 1308, 1310, 1310, 1312, 1312, 1314, 1314, 1316, 1316, 1318, 1318, 1320,+ 1320, 1322, 1322, 1324, 1324, 1326, 1326, 1329, 1366, 4256, 4293, 4295, 4295, 4301,+ 4301, 5024, 5109, 7312, 7354, 7357, 7359, 7680, 7680, 7682, 7682, 7684, 7684, 7686,+ 7686, 7688, 7688, 7690, 7690, 7692, 7692, 7694, 7694, 7696, 7696, 7698, 7698, 7700,+ 7700, 7702, 7702, 7704, 7704, 7706, 7706, 7708, 7708, 7710, 7710, 7712, 7712, 7714,+ 7714, 7716, 7716, 7718, 7718, 7720, 7720, 7722, 7722, 7724, 7724, 7726, 7726, 7728,+ 7728, 7730, 7730, 7732, 7732, 7734, 7734, 7736, 7736, 7738, 7738, 7740, 7740, 7742,+ 7742, 7744, 7744, 7746, 7746, 7748, 7748, 7750, 7750, 7752, 7752, 7754, 7754, 7756,+ 7756, 7758, 7758, 7760, 7760, 7762, 7762, 7764, 7764, 7766, 7766, 7768, 7768, 7770,+ 7770, 7772, 7772, 7774, 7774, 7776, 7776, 7778, 7778, 7780, 7780, 7782, 7782, 7784,+ 7784, 7786, 7786, 7788, 7788, 7790, 7790, 7792, 7792, 7794, 7794, 7796, 7796, 7798,+ 7798, 7800, 7800, 7802, 7802, 7804, 7804, 7806, 7806, 7808, 7808, 7810, 7810, 7812,+ 7812, 7814, 7814, 7816, 7816, 7818, 7818, 7820, 7820, 7822, 7822, 7824, 7824, 7826,+ 7826, 7828, 7828, 7838, 7838, 7840, 7840, 7842, 7842, 7844, 7844, 7846, 7846, 7848,+ 7848, 7850, 7850, 7852, 7852, 7854, 7854, 7856, 7856, 7858, 7858, 7860, 7860, 7862,+ 7862, 7864, 7864, 7866, 7866, 7868, 7868, 7870, 7870, 7872, 7872, 7874, 7874, 7876,+ 7876, 7878, 7878, 7880, 7880, 7882, 7882, 7884, 7884, 7886, 7886, 7888, 7888, 7890,+ 7890, 7892, 7892, 7894, 7894, 7896, 7896, 7898, 7898, 7900, 7900, 7902, 7902, 7904,+ 7904, 7906, 7906, 7908, 7908, 7910, 7910, 7912, 7912, 7914, 7914, 7916, 7916, 7918,+ 7918, 7920, 7920, 7922, 7922, 7924, 7924, 7926, 7926, 7928, 7928, 7930, 7930, 7932,+ 7932, 7934, 7934, 7944, 7951, 7960, 7965, 7976, 7983, 7992, 7999, 8008, 8013, 8025,+ 8025, 8027, 8027, 8029, 8029, 8031, 8031, 8040, 8047, 8120, 8123, 8136, 8139, 8152,+ 8155, 8168, 8172, 8184, 8187, 8450, 8450, 8455, 8455, 8459, 8461, 8464, 8466, 8469,+ 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8496, 8499, 8510,+ 8511, 8517, 8517, 8544, 8559, 8579, 8579, 9398, 9423, 11264, 11311, 11360, 11360,+ 11362, 11364, 11367, 11367, 11369, 11369, 11371, 11371, 11373, 11376, 11378, 11378,+ 11381, 11381, 11390, 11392, 11394, 11394, 11396, 11396, 11398, 11398, 11400, 11400,+ 11402, 11402, 11404, 11404, 11406, 11406, 11408, 11408, 11410, 11410, 11412, 11412,+ 11414, 11414, 11416, 11416, 11418, 11418, 11420, 11420, 11422, 11422, 11424, 11424,+ 11426, 11426, 11428, 11428, 11430, 11430, 11432, 11432, 11434, 11434, 11436, 11436,+ 11438, 11438, 11440, 11440, 11442, 11442, 11444, 11444, 11446, 11446, 11448, 11448,+ 11450, 11450, 11452, 11452, 11454, 11454, 11456, 11456, 11458, 11458, 11460, 11460,+ 11462, 11462, 11464, 11464, 11466, 11466, 11468, 11468, 11470, 11470, 11472, 11472,+ 11474, 11474, 11476, 11476, 11478, 11478, 11480, 11480, 11482, 11482, 11484, 11484,+ 11486, 11486, 11488, 11488, 11490, 11490, 11499, 11499, 11501, 11501, 11506, 11506,+ 42560, 42560, 42562, 42562, 42564, 42564, 42566, 42566, 42568, 42568, 42570, 42570,+ 42572, 42572, 42574, 42574, 42576, 42576, 42578, 42578, 42580, 42580, 42582, 42582,+ 42584, 42584, 42586, 42586, 42588, 42588, 42590, 42590, 42592, 42592, 42594, 42594,+ 42596, 42596, 42598, 42598, 42600, 42600, 42602, 42602, 42604, 42604, 42624, 42624,+ 42626, 42626, 42628, 42628, 42630, 42630, 42632, 42632, 42634, 42634, 42636, 42636,+ 42638, 42638, 42640, 42640, 42642, 42642, 42644, 42644, 42646, 42646, 42648, 42648,+ 42650, 42650, 42786, 42786, 42788, 42788, 42790, 42790, 42792, 42792, 42794, 42794,+ 42796, 42796, 42798, 42798, 42802, 42802, 42804, 42804, 42806, 42806, 42808, 42808,+ 42810, 42810, 42812, 42812, 42814, 42814, 42816, 42816, 42818, 42818, 42820, 42820,+ 42822, 42822, 42824, 42824, 42826, 42826, 42828, 42828, 42830, 42830, 42832, 42832,+ 42834, 42834, 42836, 42836, 42838, 42838, 42840, 42840, 42842, 42842, 42844, 42844,+ 42846, 42846, 42848, 42848, 42850, 42850, 42852, 42852, 42854, 42854, 42856, 42856,+ 42858, 42858, 42860, 42860, 42862, 42862, 42873, 42873, 42875, 42875, 42877, 42878,+ 42880, 42880, 42882, 42882, 42884, 42884, 42886, 42886, 42891, 42891, 42893, 42893,+ 42896, 42896, 42898, 42898, 42902, 42902, 42904, 42904, 42906, 42906, 42908, 42908,+ 42910, 42910, 42912, 42912, 42914, 42914, 42916, 42916, 42918, 42918, 42920, 42920,+ 42922, 42926, 42928, 42932, 42934, 42934, 42936, 42936, 42938, 42938, 42940, 42940,+ 42942, 42942, 42944, 42944, 42946, 42946, 42948, 42951, 42953, 42953, 42960, 42960,+ 42966, 42966, 42968, 42968, 42997, 42997, 65313, 65338, 66560, 66599, 66736, 66771,+ 66928, 66938, 66940, 66954, 66956, 66962, 66964, 66965, 68736, 68786, 71840, 71871,+ 93760, 93791, 119808, 119833, 119860, 119885, 119912, 119937, 119964, 119964, 119966,+ 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119989, 120016, 120041,+ 120068, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120120, 120121, 120123,+ 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120172, 120197, 120224, 120249,+ 120276, 120301, 120328, 120353, 120380, 120405, 120432, 120457, 120488, 120512, 120546,+ 120570, 120604, 120628, 120662, 120686, 120720, 120744, 120778, 120778, 125184, 125217,+ 127280, 127305, 127312, 127337, 127344, 127369+ };++ /**+ * Python's {@code str.islower()} for one character.+ *+ * <p>671 ranges covering 2544 code points, recorded+ * from CPython 15.0.0.+ */+ private static final int[] LOWER = {+ 97, 122, 170, 170, 181, 181, 186, 186, 223, 246, 248, 255, 257, 257, 259, 259, 261,+ 261, 263, 263, 265, 265, 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277,+ 279, 279, 281, 281, 283, 283, 285, 285, 287, 287, 289, 289, 291, 291, 293, 293, 295,+ 295, 297, 297, 299, 299, 301, 301, 303, 303, 305, 305, 307, 307, 309, 309, 311, 312,+ 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, 324, 324, 326, 326, 328, 329, 331,+ 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, 343, 343, 345, 345, 347, 347,+ 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361, 361, 363, 363, 365,+ 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 378, 378, 380, 380, 382, 384,+ 387, 387, 389, 389, 392, 392, 396, 397, 402, 402, 405, 405, 409, 411, 414, 414, 417,+ 417, 419, 419, 421, 421, 424, 424, 426, 427, 429, 429, 432, 432, 436, 436, 438, 438,+ 441, 442, 445, 447, 454, 454, 457, 457, 460, 460, 462, 462, 464, 464, 466, 466, 468,+ 468, 470, 470, 472, 472, 474, 474, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485,+ 487, 487, 489, 489, 491, 491, 493, 493, 495, 496, 499, 499, 501, 501, 505, 505, 507,+ 507, 509, 509, 511, 511, 513, 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523,+ 525, 525, 527, 527, 529, 529, 531, 531, 533, 533, 535, 535, 537, 537, 539, 539, 541,+ 541, 543, 543, 545, 545, 547, 547, 549, 549, 551, 551, 553, 553, 555, 555, 557, 557,+ 559, 559, 561, 561, 563, 569, 572, 572, 575, 576, 578, 578, 583, 583, 585, 585, 587,+ 587, 589, 589, 591, 659, 661, 696, 704, 705, 736, 740, 837, 837, 881, 881, 883, 883,+ 887, 887, 890, 893, 912, 912, 940, 974, 976, 977, 981, 983, 985, 985, 987, 987, 989,+ 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005,+ 1005, 1007, 1011, 1013, 1013, 1016, 1016, 1019, 1020, 1072, 1119, 1121, 1121, 1123,+ 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137,+ 1137, 1139, 1139, 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151,+ 1151, 1153, 1153, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173,+ 1173, 1175, 1175, 1177, 1177, 1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187,+ 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, 1197, 1199, 1199, 1201,+ 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215,+ 1215, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230,+ 1231, 1233, 1233, 1235, 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245,+ 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, 1255, 1255, 1257, 1257, 1259,+ 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273,+ 1273, 1275, 1275, 1277, 1277, 1279, 1279, 1281, 1281, 1283, 1283, 1285, 1285, 1287,+ 1287, 1289, 1289, 1291, 1291, 1293, 1293, 1295, 1295, 1297, 1297, 1299, 1299, 1301,+ 1301, 1303, 1303, 1305, 1305, 1307, 1307, 1309, 1309, 1311, 1311, 1313, 1313, 1315,+ 1315, 1317, 1317, 1319, 1319, 1321, 1321, 1323, 1323, 1325, 1325, 1327, 1327, 1376,+ 1416, 4304, 4346, 4348, 4351, 5112, 5117, 7296, 7304, 7424, 7615, 7681, 7681, 7683,+ 7683, 7685, 7685, 7687, 7687, 7689, 7689, 7691, 7691, 7693, 7693, 7695, 7695, 7697,+ 7697, 7699, 7699, 7701, 7701, 7703, 7703, 7705, 7705, 7707, 7707, 7709, 7709, 7711,+ 7711, 7713, 7713, 7715, 7715, 7717, 7717, 7719, 7719, 7721, 7721, 7723, 7723, 7725,+ 7725, 7727, 7727, 7729, 7729, 7731, 7731, 7733, 7733, 7735, 7735, 7737, 7737, 7739,+ 7739, 7741, 7741, 7743, 7743, 7745, 7745, 7747, 7747, 7749, 7749, 7751, 7751, 7753,+ 7753, 7755, 7755, 7757, 7757, 7759, 7759, 7761, 7761, 7763, 7763, 7765, 7765, 7767,+ 7767, 7769, 7769, 7771, 7771, 7773, 7773, 7775, 7775, 7777, 7777, 7779, 7779, 7781,+ 7781, 7783, 7783, 7785, 7785, 7787, 7787, 7789, 7789, 7791, 7791, 7793, 7793, 7795,+ 7795, 7797, 7797, 7799, 7799, 7801, 7801, 7803, 7803, 7805, 7805, 7807, 7807, 7809,+ 7809, 7811, 7811, 7813, 7813, 7815, 7815, 7817, 7817, 7819, 7819, 7821, 7821, 7823,+ 7823, 7825, 7825, 7827, 7827, 7829, 7837, 7839, 7839, 7841, 7841, 7843, 7843, 7845,+ 7845, 7847, 7847, 7849, 7849, 7851, 7851, 7853, 7853, 7855, 7855, 7857, 7857, 7859,+ 7859, 7861, 7861, 7863, 7863, 7865, 7865, 7867, 7867, 7869, 7869, 7871, 7871, 7873,+ 7873, 7875, 7875, 7877, 7877, 7879, 7879, 7881, 7881, 7883, 7883, 7885, 7885, 7887,+ 7887, 7889, 7889, 7891, 7891, 7893, 7893, 7895, 7895, 7897, 7897, 7899, 7899, 7901,+ 7901, 7903, 7903, 7905, 7905, 7907, 7907, 7909, 7909, 7911, 7911, 7913, 7913, 7915,+ 7915, 7917, 7917, 7919, 7919, 7921, 7921, 7923, 7923, 7925, 7925, 7927, 7927, 7929,+ 7929, 7931, 7931, 7933, 7933, 7935, 7943, 7952, 7957, 7968, 7975, 7984, 7991, 8000,+ 8005, 8016, 8023, 8032, 8039, 8048, 8061, 8064, 8071, 8080, 8087, 8096, 8103, 8112,+ 8116, 8118, 8119, 8126, 8126, 8130, 8132, 8134, 8135, 8144, 8147, 8150, 8151, 8160,+ 8167, 8178, 8180, 8182, 8183, 8305, 8305, 8319, 8319, 8336, 8348, 8458, 8458, 8462,+ 8463, 8467, 8467, 8495, 8495, 8500, 8500, 8505, 8505, 8508, 8509, 8518, 8521, 8526,+ 8526, 8560, 8575, 8580, 8580, 9424, 9449, 11312, 11359, 11361, 11361, 11365, 11366,+ 11368, 11368, 11370, 11370, 11372, 11372, 11377, 11377, 11379, 11380, 11382, 11389,+ 11393, 11393, 11395, 11395, 11397, 11397, 11399, 11399, 11401, 11401, 11403, 11403,+ 11405, 11405, 11407, 11407, 11409, 11409, 11411, 11411, 11413, 11413, 11415, 11415,+ 11417, 11417, 11419, 11419, 11421, 11421, 11423, 11423, 11425, 11425, 11427, 11427,+ 11429, 11429, 11431, 11431, 11433, 11433, 11435, 11435, 11437, 11437, 11439, 11439,+ 11441, 11441, 11443, 11443, 11445, 11445, 11447, 11447, 11449, 11449, 11451, 11451,+ 11453, 11453, 11455, 11455, 11457, 11457, 11459, 11459, 11461, 11461, 11463, 11463,+ 11465, 11465, 11467, 11467, 11469, 11469, 11471, 11471, 11473, 11473, 11475, 11475,+ 11477, 11477, 11479, 11479, 11481, 11481, 11483, 11483, 11485, 11485, 11487, 11487,+ 11489, 11489, 11491, 11492, 11500, 11500, 11502, 11502, 11507, 11507, 11520, 11557,+ 11559, 11559, 11565, 11565, 42561, 42561, 42563, 42563, 42565, 42565, 42567, 42567,+ 42569, 42569, 42571, 42571, 42573, 42573, 42575, 42575, 42577, 42577, 42579, 42579,+ 42581, 42581, 42583, 42583, 42585, 42585, 42587, 42587, 42589, 42589, 42591, 42591,+ 42593, 42593, 42595, 42595, 42597, 42597, 42599, 42599, 42601, 42601, 42603, 42603,+ 42605, 42605, 42625, 42625, 42627, 42627, 42629, 42629, 42631, 42631, 42633, 42633,+ 42635, 42635, 42637, 42637, 42639, 42639, 42641, 42641, 42643, 42643, 42645, 42645,+ 42647, 42647, 42649, 42649, 42651, 42653, 42787, 42787, 42789, 42789, 42791, 42791,+ 42793, 42793, 42795, 42795, 42797, 42797, 42799, 42801, 42803, 42803, 42805, 42805,+ 42807, 42807, 42809, 42809, 42811, 42811, 42813, 42813, 42815, 42815, 42817, 42817,+ 42819, 42819, 42821, 42821, 42823, 42823, 42825, 42825, 42827, 42827, 42829, 42829,+ 42831, 42831, 42833, 42833, 42835, 42835, 42837, 42837, 42839, 42839, 42841, 42841,+ 42843, 42843, 42845, 42845, 42847, 42847, 42849, 42849, 42851, 42851, 42853, 42853,+ 42855, 42855, 42857, 42857, 42859, 42859, 42861, 42861, 42863, 42872, 42874, 42874,+ 42876, 42876, 42879, 42879, 42881, 42881, 42883, 42883, 42885, 42885, 42887, 42887,+ 42892, 42892, 42894, 42894, 42897, 42897, 42899, 42901, 42903, 42903, 42905, 42905,+ 42907, 42907, 42909, 42909, 42911, 42911, 42913, 42913, 42915, 42915, 42917, 42917,+ 42919, 42919, 42921, 42921, 42927, 42927, 42933, 42933, 42935, 42935, 42937, 42937,+ 42939, 42939, 42941, 42941, 42943, 42943, 42945, 42945, 42947, 42947, 42952, 42952,+ 42954, 42954, 42961, 42961, 42963, 42963, 42965, 42965, 42967, 42967, 42969, 42969,+ 42994, 42996, 42998, 42998, 43000, 43002, 43824, 43866, 43868, 43881, 43888, 43967,+ 64256, 64262, 64275, 64279, 65345, 65370, 66600, 66639, 66776, 66811, 66967, 66977,+ 66979, 66993, 66995, 67001, 67003, 67004, 67456, 67456, 67459, 67461, 67463, 67504,+ 67506, 67514, 68800, 68850, 71872, 71903, 93792, 93823, 119834, 119859, 119886, 119892,+ 119894, 119911, 119938, 119963, 119990, 119993, 119995, 119995, 119997, 120003, 120005,+ 120015, 120042, 120067, 120094, 120119, 120146, 120171, 120198, 120223, 120250, 120275,+ 120302, 120327, 120354, 120379, 120406, 120431, 120458, 120485, 120514, 120538, 120540,+ 120545, 120572, 120596, 120598, 120603, 120630, 120654, 120656, 120661, 120688, 120712,+ 120714, 120719, 120746, 120770, 120772, 120777, 120779, 120779, 122624, 122633, 122635,+ 122654, 122661, 122666, 122928, 122989, 125218, 125251+ };+/** Python's {@code str.isalpha()} for one code point. */public static boolean isAlpha(int codePoint) {return contains(ALPHA, codePoint);@@ -319,6 +550,86 @@ public final class UnicodeTables {return contains(COMBINING, codePoint);}+ /** Python's {@code \d}: the Nd category. The JDK misses 30 of these at Unicode 13. */+ public static boolean isDigit(int codePoint) {+ return contains(DIGIT, codePoint);+ }++ /**+ * Python's {@code \w} for a str: alphanumeric or underscore.+ *+ * <p>Spelled out because the regexes in {@code laya/lang.py} subtract from it --+ * {@code [^\W\d_]} is this minus digits and underscore, {@code [^\W_]} is this minus+ * underscore -- and Java's own {@code \w} is a different set again (it admits combining+ * marks and join controls, which Python's does not).+ */+ public static boolean isPythonWordChar(int codePoint) {+ return codePoint == '_' || isWordChar(codePoint) || isDigit(codePoint);+ }++ /** Python's {@code [^\W_]}: a word character that is not the underscore. */+ public static boolean isWordOrDigit(int codePoint) {+ return isWordChar(codePoint) || isDigit(codePoint);+ }++ /**+ * Python's {@code str.isupper()} for one code point.+ *+ * <p>A table, because {@code Character.isUpperCase} misses 40 of these and+ * {@code Character.isLowerCase} misses 200 at Unicode 13 -- U+10FC, a Georgian modifier+ * letter Python calls lowercase through Other_Lowercase, among them. Detection reads the case+ * of the first letter of a non-Latin run to tell a proper noun from a request, so a letter+ * whose case the JDK does not know would change that decision.+ */+ public static boolean isUpper(int codePoint) {+ return contains(UPPER, codePoint);+ }++ /** Python's {@code str.islower()} for one code point. */+ public static boolean isLower(int codePoint) {+ return contains(LOWER, codePoint);+ }++ /** Whether a code point has case at all, which is what {@code str.isupper()} requires one of. */+ public static boolean isCased(int codePoint) {+ return isUpper(codePoint) || isLower(codePoint)+ || Character.getType(codePoint) == Character.TITLECASE_LETTER;+ }++ /**+ * Python's {@code str.isupper()} for a whole string: at least one cased code point, and every+ * cased code point uppercase.+ */+ public static boolean isUpperString(String text) {+ boolean sawCased = false;+ int i = 0;+ while (i < text.length()) {+ int codePoint = text.codePointAt(i);+ i += Character.charCount(codePoint);+ if (!isCased(codePoint)) {+ continue;+ }+ if (!isUpper(codePoint)) {+ return false;+ }+ sawCased = true;+ }+ return sawCased;+ }++ /** Whether any code point of {@code text} is lowercase, as Python's {@code islower} sees it. */+ public static boolean hasLower(String text) {+ int i = 0;+ while (i < text.length()) {+ int codePoint = text.codePointAt(i);+ i += Character.charCount(codePoint);+ if (isLower(codePoint)) {+ return true;+ }+ }+ return false;+ }+/*** Python's {@code str.lower()}, which is not Java's.*diff --git a/laya-java/scripts/gen_language_tables.py b/laya-java/scripts/gen_language_tables.pynew file mode 100755index 00000000..8062a0b1--- /dev/null+++ b/laya-java/scripts/gen_language_tables.py@@ -0,0 +1,203 @@+#!/usr/bin/env python3+"""Generate `laya.lang`'s word lists and script ranges as Java source.++Compiled in rather than read from `fixtures/lang_tables.json`, because a runtime library must not+depend on a test fixture being on disk. The fixture keeps its job: `LanguageTablesTest` asserts the+compiled copy equals it, so a table that drifts from Python fails a test rather than changing a+routing decision quietly.++TWO ORDERINGS ARE LOAD-BEARING and are preserved deliberately:++ * `_SCRIPT_RANGES` is scanned in order and the FIRST matching range wins, so a reordering+ changes which script a code point is counted under.+ * `_STOP` is iterated to build the per-language scores, and `max` returns the FIRST of equal+ values, so its insertion order decides ties between languages with the same score.++ python laya-java/scripts/gen_language_tables.py # write+ python laya-java/scripts/gen_language_tables.py --check # fail if the committed copy differs+"""+import argparse+import os+import sys++HERE = os.path.dirname(os.path.abspath(__file__))+REPO = os.path.dirname(os.path.dirname(HERE))+TARGET = os.path.join(+ HERE, "..", "laya-java", "src", "main", "java", "com", "convaiinnovations", "laya", "lang",+ "LanguageTables.java")+++def quote(text):+ """A Java string literal, with every non-ASCII character escaped so the file stays ASCII."""+ out = ['"']+ for ch in text:+ if ch == '"':+ out.append('\\"')+ elif ch == "\\":+ out.append("\\\\")+ elif " " <= ch <= "~":+ out.append(ch)+ else:+ encoded = ch.encode("utf-16-be")+ for i in range(0, len(encoded), 2):+ out.append("\\u%02X%02X" % (encoded[i], encoded[i + 1]))+ out.append('"')+ return "".join(out)+++def words_block(name, words, doc, sort=True):+ items = sorted(words) if sort else list(words)+ lines, current = [], " "+ for word in items:+ piece = quote(word) + ", "+ if len(current) + len(piece) > 104:+ lines.append(current.rstrip())+ current = " "+ current += piece+ if current.strip():+ lines.append(current.rstrip().rstrip(","))+ return (' /** %s */\n public static final Set<String> %s = Set.of(\n%s);\n'+ % (doc, name, "\n".join(lines)))+++def render():+ sys.path.insert(0, REPO)+ from laya import lang++ script_rows = []+ for name, ranges in lang._SCRIPT_RANGES:+ pairs = ", ".join("%d, %d" % (lo, hi) for lo, hi in ranges)+ script_rows.append(' new Script(%s, new int[] {%s})' % (quote(name), pairs))++ stop_rows = []+ for code, words in lang._STOP.items():+ items = ", ".join(quote(w) for w in sorted(words))+ stop_rows.append(' STOP.put(%s, Set.of(%s));' % (quote(code), items))++ header = '''// GENERATED by laya-java/scripts/gen_language_tables.py -- do not edit.+//+// Regenerate with:+// python laya-java/scripts/gen_language_tables.py+// and `--check` fails when the committed copy no longer matches laya/lang.py.+package com.convaiinnovations.laya.lang;++import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;++/**+ * The word lists, script ranges and thresholds {@code laya/lang.py} detects with.+ *+ * <p>Compiled in rather than read from {@code fixtures/lang_tables.json}: a runtime library must+ * not need a test fixture on disk. The fixture keeps its job -- a test asserts this copy equals it,+ * so a table that drifts from Python fails a test rather than changing a routing decision quietly.+ *+ * <p><b>Two orderings are load-bearing.</b> {@link #SCRIPT_RANGES} is scanned in order and the+ * first matching range wins, so a reordering changes which script a code point counts under. And+ * {@link #STOP_WORDS} is iterated to score languages, where the winner is the first of equal+ * scores, so its order decides ties.+ */+public final class LanguageTables {++ private LanguageTables() {+ }++ /** One named non-Latin script and the code-point ranges that belong to it. */+ public record Script(String name, int[] ranges) {+ }++'''+ body = (+ ' /**\n'+ ' * Named scripts, in scan order. The first range that contains a code point wins, so\n'+ ' * this is a sequence and not a set.\n'+ ' */\n'+ ' public static final List<Script> SCRIPT_RANGES = List.of(\n'+ + ",\n".join(script_rows) + ");\n\n"+ ' /**\n'+ ' * Function words per language, in the order Python declares them.\n'+ ' *\n'+ ' * <p>A {@link LinkedHashMap}, because the language that wins a tied score is the\n'+ ' * first one iterated.\n'+ ' */\n'+ ' public static final Map<String, Set<String>> STOP_WORDS;\n\n'+ ' static {\n'+ ' Map<String, Set<String>> STOP = new LinkedHashMap<>();\n'+ + "\n".join(stop_rows) + "\n"+ ' STOP_WORDS = Collections.unmodifiableMap(STOP);\n'+ ' }\n\n'+ + words_block("SHORT_SWEDISH_WORDS", lang._SHORT_SWEDISH_WORDS,+ "Distinctive Swedish words that name the language from a short input.")+ + "\n"+ + words_block("NON_EN_DIACRITICS", lang._NON_EN_DIACRITICS,+ "Letters whose presence is evidence the text is not English.")+ + "\n"+ + words_block("SHARED_WORDS", lang._SHARED_WORDS,+ "Function words several lists hold, which therefore name no language.")+ + "\n"+ + words_block("NORDIC_OVERLAP_WORDS", lang._NORDIC_OVERLAP_WORDS,+ "Markers shared by Swedish and Danish: evidence of non-English, not of which.")+ + "\n"+ + words_block("EN_ONLY_WORDS", lang._EN_ONLY_WORDS,+ "English function words no other list holds.")+ + "\n"+ + words_block("EN_COLLISION_WORDS", lang._EN_COLLISION_WORDS,+ "Words that collide with English, counted once however often they repeat.")+ + '''+ /**+ * A diacritic rate at or above this is evidence the text is not English, even with no stopword+ * match. Measured over every character, so one accented loanword in a short sentence clears it+ * -- which is what the English rescue exists to answer.+ */+ public static final double NON_EN_DIACRITIC_RATE = %r;++ /** Above this rate, English function words no longer rescue the text. */+ public static final double ENGLISH_RESCUE_DIACRITIC_RATE = %r;++ /** Non-Latin share at which non-Latin text is not for the English checkpoint. */+ public static final double NON_LATIN_FRACTION = %r;++ /** The lower share that counts when there are enough letters to dilute it. */+ public static final double NON_LATIN_MIN_FRACTION = %r;++ /** How many non-Latin letters make the lower share count. */+ public static final int NON_LATIN_MIN_LETTERS = %d;+}+''' % (lang.NON_EN_DIACRITIC_RATE, lang.ENGLISH_RESCUE_DIACRITIC_RATE,+ lang.NON_LATIN_FRACTION, lang.NON_LATIN_MIN_FRACTION, lang.NON_LATIN_MIN_LETTERS)+ )+ return header + body+++def main(argv=None):+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--check", action="store_true",+ help="do not write; exit 1 if the committed copy would change")+ args = parser.parse_args(argv)+ fresh = render()+ target = os.path.normpath(TARGET)+ if args.check:+ try:+ with open(target, encoding="utf-8") as handle:+ current = handle.read()+ except OSError:+ print("gen_language_tables: %s is missing" % target, file=sys.stderr)+ return 1+ if current != fresh:+ print("gen_language_tables: %s is stale; run "+ "laya-java/scripts/gen_language_tables.py and commit the result" % target,+ file=sys.stderr)+ return 1+ print("gen_language_tables: LanguageTables.java matches laya/lang.py")+ return 0+ os.makedirs(os.path.dirname(target), exist_ok=True)+ with open(target, "w", encoding="utf-8") as handle:+ handle.write(fresh)+ print("wrote %s (%d bytes)" % (os.path.relpath(target, REPO), len(fresh)))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/laya-java/scripts/gen_unicode_tables.py b/laya-java/scripts/gen_unicode_tables.pyindex f0c892cc..3e8090dd 100755--- a/laya-java/scripts/gen_unicode_tables.py+++ b/laya-java/scripts/gen_unicode_tables.py@@ -35,6 +35,7 @@ TARGET = os.path.join("UnicodeTables.java")WORD = re.compile(r"[^\W\d_]", re.UNICODE)+DIGIT = re.compile(r"\d", re.UNICODE)MAX = 0x110000SURROGATES = range(0xD800, 0xE000)@@ -78,6 +79,9 @@ def render():alpha = ranges(lambda cp: chr(cp).isalpha())word = ranges(lambda cp: WORD.fullmatch(chr(cp)) is not None)combining = ranges(lambda cp: unicodedata.combining(chr(cp)) != 0)+ digit = ranges(lambda cp: DIGIT.fullmatch(chr(cp)) is not None)+ upper = ranges(lambda cp: chr(cp).isupper())+ lower = ranges(lambda cp: chr(cp).islower())# Every code point whose lowercase is more than one code point. In Unicode 15 there is# exactly one, and `laya/lang.py` depends on it: it replaces U+0130 before lowering, because@@ -132,6 +136,14 @@ public final class UnicodeTables {+ "\n"+ table("COMBINING", combining,"Code points with a non-zero canonical combining class.")+ + "\n"+ + table("DIGIT", digit,+ "Python's {@code \\\\d}: the Nd category. Needed because Python's {@code \\\\w} is\n"+ " * letters plus every numeric category, and the regexes subtract {@code \\\\d} from it.")+ + "\n"+ + table("UPPER", upper, "Python's {@code str.isupper()} for one character.")+ + "\n"+ + table("LOWER", lower, "Python's {@code str.islower()} for one character."))multi_cases = "\n".join(@@ -153,6 +165,86 @@ public final class UnicodeTables {return contains(COMBINING, codePoint);}+ /** Python's {@code \\d}: the Nd category. The JDK misses 30 of these at Unicode 13. */+ public static boolean isDigit(int codePoint) {+ return contains(DIGIT, codePoint);+ }++ /**+ * Python's {@code \\w} for a str: alphanumeric or underscore.+ *+ * <p>Spelled out because the regexes in {@code laya/lang.py} subtract from it --+ * {@code [^\\W\\d_]} is this minus digits and underscore, {@code [^\\W_]} is this minus+ * underscore -- and Java's own {@code \\w} is a different set again (it admits combining+ * marks and join controls, which Python's does not).+ */+ public static boolean isPythonWordChar(int codePoint) {+ return codePoint == '_' || isWordChar(codePoint) || isDigit(codePoint);+ }++ /** Python's {@code [^\\W_]}: a word character that is not the underscore. */+ public static boolean isWordOrDigit(int codePoint) {+ return isWordChar(codePoint) || isDigit(codePoint);+ }++ /**+ * Python's {@code str.isupper()} for one code point.+ *+ * <p>A table, because {@code Character.isUpperCase} misses 40 of these and+ * {@code Character.isLowerCase} misses 200 at Unicode 13 -- U+10FC, a Georgian modifier+ * letter Python calls lowercase through Other_Lowercase, among them. Detection reads the case+ * of the first letter of a non-Latin run to tell a proper noun from a request, so a letter+ * whose case the JDK does not know would change that decision.+ */+ public static boolean isUpper(int codePoint) {+ return contains(UPPER, codePoint);+ }++ /** Python's {@code str.islower()} for one code point. */+ public static boolean isLower(int codePoint) {+ return contains(LOWER, codePoint);+ }++ /** Whether a code point has case at all, which is what {@code str.isupper()} requires one of. */+ public static boolean isCased(int codePoint) {+ return isUpper(codePoint) || isLower(codePoint)+ || Character.getType(codePoint) == Character.TITLECASE_LETTER;+ }++ /**+ * Python's {@code str.isupper()} for a whole string: at least one cased code point, and every+ * cased code point uppercase.+ */+ public static boolean isUpperString(String text) {+ boolean sawCased = false;+ int i = 0;+ while (i < text.length()) {+ int codePoint = text.codePointAt(i);+ i += Character.charCount(codePoint);+ if (!isCased(codePoint)) {+ continue;+ }+ if (!isUpper(codePoint)) {+ return false;+ }+ sawCased = true;+ }+ return sawCased;+ }++ /** Whether any code point of {@code text} is lowercase, as Python's {@code islower} sees it. */+ public static boolean hasLower(String text) {+ int i = 0;+ while (i < text.length()) {+ int codePoint = text.codePointAt(i);+ i += Character.charCount(codePoint);+ if (isLower(codePoint)) {+ return true;+ }+ }+ return false;+ }+/*** Python's {@code str.lower()}, which is not Java's.*