Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(dotnet): match Python on astral digits, numeric types and tuple states
kevin-gatimuae723fc6243f
diff --git a/laya-dotnet/CHANGELOG.md b/laya-dotnet/CHANGELOG.mdindex c29ad72c..f16d18cd 100644--- a/laya-dotnet/CHANGELOG.md+++ b/laya-dotnet/CHANGELOG.md@@ -65,6 +65,17 @@ goldens: 0.4.12).booleans are accepted. See "Differences from Python" in the README. `str.lower()` is exact (final sigma,U+0130), only lists and tuples are walked for text (a set holds none and is not JSON serializable), and any`IReadOnlyDictionary` is walked as a mapping.+- **Behaviour change for callers passing lazy sequences as a state:** only an `IList` (arrays, `List<T>`) or a tuple is+ serialized as a JSON array and walked for language detection; a set, `Queue<T>`, LINQ iterator or any other+ non-list enumerable now throws `JsonException` when serialized (and holds no text for detection), as Python's+ `json.dumps` rejects a set or generator. Pass `ToList()` / `ToArray()`. A tuple is no longer left-truncated when+ over budget (Python truncates from the left only for a `list`).+- `Calibration.ClampTemperature(object?)` and `TemperatureRejected` accept every boxed numeric type (`int`, `short`,+ `byte`, `sbyte`, `ushort`, `uint`, `ulong`, `float`, `decimal`, `Half`, `nint`, `BigInteger`, `Int128`), converted as+ Python's `float()` rounds.+- Email patterns see astral decimal digits (Unicode Nd, such as the mathematical double-struck digits) as digits, so a+ reply header whose date is written with them is recognised as in Python, while astral numbers that are not decimal+ digits stay word characters only.- Case-insensitive email matching also treats the long s with `s` and the Kelvin sign with `k`, as `re.I` does.- Email patterns count and bound code points, not UTF-16 units, so characters outside the BMP next to akeyword or inside a `.{0,300}` window behave as in Python.diff --git a/laya-dotnet/README.md b/laya-dotnet/README.mdindex 8c987195..d0b91312 100644--- a/laya-dotnet/README.md+++ b/laya-dotnet/README.md@@ -255,7 +255,8 @@ its position. `Add` throws instead.|---|---|| `string` | used as written || `IDictionary`, `IEnumerable<KeyValuePair<string, object?>>` | a JSON object, in the dictionary's enumeration order |-| any other `IEnumerable` (arrays, lists, conversation turns) | a JSON array |+| `IList` (arrays, `List<T>`, conversation turns) and tuples (`ValueTuple`, `Tuple`) | a JSON array; only an `IList` is left-truncated when over budget, as Python truncates a `list` (a tuple keeps its start) |+| a set, a `Queue<T>`, a LINQ iterator or any other non-list enumerable | **not accepted**: serializing it throws `JsonException`, as Python's `json.dumps` rejects a set or a generator. Pass `ToList()` / `ToArray()` || `null`, `bool`, integer types, `BigInteger`, `double`, `float`, `decimal` | the JSON scalar |Serialization matches Python's `json.dumps` byte for byte (`", "` / `": "` separators, floatdiff --git a/laya-dotnet/src/Laya/LayaEmail.cs b/laya-dotnet/src/Laya/LayaEmail.csindex e7eba4b3..236dd21d 100644--- a/laya-dotnet/src/Laya/LayaEmail.cs+++ b/laya-dotnet/src/Laya/LayaEmail.cs@@ -97,14 +97,17 @@ public static partial class LayaEmail// Python's `re` works on code points; .NET's on UTF-16 units, where a character outside the BMP is// two units that `.`, `{0,n}` counts and `\b` would treat as two non-word characters. Every// pattern here is only asked "does it match", so each such character is first replaced by one BMP- // stand-in of the same word class (a letter or number by U+4E00, anything else by U+2603), which- // no pattern mentions.+ // stand-in of the same classes: a decimal digit (Nd) by U+0663, which `\d` and `\w` accept and the explicit+ // `[0-9]` classes do not, as for Python's Unicode `\d`; any other letter or number (`\w` but not `\d`) by U+4E00;+ // anything else by U+2603. No pattern mentions any of them.private static string CodePoints(string s){if (!s.Any(char.IsSurrogate)) return s;var sb = new StringBuilder(s.Length);foreach (var rune in s.EnumerateRunes())- sb.Append(rune.IsBmp ? rune.ToString() : Rune.IsLetter(rune) || Rune.IsNumber(rune) ? "一" : "☃");+ sb.Append(rune.IsBmp ? rune.ToString()+ : Rune.GetUnicodeCategory(rune) == System.Globalization.UnicodeCategory.DecimalDigitNumber ? "\u0663"+ : Rune.IsLetter(rune) || Rune.IsNumber(rune) ? "\u4e00" : "\u2603");return sb.ToString();}diff --git a/laya-dotnet/src/Laya/PythonFloat.cs b/laya-dotnet/src/Laya/PythonFloat.csindex 631a064a..6766fbcd 100644--- a/laya-dotnet/src/Laya/PythonFloat.cs+++ b/laya-dotnet/src/Laya/PythonFloat.cs@@ -64,9 +64,19 @@ internal static partial class PythonFloat{case bool b: value = b ? 1.0 : 0.0; return true;case double d: value = d; return true;- case long l: value = l; return true;- case BigInteger big: value = (double)big; return true;case string s: return TryParse(s, out value);+ // Any number float() accepts. An integer rounds to nearest, ties to even, as Python's int -> float does;+ // double.Parse reads the exact decimal text and rounds the same way, which a cast from decimal or from+ // the 64-bit unsigned types does not promise.+ case nint n: value = n; return true;+ case long or int or short or sbyte or byte or ushort or uint:+ value = Convert.ToInt64(raw, CultureInfo.InvariantCulture);+ return true;+ case ulong or nuint or BigInteger or decimal or Int128 or UInt128:+ value = double.Parse(Convert.ToString(raw, CultureInfo.InvariantCulture)!, NumberStyles.Float, CultureInfo.InvariantCulture);+ return true;+ case float f: value = f; return true;+ case Half h: value = (double)h; return true;default: value = 0; return false;}}diff --git a/laya-dotnet/src/Laya/PythonJson.cs b/laya-dotnet/src/Laya/PythonJson.csindex e907b5d2..be606127 100644--- a/laya-dotnet/src/Laya/PythonJson.cs+++ b/laya-dotnet/src/Laya/PythonJson.cs@@ -124,11 +124,11 @@ public static class PythonJson/// <summary>/// Whether <paramref name="value"/> is the .NET analogue of Python's <c>isinstance(state,- /// list)</c>: any non-string, non-dict-like enumerable. There is no exact "list vs. tuple"- /// distinction on this side, so anything <see cref="Dumps"/> would render as a JSON array- /// counts, matching how <c>serialize_state</c> already treats the value.+ /// list)</c>, which decides left truncation: an array or an <see cref="IList"/> that is not+ /// dictionary-like. A tuple (<c>ITuple</c>) is still rendered as a JSON array but is not a list,+ /// so it is truncated from the right, as Python truncates a tuple./// </summary>- public static bool IsListState(object? value) => IsListLike(value);+ public static bool IsListState(object? value) => value is IList && !IsDictLike(value);private static void WriteValue(StringBuilder sb, object? value,diff --git a/laya-dotnet/tests/Laya.Tests/ReviewParityTests.cs b/laya-dotnet/tests/Laya.Tests/ReviewParityTests.csnew file mode 100644index 00000000..9d31dcc0--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/ReviewParityTests.cs@@ -0,0 +1,117 @@+using System.Numerics;++namespace Laya.Tests;++/// <summary>+/// Parity checks added after review: boxed numeric types in <c>clamp_temperature</c> / <c>temperature_rejected</c>,+/// and left truncation of a state only for a list. Expectations come from running the Python reference+/// (<c>float(x)</c> for each numeric type, <c>build_sequence(..., truncate_left=isinstance(state, list))</c>).+/// </summary>+public sealed class ReviewParityTests+{+ public static TheoryData<string, string, string> Numbers() => new()+ {+ { "byte", "0", "0.0" },+ { "byte", "255", "255.0" },+ { "sbyte", "-128", "-128.0" },+ { "sbyte", "127", "127.0" },+ { "short", "-32768", "-32768.0" },+ { "short", "32767", "32767.0" },+ { "ushort", "0", "0.0" },+ { "ushort", "65535", "65535.0" },+ { "int", "-2147483648", "-2147483648.0" },+ { "int", "2147483647", "2147483647.0" },+ { "int", "16777217", "16777217.0" },+ { "uint", "4294967295", "4294967295.0" },+ { "long", "9223372036854775807", "9.223372036854776e+18" },+ { "long", "-9223372036854775808", "-9.223372036854776e+18" },+ { "long", "9007199254740993", "9007199254740992.0" },+ { "long", "9007199254740995", "9007199254740996.0" },+ { "ulong", "18446744073709551615", "1.8446744073709552e+19" },+ { "ulong", "9007199254740993", "9007199254740992.0" },+ { "ulong", "9007199254740995", "9007199254740996.0" },+ { "ulong", "18446744073709550592", "1.8446744073709552e+19" },+ { "BigInteger", "9007199254740993", "9007199254740992.0" },+ { "BigInteger", "123456789012345678901234567890", "1.2345678901234568e+29" },+ { "BigInteger", "-9007199254740993", "-9007199254740992.0" },+ { "Int128", "170141183460469231731687303715884105727", "1.7014118346046923e+38" },+ { "Int128", "9007199254740993", "9007199254740992.0" },+ { "decimal", "0.1", "0.1" },+ { "decimal", "1.0000000000000001", "1.0" },+ { "decimal", "79228162514264337593543950335", "7.922816251426434e+28" },+ { "decimal", "-0.5", "-0.5" },+ { "decimal", "0.30000000000000004441", "0.30000000000000004" },+ { "decimal", "2.5", "2.5" },+ { "decimal", "4.35", "4.35" },+ { "decimal", "0.0000000000000000000000000001", "1e-28" },+ { "float", "0.1", "0.10000000149011612" },+ { "float", "2.5", "2.5" },+ { "float", "3.4028235e38", "3.4028234663852886e+38" },+ { "float", "1.17549435e-38", "1.1754943508222875e-38" },+ { "float", "-0.7", "-0.699999988079071" },+ { "Half", "0.1", "0.0999755859375" },+ { "Half", "2.5", "2.5" },+ { "Half", "65504", "65504.0" },+ { "Half", "-0.7", "-0.7001953125" },+ };++ private static object Box(string kind, string text) => kind switch+ {+ "byte" => byte.Parse(text), "sbyte" => sbyte.Parse(text), "short" => short.Parse(text), "ushort" => ushort.Parse(text),+ "int" => int.Parse(text), "uint" => uint.Parse(text), "long" => long.Parse(text), "ulong" => ulong.Parse(text),+ "BigInteger" => BigInteger.Parse(text), "Int128" => Int128.Parse(text), "decimal" => decimal.Parse(text, System.Globalization.CultureInfo.InvariantCulture),+ "float" => float.Parse(text, System.Globalization.CultureInfo.InvariantCulture),+ "Half" => Half.Parse(text, System.Globalization.CultureInfo.InvariantCulture),+ _ => throw new ArgumentException(kind),+ };++ [Theory]+ [MemberData(nameof(Numbers))]+ public void EveryBoxedNumberConvertsAsPythonsFloatDoes(string kind, string text, string repr)+ {+ var boxed = Box(kind, text);+ Assert.True(PythonFloat.TryConvert(boxed, out var value));+ Assert.Equal(repr, PythonFloat.Repr(value));+ Assert.Equal(Calibration.ClampTemperature(value), Calibration.ClampTemperature(boxed));+ Assert.Equal(value != Calibration.ClampTemperature(value), Calibration.TemperatureRejected(boxed, Calibration.ClampTemperature(value)));+ }++ private sealed class DigitsTokenizer : Laya.Tokenization.ILayaTokenizer+ {+ public int PadId => 0;+ public int SepId => 1;+ public int ClsId => 2;+ public int UnkId => 3;+ public int MaskId => 4;+ public string MaskToken => "[MASK]";+ public int[] Encode(string text) =>+ [.. System.Text.RegularExpressions.Regex.Matches(text, "[0-9]+").Select(m => int.Parse(m.Value))];+ }++ // The room for the state is four tokens, so eight numbers are over budget.+ private static int[] Built(object state)+ {+ var question = Question.Choice("pick", [new KeyValuePair<string, object?>("a", "x"), new KeyValuePair<string, object?>("b", "y")]);+ return SequenceBuilder.Build(new DigitsTokenizer(), state, question, 10, 10, PythonJson.IsListState(state)).Ids;+ }++ [Fact]+ public void AListOrArrayKeepsTheEndOfAnOverBudgetState()+ {+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1004, 1005, 1006, 1007, 1 }, Built(new List<int> { 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007 }));+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1004, 1005, 1006, 1007, 1 }, Built(new[] { 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007 }));+ }++ [Fact]+ public void ATupleKeepsTheStartOfAnOverBudgetStateAsPythonDoes() =>+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1000, 1001, 1002, 1003, 1 }, Built((1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007)));++ [Fact]+ public void ATupleIsNotAListState()+ {+ Assert.False(PythonJson.IsListState(("a", "b")));+ Assert.True(PythonJson.IsListState(new[] { "a" }));+ Assert.True(PythonJson.IsListState(new List<object?> { "a" }));+ Assert.False(PythonJson.IsListState(new Dictionary<string, object?>()));+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json b/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.jsonindex a257491d..594425cc 100644--- a/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json+++ b/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json@@ -1475,6 +1475,48 @@"body": "Please help me.\nSent from my iPhone using KIT","max_chars": 3000,"result": "Please help me."+ },+ {+ "label": "pyre/astral_digit_quote_header",+ "body": "Please help me.\nEm 𝟚𝟘𝟚𝟜 João escreveu:\n> old request\nStill mine.",+ "max_chars": 3000,+ "result": "Please help me."+ },+ {+ "label": "pyre/astral_digit_french_header",+ "body": "Please help me.\nLe 𝟚 janvier a écrit :\n> old request",+ "max_chars": 3000,+ "result": "Please help me."+ },+ {+ "label": "pyre/astral_digit_header_pair",+ "body": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚𝟜\nOld request.",+ "max_chars": 3000,+ "result": "Please help me."+ },+ {+ "label": "pyre/astral_digit_in_year_needs_four",+ "body": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚\nOld request.",+ "max_chars": 3000,+ "result": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚\nOld request."+ },+ {+ "label": "pyre/astral_number_not_digit_header",+ "body": "Please help me.\nEm 𐄇 João escreveu:\n> old request\nStill mine.",+ "max_chars": 3000,+ "result": "Please help me.\nEm 𐄇 João escreveu:\nStill mine."+ },+ {+ "label": "pyre/astral_letter_not_digit_header",+ "body": "Please help me.\nEm 𝐀 João escreveu:\n> old request\nStill mine.",+ "max_chars": 3000,+ "result": "Please help me.\nEm 𝐀 João escreveu:\nStill mine."+ },+ {+ "label": "pyre/astral_digit_device_footer",+ "body": "Please help me.\nSent from my iPhone 𝟚",+ "max_chars": 3000,+ "result": "Please help me."}],"state": [diff --git a/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json b/laya-dotnet/tests/Laya.Tests/golden/routing/meta.jsonindex d807efc3..12a79316 100644--- a/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json+++ b/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json@@ -9,7 +9,7 @@"counts": {"lang_probe": 246,"route_probe": 142,- "email_probe_clean": 246,+ "email_probe_clean": 253,"email_probe_state": 22,"shortlist_probe": 22,"hashing_embedder_probe": 22diff --git a/laya-dotnet/tools/routing_cases.py b/laya-dotnet/tools/routing_cases.pyindex bd6cda3b..651efe80 100644--- a/laya-dotnet/tools/routing_cases.py+++ b/laya-dotnet/tools/routing_cases.py@@ -928,6 +928,21 @@ EMAIL_CLEAN_CASES = [("pyre/ignorecase_long_s_pt_signoff", "Preciso de ajuda.\nſaludoſ cordiales,\nMaria", 3000),("pyre/ignorecase_kelvin_sign_signoff", "Please help me.\nThanK you,\nJohn", 3000),("pyre/ignorecase_kelvin_sign_device_footer", "Please help me.\nSent from my iPhone using KIT", 3000),+ # astral decimal digits: Python's Unicode `\d` (Nd) matches them; a number that is not a decimal digit (No) is `\w` only+ ("pyre/astral_digit_quote_header",+ "Please help me.\nEm \U0001d7da\U0001d7d8\U0001d7da\U0001d7dc João escreveu:\n> old request\nStill mine.", 3000),+ ("pyre/astral_digit_french_header",+ "Please help me.\nLe \U0001d7da janvier a écrit :\n> old request", 3000),+ ("pyre/astral_digit_header_pair",+ "Please help me.\nFrom: Maria Souza\nDate: \U0001d7da\U0001d7d8\U0001d7da\U0001d7dc\nOld request.", 3000),+ ("pyre/astral_digit_in_year_needs_four",+ "Please help me.\nFrom: Maria Souza\nDate: \U0001d7da\U0001d7d8\U0001d7da\nOld request.", 3000),+ ("pyre/astral_number_not_digit_header",+ "Please help me.\nEm \U00010107 João escreveu:\n> old request\nStill mine.", 3000),+ ("pyre/astral_letter_not_digit_header",+ "Please help me.\nEm \U0001d400 João escreveu:\n> old request\nStill mine.", 3000),+ ("pyre/astral_digit_device_footer",+ "Please help me.\nSent from my iPhone \U0001d7da", 3000),]EMAIL_STATE_CASES = [