Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(dotnet): match Python on astral digits, numeric types and tuple states

kevin-gatimuae723fc6243f
diff --git a/laya-dotnet/CHANGELOG.md b/laya-dotnet/CHANGELOG.md
index c29ad72c..f16d18cd 100644
--- a/laya-dotnet/CHANGELOG.md
+++ b/laya-dotnet/CHANGELOG.md
@@ -65,6 +65,17 @@ goldens: 0.4.12).
booleans are accepted. See "Differences from Python" in the README. `str.lower()` is exact (final sigma,
U+0130), only lists and tuples are walked for text (a set holds none and is not JSON serializable), and any
`IReadOnlyDictionary` is walked as a mapping.
+- **Behaviour change for callers passing lazy sequences as a state:** only an `IList` (arrays, `List<T>`) or a tuple is
+ serialized as a JSON array and walked for language detection; a set, `Queue<T>`, LINQ iterator or any other
+ non-list enumerable now throws `JsonException` when serialized (and holds no text for detection), as Python's
+ `json.dumps` rejects a set or generator. Pass `ToList()` / `ToArray()`. A tuple is no longer left-truncated when
+ over budget (Python truncates from the left only for a `list`).
+- `Calibration.ClampTemperature(object?)` and `TemperatureRejected` accept every boxed numeric type (`int`, `short`,
+ `byte`, `sbyte`, `ushort`, `uint`, `ulong`, `float`, `decimal`, `Half`, `nint`, `BigInteger`, `Int128`), converted as
+ Python's `float()` rounds.
+- Email patterns see astral decimal digits (Unicode Nd, such as the mathematical double-struck digits) as digits, so a
+ reply header whose date is written with them is recognised as in Python, while astral numbers that are not decimal
+ digits stay word characters only.
- Case-insensitive email matching also treats the long s with `s` and the Kelvin sign with `k`, as `re.I` does.
- Email patterns count and bound code points, not UTF-16 units, so characters outside the BMP next to a
keyword or inside a `.{0,300}` window behave as in Python.
diff --git a/laya-dotnet/README.md b/laya-dotnet/README.md
index 8c987195..d0b91312 100644
--- a/laya-dotnet/README.md
+++ b/laya-dotnet/README.md
@@ -255,7 +255,8 @@ its position. `Add` throws instead.
|---|---|
| `string` | used as written |
| `IDictionary`, `IEnumerable<KeyValuePair<string, object?>>` | a JSON object, in the dictionary's enumeration order |
-| any other `IEnumerable` (arrays, lists, conversation turns) | a JSON array |
+| `IList` (arrays, `List<T>`, conversation turns) and tuples (`ValueTuple`, `Tuple`) | a JSON array; only an `IList` is left-truncated when over budget, as Python truncates a `list` (a tuple keeps its start) |
+| a set, a `Queue<T>`, a LINQ iterator or any other non-list enumerable | **not accepted**: serializing it throws `JsonException`, as Python's `json.dumps` rejects a set or a generator. Pass `ToList()` / `ToArray()` |
| `null`, `bool`, integer types, `BigInteger`, `double`, `float`, `decimal` | the JSON scalar |
Serialization matches Python's `json.dumps` byte for byte (`", "` / `": "` separators, float
diff --git a/laya-dotnet/src/Laya/LayaEmail.cs b/laya-dotnet/src/Laya/LayaEmail.cs
index e7eba4b3..236dd21d 100644
--- a/laya-dotnet/src/Laya/LayaEmail.cs
+++ b/laya-dotnet/src/Laya/LayaEmail.cs
@@ -97,14 +97,17 @@ public static partial class LayaEmail
// Python's `re` works on code points; .NET's on UTF-16 units, where a character outside the BMP is
// two units that `.`, `{0,n}` counts and `\b` would treat as two non-word characters. Every
// pattern here is only asked "does it match", so each such character is first replaced by one BMP
- // stand-in of the same word class (a letter or number by U+4E00, anything else by U+2603), which
- // no pattern mentions.
+ // stand-in of the same classes: a decimal digit (Nd) by U+0663, which `\d` and `\w` accept and the explicit
+ // `[0-9]` classes do not, as for Python's Unicode `\d`; any other letter or number (`\w` but not `\d`) by U+4E00;
+ // anything else by U+2603. No pattern mentions any of them.
private static string CodePoints(string s)
{
if (!s.Any(char.IsSurrogate)) return s;
var sb = new StringBuilder(s.Length);
foreach (var rune in s.EnumerateRunes())
- sb.Append(rune.IsBmp ? rune.ToString() : Rune.IsLetter(rune) || Rune.IsNumber(rune) ? "一" : "☃");
+ sb.Append(rune.IsBmp ? rune.ToString()
+ : Rune.GetUnicodeCategory(rune) == System.Globalization.UnicodeCategory.DecimalDigitNumber ? "\u0663"
+ : Rune.IsLetter(rune) || Rune.IsNumber(rune) ? "\u4e00" : "\u2603");
return sb.ToString();
}
diff --git a/laya-dotnet/src/Laya/PythonFloat.cs b/laya-dotnet/src/Laya/PythonFloat.cs
index 631a064a..6766fbcd 100644
--- a/laya-dotnet/src/Laya/PythonFloat.cs
+++ b/laya-dotnet/src/Laya/PythonFloat.cs
@@ -64,9 +64,19 @@ internal static partial class PythonFloat
{
case bool b: value = b ? 1.0 : 0.0; return true;
case double d: value = d; return true;
- case long l: value = l; return true;
- case BigInteger big: value = (double)big; return true;
case string s: return TryParse(s, out value);
+ // Any number float() accepts. An integer rounds to nearest, ties to even, as Python's int -> float does;
+ // double.Parse reads the exact decimal text and rounds the same way, which a cast from decimal or from
+ // the 64-bit unsigned types does not promise.
+ case nint n: value = n; return true;
+ case long or int or short or sbyte or byte or ushort or uint:
+ value = Convert.ToInt64(raw, CultureInfo.InvariantCulture);
+ return true;
+ case ulong or nuint or BigInteger or decimal or Int128 or UInt128:
+ value = double.Parse(Convert.ToString(raw, CultureInfo.InvariantCulture)!, NumberStyles.Float, CultureInfo.InvariantCulture);
+ return true;
+ case float f: value = f; return true;
+ case Half h: value = (double)h; return true;
default: value = 0; return false;
}
}
diff --git a/laya-dotnet/src/Laya/PythonJson.cs b/laya-dotnet/src/Laya/PythonJson.cs
index e907b5d2..be606127 100644
--- a/laya-dotnet/src/Laya/PythonJson.cs
+++ b/laya-dotnet/src/Laya/PythonJson.cs
@@ -124,11 +124,11 @@ public static class PythonJson
/// <summary>
/// Whether <paramref name="value"/> is the .NET analogue of Python's <c>isinstance(state,
- /// list)</c>: any non-string, non-dict-like enumerable. There is no exact "list vs. tuple"
- /// distinction on this side, so anything <see cref="Dumps"/> would render as a JSON array
- /// counts, matching how <c>serialize_state</c> already treats the value.
+ /// list)</c>, which decides left truncation: an array or an <see cref="IList"/> that is not
+ /// dictionary-like. A tuple (<c>ITuple</c>) is still rendered as a JSON array but is not a list,
+ /// so it is truncated from the right, as Python truncates a tuple.
/// </summary>
- public static bool IsListState(object? value) => IsListLike(value);
+ public static bool IsListState(object? value) => value is IList && !IsDictLike(value);
private static void WriteValue(
StringBuilder sb, object? value,
diff --git a/laya-dotnet/tests/Laya.Tests/ReviewParityTests.cs b/laya-dotnet/tests/Laya.Tests/ReviewParityTests.cs
new file mode 100644
index 00000000..9d31dcc0
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/ReviewParityTests.cs
@@ -0,0 +1,117 @@
+using System.Numerics;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Parity checks added after review: boxed numeric types in <c>clamp_temperature</c> / <c>temperature_rejected</c>,
+/// and left truncation of a state only for a list. Expectations come from running the Python reference
+/// (<c>float(x)</c> for each numeric type, <c>build_sequence(..., truncate_left=isinstance(state, list))</c>).
+/// </summary>
+public sealed class ReviewParityTests
+{
+ public static TheoryData<string, string, string> Numbers() => new()
+ {
+ { "byte", "0", "0.0" },
+ { "byte", "255", "255.0" },
+ { "sbyte", "-128", "-128.0" },
+ { "sbyte", "127", "127.0" },
+ { "short", "-32768", "-32768.0" },
+ { "short", "32767", "32767.0" },
+ { "ushort", "0", "0.0" },
+ { "ushort", "65535", "65535.0" },
+ { "int", "-2147483648", "-2147483648.0" },
+ { "int", "2147483647", "2147483647.0" },
+ { "int", "16777217", "16777217.0" },
+ { "uint", "4294967295", "4294967295.0" },
+ { "long", "9223372036854775807", "9.223372036854776e+18" },
+ { "long", "-9223372036854775808", "-9.223372036854776e+18" },
+ { "long", "9007199254740993", "9007199254740992.0" },
+ { "long", "9007199254740995", "9007199254740996.0" },
+ { "ulong", "18446744073709551615", "1.8446744073709552e+19" },
+ { "ulong", "9007199254740993", "9007199254740992.0" },
+ { "ulong", "9007199254740995", "9007199254740996.0" },
+ { "ulong", "18446744073709550592", "1.8446744073709552e+19" },
+ { "BigInteger", "9007199254740993", "9007199254740992.0" },
+ { "BigInteger", "123456789012345678901234567890", "1.2345678901234568e+29" },
+ { "BigInteger", "-9007199254740993", "-9007199254740992.0" },
+ { "Int128", "170141183460469231731687303715884105727", "1.7014118346046923e+38" },
+ { "Int128", "9007199254740993", "9007199254740992.0" },
+ { "decimal", "0.1", "0.1" },
+ { "decimal", "1.0000000000000001", "1.0" },
+ { "decimal", "79228162514264337593543950335", "7.922816251426434e+28" },
+ { "decimal", "-0.5", "-0.5" },
+ { "decimal", "0.30000000000000004441", "0.30000000000000004" },
+ { "decimal", "2.5", "2.5" },
+ { "decimal", "4.35", "4.35" },
+ { "decimal", "0.0000000000000000000000000001", "1e-28" },
+ { "float", "0.1", "0.10000000149011612" },
+ { "float", "2.5", "2.5" },
+ { "float", "3.4028235e38", "3.4028234663852886e+38" },
+ { "float", "1.17549435e-38", "1.1754943508222875e-38" },
+ { "float", "-0.7", "-0.699999988079071" },
+ { "Half", "0.1", "0.0999755859375" },
+ { "Half", "2.5", "2.5" },
+ { "Half", "65504", "65504.0" },
+ { "Half", "-0.7", "-0.7001953125" },
+ };
+
+ private static object Box(string kind, string text) => kind switch
+ {
+ "byte" => byte.Parse(text), "sbyte" => sbyte.Parse(text), "short" => short.Parse(text), "ushort" => ushort.Parse(text),
+ "int" => int.Parse(text), "uint" => uint.Parse(text), "long" => long.Parse(text), "ulong" => ulong.Parse(text),
+ "BigInteger" => BigInteger.Parse(text), "Int128" => Int128.Parse(text), "decimal" => decimal.Parse(text, System.Globalization.CultureInfo.InvariantCulture),
+ "float" => float.Parse(text, System.Globalization.CultureInfo.InvariantCulture),
+ "Half" => Half.Parse(text, System.Globalization.CultureInfo.InvariantCulture),
+ _ => throw new ArgumentException(kind),
+ };
+
+ [Theory]
+ [MemberData(nameof(Numbers))]
+ public void EveryBoxedNumberConvertsAsPythonsFloatDoes(string kind, string text, string repr)
+ {
+ var boxed = Box(kind, text);
+ Assert.True(PythonFloat.TryConvert(boxed, out var value));
+ Assert.Equal(repr, PythonFloat.Repr(value));
+ Assert.Equal(Calibration.ClampTemperature(value), Calibration.ClampTemperature(boxed));
+ Assert.Equal(value != Calibration.ClampTemperature(value), Calibration.TemperatureRejected(boxed, Calibration.ClampTemperature(value)));
+ }
+
+ private sealed class DigitsTokenizer : Laya.Tokenization.ILayaTokenizer
+ {
+ public int PadId => 0;
+ public int SepId => 1;
+ public int ClsId => 2;
+ public int UnkId => 3;
+ public int MaskId => 4;
+ public string MaskToken => "[MASK]";
+ public int[] Encode(string text) =>
+ [.. System.Text.RegularExpressions.Regex.Matches(text, "[0-9]+").Select(m => int.Parse(m.Value))];
+ }
+
+ // The room for the state is four tokens, so eight numbers are over budget.
+ private static int[] Built(object state)
+ {
+ var question = Question.Choice("pick", [new KeyValuePair<string, object?>("a", "x"), new KeyValuePair<string, object?>("b", "y")]);
+ return SequenceBuilder.Build(new DigitsTokenizer(), state, question, 10, 10, PythonJson.IsListState(state)).Ids;
+ }
+
+ [Fact]
+ public void AListOrArrayKeepsTheEndOfAnOverBudgetState()
+ {
+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1004, 1005, 1006, 1007, 1 }, Built(new List<int> { 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007 }));
+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1004, 1005, 1006, 1007, 1 }, Built(new[] { 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007 }));
+ }
+
+ [Fact]
+ public void ATupleKeepsTheStartOfAnOverBudgetStateAsPythonDoes() =>
+ Assert.Equal(new[] { 2, 1, 4, 4, 1, 1000, 1001, 1002, 1003, 1 }, Built((1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007)));
+
+ [Fact]
+ public void ATupleIsNotAListState()
+ {
+ Assert.False(PythonJson.IsListState(("a", "b")));
+ Assert.True(PythonJson.IsListState(new[] { "a" }));
+ Assert.True(PythonJson.IsListState(new List<object?> { "a" }));
+ Assert.False(PythonJson.IsListState(new Dictionary<string, object?>()));
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json b/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json
index a257491d..594425cc 100644
--- a/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json
+++ b/laya-dotnet/tests/Laya.Tests/golden/routing/email_probe.json
@@ -1475,6 +1475,48 @@
"body": "Please help me.\nSent from my iPhone using KIT",
"max_chars": 3000,
"result": "Please help me."
+ },
+ {
+ "label": "pyre/astral_digit_quote_header",
+ "body": "Please help me.\nEm 𝟚𝟘𝟚𝟜 João escreveu:\n> old request\nStill mine.",
+ "max_chars": 3000,
+ "result": "Please help me."
+ },
+ {
+ "label": "pyre/astral_digit_french_header",
+ "body": "Please help me.\nLe 𝟚 janvier a écrit :\n> old request",
+ "max_chars": 3000,
+ "result": "Please help me."
+ },
+ {
+ "label": "pyre/astral_digit_header_pair",
+ "body": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚𝟜\nOld request.",
+ "max_chars": 3000,
+ "result": "Please help me."
+ },
+ {
+ "label": "pyre/astral_digit_in_year_needs_four",
+ "body": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚\nOld request.",
+ "max_chars": 3000,
+ "result": "Please help me.\nFrom: Maria Souza\nDate: 𝟚𝟘𝟚\nOld request."
+ },
+ {
+ "label": "pyre/astral_number_not_digit_header",
+ "body": "Please help me.\nEm 𐄇 João escreveu:\n> old request\nStill mine.",
+ "max_chars": 3000,
+ "result": "Please help me.\nEm 𐄇 João escreveu:\nStill mine."
+ },
+ {
+ "label": "pyre/astral_letter_not_digit_header",
+ "body": "Please help me.\nEm 𝐀 João escreveu:\n> old request\nStill mine.",
+ "max_chars": 3000,
+ "result": "Please help me.\nEm 𝐀 João escreveu:\nStill mine."
+ },
+ {
+ "label": "pyre/astral_digit_device_footer",
+ "body": "Please help me.\nSent from my iPhone 𝟚",
+ "max_chars": 3000,
+ "result": "Please help me."
}
],
"state": [
diff --git a/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json b/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json
index d807efc3..12a79316 100644
--- a/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json
+++ b/laya-dotnet/tests/Laya.Tests/golden/routing/meta.json
@@ -9,7 +9,7 @@
"counts": {
"lang_probe": 246,
"route_probe": 142,
- "email_probe_clean": 246,
+ "email_probe_clean": 253,
"email_probe_state": 22,
"shortlist_probe": 22,
"hashing_embedder_probe": 22
diff --git a/laya-dotnet/tools/routing_cases.py b/laya-dotnet/tools/routing_cases.py
index bd6cda3b..651efe80 100644
--- a/laya-dotnet/tools/routing_cases.py
+++ b/laya-dotnet/tools/routing_cases.py
@@ -928,6 +928,21 @@ EMAIL_CLEAN_CASES = [
("pyre/ignorecase_long_s_pt_signoff", "Preciso de ajuda.\nſaludoſ cordiales,\nMaria", 3000),
("pyre/ignorecase_kelvin_sign_signoff", "Please help me.\nThanK you,\nJohn", 3000),
("pyre/ignorecase_kelvin_sign_device_footer", "Please help me.\nSent from my iPhone using KIT", 3000),
+ # astral decimal digits: Python's Unicode `\d` (Nd) matches them; a number that is not a decimal digit (No) is `\w` only
+ ("pyre/astral_digit_quote_header",
+ "Please help me.\nEm \U0001d7da\U0001d7d8\U0001d7da\U0001d7dc João escreveu:\n> old request\nStill mine.", 3000),
+ ("pyre/astral_digit_french_header",
+ "Please help me.\nLe \U0001d7da janvier a écrit :\n> old request", 3000),
+ ("pyre/astral_digit_header_pair",
+ "Please help me.\nFrom: Maria Souza\nDate: \U0001d7da\U0001d7d8\U0001d7da\U0001d7dc\nOld request.", 3000),
+ ("pyre/astral_digit_in_year_needs_four",
+ "Please help me.\nFrom: Maria Souza\nDate: \U0001d7da\U0001d7d8\U0001d7da\nOld request.", 3000),
+ ("pyre/astral_number_not_digit_header",
+ "Please help me.\nEm \U00010107 João escreveu:\n> old request\nStill mine.", 3000),
+ ("pyre/astral_letter_not_digit_header",
+ "Please help me.\nEm \U0001d400 João escreveu:\n> old request\nStill mine.", 3000),
+ ("pyre/astral_digit_device_footer",
+ "Please help me.\nSent from my iPhone \U0001d7da", 3000),
]
EMAIL_STATE_CASES = [