Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(dotnet): refuse a wrongly typed or negative max_len at load
kevin-gatimu3f7cfa2a9d5e
diff --git a/laya-dotnet/CHANGELOG.md b/laya-dotnet/CHANGELOG.mdindex 6edbe76e..c29ad72c 100644--- a/laya-dotnet/CHANGELOG.md+++ b/laya-dotnet/CHANGELOG.md@@ -60,7 +60,9 @@ goldens: 0.4.12).does (a string yields its characters, a dict its keys, a number or null raises `TypeError`), `temperature_by_options`must be a mapping (`AttributeError`), `option_layout` other than `sequential` is refused, a `temperature` list shorterthan three fails at the first prediction that needs the missing entry (`IndexError`), and a wrong-typed `max_len` /- `head_max_len` loads and fails at the first prediction with Python's `TypeError`. `str.lower()` is exact (final sigma,+ `head_max_len` is refused when the config is loaded, with the `TypeError` text Python raises at the first prediction+ (a float, string, null, list or object; a negative integer is refused with its own message); integers `>= 0` and+ booleans are accepted. See "Differences from Python" in the README. `str.lower()` is exact (final sigma,U+0130), only lists and tuples are walked for text (a set holds none and is not JSON serializable), and any`IReadOnlyDictionary` is walked as a mapping.- Case-insensitive email matching also treats the long s with `s` and the Kelvin sign with `k`, as `re.I` does.diff --git a/laya-dotnet/README.md b/laya-dotnet/README.mdindex fd006a3c..8e66f6a1 100644--- a/laya-dotnet/README.md+++ b/laya-dotnet/README.md@@ -951,6 +951,23 @@ disclaimer, signature and reply-header rules, including Portuguese, Spanish and`truncated_questions` are exposed for `PredictLong` only; ordinary `Predict` still reportsjust input/output tokens.+## Differences from Python++Where this SDK deliberately does not match the Python reference, so nobody has to find out from a failing load:++- **`max_len` and `head_max_len` are checked when the config is loaded.** An integer `>= 0` or a boolean (`true` is 1,+ `false` is 0, as in Python) is accepted. Any other value is refused by `LayaConfig.Parse` / `LayaEngine.Create` with+ an `InvalidDataException`: a float, string, `null`, list or object gets the `TypeError` text Python raises at the first+ prediction (head first, then `max_len`), and a negative integer gets a message naming the field and value, because+ Python then slices from the end, which this port does not support. Python loads such a checkpoint and fails later. The+ related corner: a *fractional* `head_max_len` that Python happens to survive (a very small float) is refused here too.+- **JSON nesting.** Config and tokenizer JSON is read like `json.loads`, with the nesting cut-off set to 2,997 levels,+ what CPython 3.12 measured from a shallow call. Python's real limit depends on the call depth at the point of loading.+- **Config files are decoded as strict UTF-8.** Python opens them with the locale's encoding (cp1252 on a default Windows+ install), which only differs for non-ASCII bytes; a BOM is rejected, as `json.loads` does.+- **A `temperature` list shorter than three** loads, and fails with `IndexError: list index out of range` at the first+ prediction that needs the missing entry, as Python does.+---## Licensediff --git a/laya-dotnet/src/Laya/LayaConfig.cs b/laya-dotnet/src/Laya/LayaConfig.csindex 1c977ec1..25b8ccf2 100644--- a/laya-dotnet/src/Laya/LayaConfig.cs+++ b/laya-dotnet/src/Laya/LayaConfig.cs@@ -11,10 +11,8 @@ public sealed class LayaConfig{private LayaConfig(string? encoder, int maxLen, int headMaxLen,IReadOnlyList<object?> temperatureValues,- IReadOnlyDictionary<string, object?> temperatureByOptionsValues,- DeferredLengthError? deferredLength)+ IReadOnlyDictionary<string, object?> temperatureByOptionsValues){- DeferredLength = deferredLength;TemperatureRawValues = temperatureValues;TemperatureByOptionsRawValues = temperatureByOptionsValues;var temperatureRaw = temperatureValues.Select(AsDouble).ToList();@@ -143,14 +141,13 @@ public sealed class LayaConfig}ThrowIfOverflow(byOptions.Values);+ var lengths = Lengths(root);return new LayaConfig(root.TryGetValue("encoder", out var enc) && enc is string encoder ? encoder : null,- Length(root, "max_len", 512),- Length(root, "head_max_len", 192),+ lengths.MaxLen,+ lengths.HeadMaxLen,temperature,- byOptions,- DeferredLengthError.For(root.TryGetValue("max_len", out var maxRaw) ? maxRaw : 512L,- root.TryGetValue("head_max_len", out var headRaw) ? headRaw : 192L));+ byOptions);}// `for t in raw`: a list's items, a string's characters (code points), a dict's keys.@@ -184,76 +181,37 @@ public sealed class LayaConfigthrow new InvalidDataException("OverflowError: int too large to convert to float");}- // `cfg.get(key, fallback)` for max_len / head_max_len. Python keeps whatever the JSON held and only fails when a- // prediction does arithmetic or slicing with it, so a wrong type is recorded (kind) and raised then, not here;- // `true` / `false` are the ints 1 / 0. A negative integer slices from the end in Python, which this port does not- // support, so it is refused at load.- private static int Length(IReadOnlyDictionary<string, object?> root, string key, int fallback)+ // `cfg.get("max_len", 512)` and `cfg.get("head_max_len", 192)`. Python keeps whatever the JSON held and fails at+ // the first prediction that does arithmetic or slicing with it; this port refuses a value it cannot use when the+ // config is loaded, with the TypeError Python raises at that prediction (build_sequence meets head_max_len first,+ // then max_len). An int >= 0 or a bool (true is 1, false is 0, as in Python) is accepted. Anything else is refused:+ // a float, string, null, list or dict, and a negative int (Python slices from the end for it, which this port+ // does not support).+ private static (int MaxLen, int HeadMaxLen) Lengths(IReadOnlyDictionary<string, object?> root){- if (!root.TryGetValue(key, out var v)) return fallback;- switch (v)- {- case bool b: return b ? 1 : 0;- case long l when l < 0: throw NegativeLength(key, l.ToString(System.Globalization.CultureInfo.InvariantCulture));- case long l: return (int)Math.Min(l, int.MaxValue);- case System.Numerics.BigInteger big when big.Sign < 0: throw NegativeLength(key, big.ToString(System.Globalization.CultureInfo.InvariantCulture));- case System.Numerics.BigInteger: return int.MaxValue;- default: return fallback; // float, str, null, list, dict: DeferredLengthError raises at prediction- }- }-- private static InvalidDataException NegativeLength(string key, string literal) =>- new($"{key}={literal} is negative: Python slices from the end for it, which this port does not support");-- /// <summary>- /// A <c>max_len</c> / <c>head_max_len</c> of the wrong JSON type: loading succeeds and the first prediction raises- /// Python's <c>TypeError</c>.- /// </summary>- internal sealed record DeferredLengthError(string WithQuestions, string? NoQuestions, double? NoQuestionsWindow)- {- internal static DeferredLengthError? For(object? maxLen, object? headMaxLen)- {- var left = PythonFloat.TypeName(maxLen);- var right = PythonFloat.TypeName(headMaxLen);- var leftBad = left is not ("int" or "bool");- var rightBad = right is not ("int" or "bool");- if (!leftBad && !rightBad) return null;- // build_sequence meets head_max_len first (build_head), then max_len. A non-numeric head fails the subtraction- // at once; a float head normally fails the slice of the head tokens (which a very small float survives: not- // ported); then a max_len that is not an int fails its own subtraction or, for a float, the final slice.- const string SliceError = "TypeError: slice indices must be integers or None or have an __index__ method";- string Operand(string kind) => $"TypeError: unsupported operand type(s) for -: '{kind}' and 'int'";- var withQuestions = rightBad && right != "float" ? Operand(right)- : leftBad ? (left == "float" ? SliceError : Operand(left))- : SliceError;- // Without questions window_budget starts with `max_len - head_max_len`; a float on its own survives that.- static bool Numeric(string k) => k is "int" or "bool" or "float";- if (!(Numeric(left) && Numeric(right)))- return new DeferredLengthError(withQuestions, $"TypeError: unsupported operand type(s) for -: '{left}' and '{right}'", null);- // Only floats are wrong: window_budget runs, with `max(64, max_len - head_max_len - 8)` as a float window,- // and a state longer than that window fails later with the slice TypeError.- static double Number(object? v) => v switch { bool b => b ? 1 : 0, long l => l, System.Numerics.BigInteger big => (double)big, double d => d, _ => 0 };- return new DeferredLengthError(withQuestions, null, Math.Max(64.0, Number(maxLen) - Number(headMaxLen) - 8));- }- }-- /// <summary>The wrong-typed length setting's <c>TypeError</c>, raised by a prediction as Python does.</summary>- internal void ThrowIfLengthDeferred(bool withQuestions)- {- if (DeferredLength is not { } deferred) return;- var message = withQuestions ? deferred.WithQuestions : deferred.NoQuestions;- if (message is not null) throw new InvalidOperationException(message);+ var maxLen = root.TryGetValue("max_len", out var maxRaw) ? maxRaw : 512L;+ var headMaxLen = root.TryGetValue("head_max_len", out var headRaw) ? headRaw : 192L;+ var left = PythonFloat.TypeName(maxLen);+ var right = PythonFloat.TypeName(headMaxLen);+ const string SliceError = "TypeError: slice indices must be integers or None or have an __index__ method";+ string Operand(string kind) => $"TypeError: unsupported operand type(s) for -: '{kind}' and 'int'";+ var leftBad = left is not ("int" or "bool");+ var rightBad = right is not ("int" or "bool");+ if (rightBad && right != "float") throw new InvalidDataException(Operand(right));+ if (leftBad) throw new InvalidDataException(left == "float" ? SliceError : Operand(left));+ if (rightBad) throw new InvalidDataException(SliceError);+ return (Length("max_len", maxLen), Length("head_max_len", headMaxLen));}- /// <summary>- /// For a prediction without questions under float-only wrong lengths: a state of more tokens than the float window- /// Python scans with fails the slice <c>TypeError</c>, a shorter one reads in one pass.- /// </summary>- internal void ThrowIfStateExceedsDeferredWindow(int stateTokens)+ private static int Length(string key, object? v) => v switch{- if (DeferredLength is { NoQuestionsWindow: { } window } deferred && stateTokens > window)- throw new InvalidOperationException(deferred.WithQuestions);- }+ bool b => b ? 1 : 0,+ long l when l < 0 => throw NegativeLength(key, l.ToString(System.Globalization.CultureInfo.InvariantCulture)),+ long l => (int)Math.Min(l, int.MaxValue),+ System.Numerics.BigInteger big when big.Sign < 0 => throw NegativeLength(key, big.ToString(System.Globalization.CultureInfo.InvariantCulture)),+ _ => int.MaxValue, // an integer beyond 64 bits+ };- internal DeferredLengthError? DeferredLength { get; }+ private static InvalidDataException NegativeLength(string key, string literal) =>+ new($"{key}={literal} is negative: Python slices from the end for it, which this port does not support");}diff --git a/laya-dotnet/src/Laya/LayaEngine.cs b/laya-dotnet/src/Laya/LayaEngine.csindex 117f4bf4..1d973011 100644--- a/laya-dotnet/src/Laya/LayaEngine.cs+++ b/laya-dotnet/src/Laya/LayaEngine.cs@@ -198,7 +198,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictorArgumentNullException.ThrowIfNull(questions);ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);if (questions.Count == 0) throw new ArgumentException("no questions to answer", nameof(questions));- Config.ThrowIfLengthDeferred(withQuestions: true);return PredictStates([state], questions, diagnostics: false)[0];}@@ -227,7 +226,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictorObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);if (aggregate != "auto")throw new ArgumentException("predict_long: only aggregate='auto' is supported", nameof(aggregate));- Config.ThrowIfLengthDeferred(withQuestions: questions.Count > 0);int budget, step, room;int[] stateIds;@@ -237,7 +235,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictor_tokenizer, questions, Config.MaxLen, Config.HeadMaxLen, window, stride);stateIds = _tokenizer.Encode(PythonJson.State(state).Replace(_tokenizer.MaskToken, " ", StringComparison.Ordinal));}- if (questions.Count == 0) Config.ThrowIfStateExceedsDeferredWindow(stateIds.Length);// Predict reads the whole room the questions leave, so the default window must not// force a scan there; an explicit window still bounds the single pass (Python 3e9bda30).if (stateIds.Length <= (window is > 0 ? budget : room))diff --git a/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs b/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.csindex e4ed6202..76f2538c 100644--- a/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs+++ b/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs@@ -4,7 +4,7 @@ namespace Laya.Tests;/// <summary>/// <see cref="LayaConfig.Parse"/> against Python's loader (<c>onnx_agent.py:192-262</c>) on configs whose fields have the-/// wrong JSON type, and against <c>build_sequence</c> / <c>window_budget</c> for a wrong-typed <c>max_len</c> or+/// wrong JSON type, and against <c>build_sequence</c> for a wrong-typed <c>max_len</c> or/// <c>head_max_len</c>. Expectations were produced by running the Python code on the same documents. One corner is/// not ported and so not listed: a small float <c>head_max_len</c> that <c>build_head</c> survives (the typical float fails)./// </summary>@@ -72,102 +72,96 @@ public sealed class ConfigFieldTypeTestsAssert.Equal<IEnumerable<string>>(clamped, config.ClampedTemperatures);}- public static TheoryData<string, string?, string?> Lengths() => new()+ public static TheoryData<string, string?> Lengths() => new(){- { "{}", null, null },- { "{\"head_max_len\": 512}", null, null },- { "{\"head_max_len\": true}", null, null },- { "{\"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'str'" },- { "{\"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'" },- { "{\"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'list'" },- { "{\"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'dict'" },- { "{\"max_len\": 512}", null, null },- { "{\"max_len\": 512, \"head_max_len\": 512}", null, null },- { "{\"max_len\": 512, \"head_max_len\": true}", null, null },- { "{\"max_len\": 512, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'str'" },- { "{\"max_len\": 512, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'" },- { "{\"max_len\": 512, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'list'" },- { "{\"max_len\": 512, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'dict'" },- { "{\"max_len\": true}", null, null },- { "{\"max_len\": true, \"head_max_len\": 512}", null, null },- { "{\"max_len\": true, \"head_max_len\": true}", null, null },- { "{\"max_len\": true, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'str'" },- { "{\"max_len\": true, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'NoneType'" },- { "{\"max_len\": true, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'list'" },- { "{\"max_len\": true, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'dict'" },- { "{\"max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method", null },- { "{\"max_len\": 1.5, \"head_max_len\": 512}", "TypeError: slice indices must be integers or None or have an __index__ method", null },- { "{\"max_len\": 1.5, \"head_max_len\": true}", "TypeError: slice indices must be integers or None or have an __index__ method", null },- { "{\"max_len\": 1.5, \"head_max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method", null },- { "{\"max_len\": 1.5, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'str'" },- { "{\"max_len\": 1.5, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'NoneType'" },- { "{\"max_len\": 1.5, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'list'" },- { "{\"max_len\": 1.5, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'dict'" },- { "{\"max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },- { "{\"max_len\": \"x\", \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },- { "{\"max_len\": \"x\", \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'bool'" },- { "{\"max_len\": \"x\", \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'float'" },- { "{\"max_len\": \"x\", \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'str'" },- { "{\"max_len\": \"x\", \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'NoneType'" },- { "{\"max_len\": \"x\", \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'list'" },- { "{\"max_len\": \"x\", \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'dict'" },- { "{\"max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },- { "{\"max_len\": null, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },- { "{\"max_len\": null, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'bool'" },- { "{\"max_len\": null, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'float'" },- { "{\"max_len\": null, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'str'" },- { "{\"max_len\": null, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'NoneType'" },- { "{\"max_len\": null, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'list'" },- { "{\"max_len\": null, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'dict'" },- { "{\"max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },- { "{\"max_len\": [1], \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },- { "{\"max_len\": [1], \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'bool'" },- { "{\"max_len\": [1], \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'float'" },- { "{\"max_len\": [1], \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'str'" },- { "{\"max_len\": [1], \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'NoneType'" },- { "{\"max_len\": [1], \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'list'" },- { "{\"max_len\": [1], \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'dict'" },- { "{\"max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },- { "{\"max_len\": {}, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },- { "{\"max_len\": {}, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'bool'" },- { "{\"max_len\": {}, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'float'" },- { "{\"max_len\": {}, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'str'" },- { "{\"max_len\": {}, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'NoneType'" },- { "{\"max_len\": {}, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'list'" },- { "{\"max_len\": {}, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'dict'" },+ { "{}", null },+ { "{\"head_max_len\": 512}", null },+ { "{\"head_max_len\": true}", null },+ { "{\"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": 512}", null },+ { "{\"max_len\": 512, \"head_max_len\": 512}", null },+ { "{\"max_len\": 512, \"head_max_len\": true}", null },+ { "{\"max_len\": 512, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": 512, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": 512, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": 512, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": true}", null },+ { "{\"max_len\": true, \"head_max_len\": 512}", null },+ { "{\"max_len\": true, \"head_max_len\": true}", null },+ { "{\"max_len\": true, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": true, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": true, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": true, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method" },+ { "{\"max_len\": 1.5, \"head_max_len\": 512}", "TypeError: slice indices must be integers or None or have an __index__ method" },+ { "{\"max_len\": 1.5, \"head_max_len\": true}", "TypeError: slice indices must be integers or None or have an __index__ method" },+ { "{\"max_len\": 1.5, \"head_max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method" },+ { "{\"max_len\": 1.5, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": 1.5, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": 1.5, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": 1.5, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": \"x\", \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": null, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": [1], \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },+ { "{\"max_len\": {}, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },};[Theory][MemberData(nameof(Lengths))]- public void WrongTypedLengthsFailAtPredictionLikePython(string json, string? withQuestions, string? noQuestions)+ public void WrongTypedLengthsAreRefusedAtLoadWithPythonsTypeError(string json, string? error){- var config = LayaConfig.Parse(json); // Python loads these; the first prediction fails- Check(config, true, withQuestions);- Check(config, false, noQuestions);+ if (error is null) { LayaConfig.Parse(json); return; }+ Assert.Equal(error, Assert.Throws<InvalidDataException>(() => LayaConfig.Parse(json)).Message);}- private static void Check(LayaConfig config, bool withQuestions, string? expected)- {- if (expected is null) { config.ThrowIfLengthDeferred(withQuestions); return; }- Assert.Equal(expected, Assert.Throws<InvalidOperationException>(() => config.ThrowIfLengthDeferred(withQuestions)).Message);- }+ [Theory]+ [InlineData("{\"max_len\": -1}", "max_len=-1 is negative: Python slices from the end for it, which this port does not support")]+ [InlineData("{\"head_max_len\": -192}", "head_max_len=-192 is negative: Python slices from the end for it, which this port does not support")]+ [InlineData("{\"max_len\": -99999999999999999999999}", "max_len=-99999999999999999999999 is negative: Python slices from the end for it, which this port does not support")]+ public void NegativeLengthsAreRefusedAtLoad(string json, string error) =>+ Assert.Equal(error, Assert.Throws<InvalidDataException>(() => LayaConfig.Parse(json)).Message);- // Python scans a prediction without questions against the float window `max(64, max_len - head_max_len - 8)`:- // a state of at most that many tokens reads in one pass, a longer one fails with the slice TypeError.[Theory]- [InlineData("{\"max_len\": 1.5, \"head_max_len\": 512}", 64.0)]- [InlineData("{\"max_len\": 1000.5, \"head_max_len\": 192}", 800.5)]- [InlineData("{\"max_len\": 700.25, \"head_max_len\": 100.5}", 591.75)]- [InlineData("{\"max_len\": true, \"head_max_len\": 1.5}", 64.0)]- [InlineData("{\"max_len\": 512, \"head_max_len\": 300.5}", 203.5)]- [InlineData("{\"max_len\": 900.5, \"head_max_len\": true}", 891.5)]- [InlineData("{\"max_len\": 64.0, \"head_max_len\": 0}", 64.0)]- public void AFloatOnlyWrongLengthLeavesAFloatWindow(string json, double window)+ [InlineData("{\"max_len\": 0, \"head_max_len\": 0}", 0, 0)]+ [InlineData("{\"max_len\": true, \"head_max_len\": false}", 1, 0)]+ [InlineData("{\"max_len\": 1024, \"head_max_len\": 256}", 1024, 256)]+ [InlineData("{}", 512, 192)]+ [InlineData("{\"max_len\": 99999999999999999999}", int.MaxValue, 192)]+ public void NonNegativeIntsAndBoolsAreAccepted(string json, int maxLen, int headMaxLen){var config = LayaConfig.Parse(json);- config.ThrowIfLengthDeferred(withQuestions: false);- config.ThrowIfStateExceedsDeferredWindow((int)Math.Floor(window));- Assert.Throws<InvalidOperationException>(() => config.ThrowIfStateExceedsDeferredWindow((int)Math.Floor(window) + 1));+ Assert.Equal((maxLen, headMaxLen), (config.MaxLen, config.HeadMaxLen));}// Python evaluates `temperature[qt]` as the default of the by-options lookup, so a short list fails for that questiondiff --git a/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs b/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.csindex 70be4800..15152403 100644--- a/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs+++ b/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs@@ -552,11 +552,15 @@ public sealed class WarnOnceTestsAssert.Single(messages);}+ // One un-optimized call site shared by every thread: a jitted-and-re-jitted lambda may report a different IL offset.+ [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining | System.Runtime.CompilerServices.MethodImplOptions.NoOptimization)]+ private static void LoadFromOneSite() => LayaConfig.Parse(Config);+[Fact]public void ConcurrentLoadsWarnOnce(){WarnOnce.Reset();- var messages = Capturing(() => Parallel.For(0, 32, _ => LayaConfig.Parse(Config)), "temperature[0]=0.2 ", anyThread: true);+ var messages = Capturing(() => Parallel.For(0, 32, _ => LoadFromOneSite()), "temperature[0]=0.2 ", anyThread: true);Assert.Single(messages);}