Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(dotnet): refuse a wrongly typed or negative max_len at load

kevin-gatimu3f7cfa2a9d5e
diff --git a/laya-dotnet/CHANGELOG.md b/laya-dotnet/CHANGELOG.md
index 6edbe76e..c29ad72c 100644
--- a/laya-dotnet/CHANGELOG.md
+++ b/laya-dotnet/CHANGELOG.md
@@ -60,7 +60,9 @@ goldens: 0.4.12).
does (a string yields its characters, a dict its keys, a number or null raises `TypeError`), `temperature_by_options`
must be a mapping (`AttributeError`), `option_layout` other than `sequential` is refused, a `temperature` list shorter
than three fails at the first prediction that needs the missing entry (`IndexError`), and a wrong-typed `max_len` /
- `head_max_len` loads and fails at the first prediction with Python's `TypeError`. `str.lower()` is exact (final sigma,
+ `head_max_len` is refused when the config is loaded, with the `TypeError` text Python raises at the first prediction
+ (a float, string, null, list or object; a negative integer is refused with its own message); integers `>= 0` and
+ booleans are accepted. See "Differences from Python" in the README. `str.lower()` is exact (final sigma,
U+0130), only lists and tuples are walked for text (a set holds none and is not JSON serializable), and any
`IReadOnlyDictionary` is walked as a mapping.
- Case-insensitive email matching also treats the long s with `s` and the Kelvin sign with `k`, as `re.I` does.
diff --git a/laya-dotnet/README.md b/laya-dotnet/README.md
index fd006a3c..8e66f6a1 100644
--- a/laya-dotnet/README.md
+++ b/laya-dotnet/README.md
@@ -951,6 +951,23 @@ disclaimer, signature and reply-header rules, including Portuguese, Spanish and
`truncated_questions` are exposed for `PredictLong` only; ordinary `Predict` still reports
just input/output tokens.
+## Differences from Python
+
+Where this SDK deliberately does not match the Python reference, so nobody has to find out from a failing load:
+
+- **`max_len` and `head_max_len` are checked when the config is loaded.** An integer `>= 0` or a boolean (`true` is 1,
+ `false` is 0, as in Python) is accepted. Any other value is refused by `LayaConfig.Parse` / `LayaEngine.Create` with
+ an `InvalidDataException`: a float, string, `null`, list or object gets the `TypeError` text Python raises at the first
+ prediction (head first, then `max_len`), and a negative integer gets a message naming the field and value, because
+ Python then slices from the end, which this port does not support. Python loads such a checkpoint and fails later. The
+ related corner: a *fractional* `head_max_len` that Python happens to survive (a very small float) is refused here too.
+- **JSON nesting.** Config and tokenizer JSON is read like `json.loads`, with the nesting cut-off set to 2,997 levels,
+ what CPython 3.12 measured from a shallow call. Python's real limit depends on the call depth at the point of loading.
+- **Config files are decoded as strict UTF-8.** Python opens them with the locale's encoding (cp1252 on a default Windows
+ install), which only differs for non-ASCII bytes; a BOM is rejected, as `json.loads` does.
+- **A `temperature` list shorter than three** loads, and fails with `IndexError: list index out of range` at the first
+ prediction that needs the missing entry, as Python does.
+
---
## License
diff --git a/laya-dotnet/src/Laya/LayaConfig.cs b/laya-dotnet/src/Laya/LayaConfig.cs
index 1c977ec1..25b8ccf2 100644
--- a/laya-dotnet/src/Laya/LayaConfig.cs
+++ b/laya-dotnet/src/Laya/LayaConfig.cs
@@ -11,10 +11,8 @@ public sealed class LayaConfig
{
private LayaConfig(string? encoder, int maxLen, int headMaxLen,
IReadOnlyList<object?> temperatureValues,
- IReadOnlyDictionary<string, object?> temperatureByOptionsValues,
- DeferredLengthError? deferredLength)
+ IReadOnlyDictionary<string, object?> temperatureByOptionsValues)
{
- DeferredLength = deferredLength;
TemperatureRawValues = temperatureValues;
TemperatureByOptionsRawValues = temperatureByOptionsValues;
var temperatureRaw = temperatureValues.Select(AsDouble).ToList();
@@ -143,14 +141,13 @@ public sealed class LayaConfig
}
ThrowIfOverflow(byOptions.Values);
+ var lengths = Lengths(root);
return new LayaConfig(
root.TryGetValue("encoder", out var enc) && enc is string encoder ? encoder : null,
- Length(root, "max_len", 512),
- Length(root, "head_max_len", 192),
+ lengths.MaxLen,
+ lengths.HeadMaxLen,
temperature,
- byOptions,
- DeferredLengthError.For(root.TryGetValue("max_len", out var maxRaw) ? maxRaw : 512L,
- root.TryGetValue("head_max_len", out var headRaw) ? headRaw : 192L));
+ byOptions);
}
// `for t in raw`: a list's items, a string's characters (code points), a dict's keys.
@@ -184,76 +181,37 @@ public sealed class LayaConfig
throw new InvalidDataException("OverflowError: int too large to convert to float");
}
- // `cfg.get(key, fallback)` for max_len / head_max_len. Python keeps whatever the JSON held and only fails when a
- // prediction does arithmetic or slicing with it, so a wrong type is recorded (kind) and raised then, not here;
- // `true` / `false` are the ints 1 / 0. A negative integer slices from the end in Python, which this port does not
- // support, so it is refused at load.
- private static int Length(IReadOnlyDictionary<string, object?> root, string key, int fallback)
+ // `cfg.get("max_len", 512)` and `cfg.get("head_max_len", 192)`. Python keeps whatever the JSON held and fails at
+ // the first prediction that does arithmetic or slicing with it; this port refuses a value it cannot use when the
+ // config is loaded, with the TypeError Python raises at that prediction (build_sequence meets head_max_len first,
+ // then max_len). An int >= 0 or a bool (true is 1, false is 0, as in Python) is accepted. Anything else is refused:
+ // a float, string, null, list or dict, and a negative int (Python slices from the end for it, which this port
+ // does not support).
+ private static (int MaxLen, int HeadMaxLen) Lengths(IReadOnlyDictionary<string, object?> root)
{
- if (!root.TryGetValue(key, out var v)) return fallback;
- switch (v)
- {
- case bool b: return b ? 1 : 0;
- case long l when l < 0: throw NegativeLength(key, l.ToString(System.Globalization.CultureInfo.InvariantCulture));
- case long l: return (int)Math.Min(l, int.MaxValue);
- case System.Numerics.BigInteger big when big.Sign < 0: throw NegativeLength(key, big.ToString(System.Globalization.CultureInfo.InvariantCulture));
- case System.Numerics.BigInteger: return int.MaxValue;
- default: return fallback; // float, str, null, list, dict: DeferredLengthError raises at prediction
- }
- }
-
- private static InvalidDataException NegativeLength(string key, string literal) =>
- new($"{key}={literal} is negative: Python slices from the end for it, which this port does not support");
-
- /// <summary>
- /// A <c>max_len</c> / <c>head_max_len</c> of the wrong JSON type: loading succeeds and the first prediction raises
- /// Python's <c>TypeError</c>.
- /// </summary>
- internal sealed record DeferredLengthError(string WithQuestions, string? NoQuestions, double? NoQuestionsWindow)
- {
- internal static DeferredLengthError? For(object? maxLen, object? headMaxLen)
- {
- var left = PythonFloat.TypeName(maxLen);
- var right = PythonFloat.TypeName(headMaxLen);
- var leftBad = left is not ("int" or "bool");
- var rightBad = right is not ("int" or "bool");
- if (!leftBad && !rightBad) return null;
- // build_sequence meets head_max_len first (build_head), then max_len. A non-numeric head fails the subtraction
- // at once; a float head normally fails the slice of the head tokens (which a very small float survives: not
- // ported); then a max_len that is not an int fails its own subtraction or, for a float, the final slice.
- const string SliceError = "TypeError: slice indices must be integers or None or have an __index__ method";
- string Operand(string kind) => $"TypeError: unsupported operand type(s) for -: '{kind}' and 'int'";
- var withQuestions = rightBad && right != "float" ? Operand(right)
- : leftBad ? (left == "float" ? SliceError : Operand(left))
- : SliceError;
- // Without questions window_budget starts with `max_len - head_max_len`; a float on its own survives that.
- static bool Numeric(string k) => k is "int" or "bool" or "float";
- if (!(Numeric(left) && Numeric(right)))
- return new DeferredLengthError(withQuestions, $"TypeError: unsupported operand type(s) for -: '{left}' and '{right}'", null);
- // Only floats are wrong: window_budget runs, with `max(64, max_len - head_max_len - 8)` as a float window,
- // and a state longer than that window fails later with the slice TypeError.
- static double Number(object? v) => v switch { bool b => b ? 1 : 0, long l => l, System.Numerics.BigInteger big => (double)big, double d => d, _ => 0 };
- return new DeferredLengthError(withQuestions, null, Math.Max(64.0, Number(maxLen) - Number(headMaxLen) - 8));
- }
- }
-
- /// <summary>The wrong-typed length setting's <c>TypeError</c>, raised by a prediction as Python does.</summary>
- internal void ThrowIfLengthDeferred(bool withQuestions)
- {
- if (DeferredLength is not { } deferred) return;
- var message = withQuestions ? deferred.WithQuestions : deferred.NoQuestions;
- if (message is not null) throw new InvalidOperationException(message);
+ var maxLen = root.TryGetValue("max_len", out var maxRaw) ? maxRaw : 512L;
+ var headMaxLen = root.TryGetValue("head_max_len", out var headRaw) ? headRaw : 192L;
+ var left = PythonFloat.TypeName(maxLen);
+ var right = PythonFloat.TypeName(headMaxLen);
+ const string SliceError = "TypeError: slice indices must be integers or None or have an __index__ method";
+ string Operand(string kind) => $"TypeError: unsupported operand type(s) for -: '{kind}' and 'int'";
+ var leftBad = left is not ("int" or "bool");
+ var rightBad = right is not ("int" or "bool");
+ if (rightBad && right != "float") throw new InvalidDataException(Operand(right));
+ if (leftBad) throw new InvalidDataException(left == "float" ? SliceError : Operand(left));
+ if (rightBad) throw new InvalidDataException(SliceError);
+ return (Length("max_len", maxLen), Length("head_max_len", headMaxLen));
}
- /// <summary>
- /// For a prediction without questions under float-only wrong lengths: a state of more tokens than the float window
- /// Python scans with fails the slice <c>TypeError</c>, a shorter one reads in one pass.
- /// </summary>
- internal void ThrowIfStateExceedsDeferredWindow(int stateTokens)
+ private static int Length(string key, object? v) => v switch
{
- if (DeferredLength is { NoQuestionsWindow: { } window } deferred && stateTokens > window)
- throw new InvalidOperationException(deferred.WithQuestions);
- }
+ bool b => b ? 1 : 0,
+ long l when l < 0 => throw NegativeLength(key, l.ToString(System.Globalization.CultureInfo.InvariantCulture)),
+ long l => (int)Math.Min(l, int.MaxValue),
+ System.Numerics.BigInteger big when big.Sign < 0 => throw NegativeLength(key, big.ToString(System.Globalization.CultureInfo.InvariantCulture)),
+ _ => int.MaxValue, // an integer beyond 64 bits
+ };
- internal DeferredLengthError? DeferredLength { get; }
+ private static InvalidDataException NegativeLength(string key, string literal) =>
+ new($"{key}={literal} is negative: Python slices from the end for it, which this port does not support");
}
diff --git a/laya-dotnet/src/Laya/LayaEngine.cs b/laya-dotnet/src/Laya/LayaEngine.cs
index 117f4bf4..1d973011 100644
--- a/laya-dotnet/src/Laya/LayaEngine.cs
+++ b/laya-dotnet/src/Laya/LayaEngine.cs
@@ -198,7 +198,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictor
ArgumentNullException.ThrowIfNull(questions);
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
if (questions.Count == 0) throw new ArgumentException("no questions to answer", nameof(questions));
- Config.ThrowIfLengthDeferred(withQuestions: true);
return PredictStates([state], questions, diagnostics: false)[0];
}
@@ -227,7 +226,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictor
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
if (aggregate != "auto")
throw new ArgumentException("predict_long: only aggregate='auto' is supported", nameof(aggregate));
- Config.ThrowIfLengthDeferred(withQuestions: questions.Count > 0);
int budget, step, room;
int[] stateIds;
@@ -237,7 +235,6 @@ public sealed class LayaEngine : IDisposable, ILayaLongPredictor
_tokenizer, questions, Config.MaxLen, Config.HeadMaxLen, window, stride);
stateIds = _tokenizer.Encode(PythonJson.State(state).Replace(_tokenizer.MaskToken, " ", StringComparison.Ordinal));
}
- if (questions.Count == 0) Config.ThrowIfStateExceedsDeferredWindow(stateIds.Length);
// Predict reads the whole room the questions leave, so the default window must not
// force a scan there; an explicit window still bounds the single pass (Python 3e9bda30).
if (stateIds.Length <= (window is > 0 ? budget : room))
diff --git a/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs b/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs
index e4ed6202..76f2538c 100644
--- a/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/ConfigFieldTypeTests.cs
@@ -4,7 +4,7 @@ namespace Laya.Tests;
/// <summary>
/// <see cref="LayaConfig.Parse"/> against Python's loader (<c>onnx_agent.py:192-262</c>) on configs whose fields have the
-/// wrong JSON type, and against <c>build_sequence</c> / <c>window_budget</c> for a wrong-typed <c>max_len</c> or
+/// wrong JSON type, and against <c>build_sequence</c> for a wrong-typed <c>max_len</c> or
/// <c>head_max_len</c>. Expectations were produced by running the Python code on the same documents. One corner is
/// not ported and so not listed: a small float <c>head_max_len</c> that <c>build_head</c> survives (the typical float fails).
/// </summary>
@@ -72,102 +72,96 @@ public sealed class ConfigFieldTypeTests
Assert.Equal<IEnumerable<string>>(clamped, config.ClampedTemperatures);
}
- public static TheoryData<string, string?, string?> Lengths() => new()
+ public static TheoryData<string, string?> Lengths() => new()
{
- { "{}", null, null },
- { "{\"head_max_len\": 512}", null, null },
- { "{\"head_max_len\": true}", null, null },
- { "{\"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'str'" },
- { "{\"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'" },
- { "{\"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'list'" },
- { "{\"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'dict'" },
- { "{\"max_len\": 512}", null, null },
- { "{\"max_len\": 512, \"head_max_len\": 512}", null, null },
- { "{\"max_len\": 512, \"head_max_len\": true}", null, null },
- { "{\"max_len\": 512, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'str'" },
- { "{\"max_len\": 512, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'" },
- { "{\"max_len\": 512, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'list'" },
- { "{\"max_len\": 512, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'int' and 'dict'" },
- { "{\"max_len\": true}", null, null },
- { "{\"max_len\": true, \"head_max_len\": 512}", null, null },
- { "{\"max_len\": true, \"head_max_len\": true}", null, null },
- { "{\"max_len\": true, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'str'" },
- { "{\"max_len\": true, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'NoneType'" },
- { "{\"max_len\": true, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'list'" },
- { "{\"max_len\": true, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'bool' and 'dict'" },
- { "{\"max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method", null },
- { "{\"max_len\": 1.5, \"head_max_len\": 512}", "TypeError: slice indices must be integers or None or have an __index__ method", null },
- { "{\"max_len\": 1.5, \"head_max_len\": true}", "TypeError: slice indices must be integers or None or have an __index__ method", null },
- { "{\"max_len\": 1.5, \"head_max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method", null },
- { "{\"max_len\": 1.5, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'str'" },
- { "{\"max_len\": 1.5, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'NoneType'" },
- { "{\"max_len\": 1.5, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'list'" },
- { "{\"max_len\": 1.5, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'float' and 'dict'" },
- { "{\"max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
- { "{\"max_len\": \"x\", \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
- { "{\"max_len\": \"x\", \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'bool'" },
- { "{\"max_len\": \"x\", \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'float'" },
- { "{\"max_len\": \"x\", \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'str'" },
- { "{\"max_len\": \"x\", \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'NoneType'" },
- { "{\"max_len\": \"x\", \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'list'" },
- { "{\"max_len\": \"x\", \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'str' and 'dict'" },
- { "{\"max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
- { "{\"max_len\": null, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
- { "{\"max_len\": null, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'bool'" },
- { "{\"max_len\": null, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'float'" },
- { "{\"max_len\": null, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'str'" },
- { "{\"max_len\": null, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'NoneType'" },
- { "{\"max_len\": null, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'list'" },
- { "{\"max_len\": null, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'dict'" },
- { "{\"max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
- { "{\"max_len\": [1], \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
- { "{\"max_len\": [1], \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'bool'" },
- { "{\"max_len\": [1], \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'float'" },
- { "{\"max_len\": [1], \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'str'" },
- { "{\"max_len\": [1], \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'NoneType'" },
- { "{\"max_len\": [1], \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'list'" },
- { "{\"max_len\": [1], \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'list' and 'dict'" },
- { "{\"max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
- { "{\"max_len\": {}, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
- { "{\"max_len\": {}, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'bool'" },
- { "{\"max_len\": {}, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'float'" },
- { "{\"max_len\": {}, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'str'" },
- { "{\"max_len\": {}, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'NoneType'" },
- { "{\"max_len\": {}, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'list'" },
- { "{\"max_len\": {}, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'", "TypeError: unsupported operand type(s) for -: 'dict' and 'dict'" },
+ { "{}", null },
+ { "{\"head_max_len\": 512}", null },
+ { "{\"head_max_len\": true}", null },
+ { "{\"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": 512}", null },
+ { "{\"max_len\": 512, \"head_max_len\": 512}", null },
+ { "{\"max_len\": 512, \"head_max_len\": true}", null },
+ { "{\"max_len\": 512, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": 512, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": 512, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": 512, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": true}", null },
+ { "{\"max_len\": true, \"head_max_len\": 512}", null },
+ { "{\"max_len\": true, \"head_max_len\": true}", null },
+ { "{\"max_len\": true, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": true, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": true, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": true, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method" },
+ { "{\"max_len\": 1.5, \"head_max_len\": 512}", "TypeError: slice indices must be integers or None or have an __index__ method" },
+ { "{\"max_len\": 1.5, \"head_max_len\": true}", "TypeError: slice indices must be integers or None or have an __index__ method" },
+ { "{\"max_len\": 1.5, \"head_max_len\": 1.5}", "TypeError: slice indices must be integers or None or have an __index__ method" },
+ { "{\"max_len\": 1.5, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": 1.5, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": 1.5, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": 1.5, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": \"x\", \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": null, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": [1], \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": 512}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": true}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": 1.5}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": \"x\"}", "TypeError: unsupported operand type(s) for -: 'str' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": null}", "TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": [1]}", "TypeError: unsupported operand type(s) for -: 'list' and 'int'" },
+ { "{\"max_len\": {}, \"head_max_len\": {}}", "TypeError: unsupported operand type(s) for -: 'dict' and 'int'" },
};
[Theory]
[MemberData(nameof(Lengths))]
- public void WrongTypedLengthsFailAtPredictionLikePython(string json, string? withQuestions, string? noQuestions)
+ public void WrongTypedLengthsAreRefusedAtLoadWithPythonsTypeError(string json, string? error)
{
- var config = LayaConfig.Parse(json); // Python loads these; the first prediction fails
- Check(config, true, withQuestions);
- Check(config, false, noQuestions);
+ if (error is null) { LayaConfig.Parse(json); return; }
+ Assert.Equal(error, Assert.Throws<InvalidDataException>(() => LayaConfig.Parse(json)).Message);
}
- private static void Check(LayaConfig config, bool withQuestions, string? expected)
- {
- if (expected is null) { config.ThrowIfLengthDeferred(withQuestions); return; }
- Assert.Equal(expected, Assert.Throws<InvalidOperationException>(() => config.ThrowIfLengthDeferred(withQuestions)).Message);
- }
+ [Theory]
+ [InlineData("{\"max_len\": -1}", "max_len=-1 is negative: Python slices from the end for it, which this port does not support")]
+ [InlineData("{\"head_max_len\": -192}", "head_max_len=-192 is negative: Python slices from the end for it, which this port does not support")]
+ [InlineData("{\"max_len\": -99999999999999999999999}", "max_len=-99999999999999999999999 is negative: Python slices from the end for it, which this port does not support")]
+ public void NegativeLengthsAreRefusedAtLoad(string json, string error) =>
+ Assert.Equal(error, Assert.Throws<InvalidDataException>(() => LayaConfig.Parse(json)).Message);
- // Python scans a prediction without questions against the float window `max(64, max_len - head_max_len - 8)`:
- // a state of at most that many tokens reads in one pass, a longer one fails with the slice TypeError.
[Theory]
- [InlineData("{\"max_len\": 1.5, \"head_max_len\": 512}", 64.0)]
- [InlineData("{\"max_len\": 1000.5, \"head_max_len\": 192}", 800.5)]
- [InlineData("{\"max_len\": 700.25, \"head_max_len\": 100.5}", 591.75)]
- [InlineData("{\"max_len\": true, \"head_max_len\": 1.5}", 64.0)]
- [InlineData("{\"max_len\": 512, \"head_max_len\": 300.5}", 203.5)]
- [InlineData("{\"max_len\": 900.5, \"head_max_len\": true}", 891.5)]
- [InlineData("{\"max_len\": 64.0, \"head_max_len\": 0}", 64.0)]
- public void AFloatOnlyWrongLengthLeavesAFloatWindow(string json, double window)
+ [InlineData("{\"max_len\": 0, \"head_max_len\": 0}", 0, 0)]
+ [InlineData("{\"max_len\": true, \"head_max_len\": false}", 1, 0)]
+ [InlineData("{\"max_len\": 1024, \"head_max_len\": 256}", 1024, 256)]
+ [InlineData("{}", 512, 192)]
+ [InlineData("{\"max_len\": 99999999999999999999}", int.MaxValue, 192)]
+ public void NonNegativeIntsAndBoolsAreAccepted(string json, int maxLen, int headMaxLen)
{
var config = LayaConfig.Parse(json);
- config.ThrowIfLengthDeferred(withQuestions: false);
- config.ThrowIfStateExceedsDeferredWindow((int)Math.Floor(window));
- Assert.Throws<InvalidOperationException>(() => config.ThrowIfStateExceedsDeferredWindow((int)Math.Floor(window) + 1));
+ Assert.Equal((maxLen, headMaxLen), (config.MaxLen, config.HeadMaxLen));
}
// Python evaluates `temperature[qt]` as the default of the by-options lookup, so a short list fails for that question
diff --git a/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs b/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs
index 70be4800..15152403 100644
--- a/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/PythonJsonReaderTests.cs
@@ -552,11 +552,15 @@ public sealed class WarnOnceTests
Assert.Single(messages);
}
+ // One un-optimized call site shared by every thread: a jitted-and-re-jitted lambda may report a different IL offset.
+ [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining | System.Runtime.CompilerServices.MethodImplOptions.NoOptimization)]
+ private static void LoadFromOneSite() => LayaConfig.Parse(Config);
+
[Fact]
public void ConcurrentLoadsWarnOnce()
{
WarnOnce.Reset();
- var messages = Capturing(() => Parallel.For(0, 32, _ => LayaConfig.Parse(Config)), "temperature[0]=0.2 ", anyThread: true);
+ var messages = Capturing(() => Parallel.For(0, 32, _ => LoadFromOneSite()), "temperature[0]=0.2 ", anyThread: true);
Assert.Single(messages);
}