Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(dotnet): parse and re-emit JSON without recursion, so depth cannot overflow the stack
thread3d7fa1bfc4fa55
diff --git a/laya-dotnet/src/Laya/PythonJsonReader.cs b/laya-dotnet/src/Laya/PythonJsonReader.csindex cf4a29e0..31e5cae9 100644--- a/laya-dotnet/src/Laya/PythonJsonReader.cs+++ b/laya-dotnet/src/Laya/PythonJsonReader.cs@@ -47,8 +47,7 @@ internal static class PythonJsonReader{var p = new State(text);if (text.StartsWith('\uFEFF')) throw p.Error("Unexpected UTF-8 BOM (decode using utf-8-sig)", 0);- p.SkipWhitespace();- var value = p.Value(0);+ var value = p.Value();p.SkipWhitespace();if (p.Pos != text.Length) throw p.Error("Extra data");return value;@@ -85,16 +84,117 @@ internal static class PythonJsonReader}/// <summary>Whether re-emitting <paramref name="value"/> as JSON text changes what a reader of it sees.</summary>- internal static bool IsAlteredByReEmission(object? value) => value switch+ /// <remarks>+ /// Walks with an explicit stack, for the same reason <see cref="Parse"/> and <see cref="Write"/> do: the+ /// value may be a document at <see cref="MaxDepth"/>, and a recursive walk would be the one remaining place+ /// a deep document could exhaust the caller's stack.+ /// </remarks>+ internal static bool IsAlteredByReEmission(object? value){- double d => !double.IsFinite(d),- string s => HasLoneSurrogate(s),- List<object?> list => list.Any(IsAlteredByReEmission),- IEnumerable<KeyValuePair<string, object?>> map => map.Any(kv => IsAlteredByReEmission(kv.Key) || IsAlteredByReEmission(kv.Value)),- _ => false,- };+ var pending = new Stack<object?>();+ pending.Push(value);+ while (pending.Count > 0)+ {+ switch (pending.Pop())+ {+ case double d when !double.IsFinite(d): return true;+ case string s when HasLoneSurrogate(s): return true;+ case List<object?> list:+ foreach (var item in list) pending.Push(item);+ break;+ case IEnumerable<KeyValuePair<string, object?>> map:+ foreach (var kv in map)+ {+ if (HasLoneSurrogate(kv.Key)) return true;+ pending.Push(kv.Value);+ }+ break;+ }+ }+ return false;+ }+ /// <summary>One open container while re-emitting, with the enumerator over its children.</summary>+ private struct WriteFrame+ {+ internal IEnumerator<object?>? Items; // an array: its elements+ internal IEnumerator<KeyValuePair<string, object?>>? Pairs; // an object: its members+ internal bool IsObject;+ }++ /// <summary>Re-emit <paramref name="value"/> as JSON text.</summary>+ /// <remarks>+ /// Iterative: <see cref="Utf8JsonWriter"/> keeps its own depth counter and does not recurse, so this walk was+ /// the only thing that made a deep document need stack proportional to its depth. A+ /// <c>tokenizer_config.json</c> nested at CPython's limit is re-emitted through here+ /// (<see cref="ParseDocument(string, out object?)"/>), so it must not be the call that overflows. Scalar+ /// spellings, key order and lone-surrogate replacement are unchanged.+ /// </remarks>private static void Write(Utf8JsonWriter w, object? value)+ {+ if (!IsContainer(value, out var items, out var pairs)) { WriteScalar(w, value); return; }+ var frames = new List<WriteFrame>();+ Open(w, items, pairs, frames);+ while (frames.Count > 0)+ {+ var top = frames.Count - 1;+ var frame = frames[top];+ if (frame.IsObject)+ {+ if (frame.Pairs!.MoveNext())+ {+ w.WritePropertyName(ReplaceLoneSurrogates(frame.Pairs.Current.Key));+ OpenChild(w, frame.Pairs.Current.Value, frames);+ continue;+ }+ w.WriteEndObject();+ }+ else+ {+ if (frame.Items!.MoveNext())+ {+ OpenChild(w, frame.Items.Current, frames);+ continue;+ }+ w.WriteEndArray();+ }+ frames.RemoveAt(top);+ }+ }++ /// <summary>Write one child: a scalar here, or a container's opening token and a frame to walk it.</summary>+ private static void OpenChild(Utf8JsonWriter w, object? value, List<WriteFrame> frames)+ {+ if (IsContainer(value, out var items, out var pairs)) Open(w, items, pairs, frames);+ else WriteScalar(w, value);+ }++ /// <summary>Write a container's opening token and push the frame that walks it.</summary>+ private static void Open(Utf8JsonWriter w, IEnumerator<object?>? items,+ IEnumerator<KeyValuePair<string, object?>>? pairs, List<WriteFrame> frames)+ {+ if (pairs is not null)+ {+ w.WriteStartObject();+ frames.Add(new WriteFrame { IsObject = true, Pairs = pairs });+ }+ else+ {+ w.WriteStartArray();+ frames.Add(new WriteFrame { Items = items });+ }+ }++ private static bool IsContainer(object? value, out IEnumerator<object?>? items,+ out IEnumerator<KeyValuePair<string, object?>>? pairs)+ {+ items = value is List<object?> list ? list.GetEnumerator() : null;+ pairs = value is IEnumerable<KeyValuePair<string, object?>> map ? map.GetEnumerator() : null;+ return items is not null || pairs is not null;+ }++ /// <summary>The scalar half of the re-emission: the same spellings the recursive walk wrote.</summary>+ private static void WriteScalar(Utf8JsonWriter w, object? value){switch (value){@@ -103,18 +203,8 @@ internal static class PythonJsonReadercase long l: w.WriteNumberValue(l); break;case BigInteger big: w.WriteRawValue(big.ToString(CultureInfo.InvariantCulture)); break;case double d when double.IsFinite(d): w.WriteNumberValue(d); break;- case double: w.WriteNullValue(); break;+ case double: w.WriteNullValue(); break; // NaN and the infinities are not JSONcase string s: w.WriteStringValue(ReplaceLoneSurrogates(s)); break;- case List<object?> list:- w.WriteStartArray();- foreach (var item in list) Write(w, item);- w.WriteEndArray();- break;- case IEnumerable<KeyValuePair<string, object?>> map:- w.WriteStartObject();- foreach (var kv in map) { w.WritePropertyName(ReplaceLoneSurrogates(kv.Key)); Write(w, kv.Value); }- w.WriteEndObject();- break;}}@@ -151,32 +241,163 @@ internal static class PythonJsonReaderwhile (Pos < text.Length && text[Pos] is ' ' or '\t' or '\n' or '\r') Pos++;}- internal object? Value(int depth)+ /// <summary>One open container while parsing: an array's list, or an object's map and pending key.</summary>+ private sealed class Frame+ {+ internal readonly List<object?>? Items;+ internal readonly OrderedDictionary<string, object?>? Map;+ internal string Key = "";++ internal Frame(List<object?> items) => Items = items;++ internal Frame(OrderedDictionary<string, object?> map, string key)+ {+ Map = map;+ Key = key;+ }++ internal bool IsObject => Map is not null;++ internal void Add(object? value)+ {+ if (Map is not null) Map[Key] = value;+ else Items!.Add(value);+ }+ }++ /// <summary>+ /// Parse one complete JSON value, leaving any trailing text for the caller (`Parse` reports "Extra data").+ /// </summary>+ /// <remarks>+ /// Iterative, for the reason <see cref="MaxDepth"/> exists at all: CPython loads documents nested nearly+ /// 3000 levels deep, and a recursive descent costs about two managed frames per level -- more than a+ /// thread with the platform-default 1 MB stack has. Reaching the limit that way is not an error the+ /// caller can catch: .NET cannot catch a stack overflow, so the process aborts instead, and a document+ /// CPython accepts (a downloaded <c>tokenizer_config.json</c>) kills the host. Containers are held in an+ /// explicit stack here, so depth costs heap and never stack.+ ///+ /// Everything observable is unchanged: tokens and their positions (the exact texts+ /// `PythonJsonErrorTests` pins), the scalar types, last-value-wins duplicate keys, and the depth limit,+ /// which counts the open containers -- the same number the recursive `depth` parameter carried.+ /// </remarks>+ internal object? Value(){- if (Pos >= text.Length) throw Error("Expecting value");- switch (text[Pos])+ var open = new Stack<Frame>();+ while (true){- case '"': Pos++; return String();- case '{':- if (depth >= MaxDepth) throw new JsonException("maximum recursion depth exceeded while decoding a JSON object from a unicode string");- Pos++;- return Object(depth + 1);- case '[':- if (depth >= MaxDepth) throw new JsonException("maximum recursion depth exceeded while decoding a JSON array from a unicode string");+ // One value, in "expect a value" position. The whitespace before it belongs here rather than in+ // the container loops: the recursive version skipped it in `Object`, `Array` and `Parse` alike.+ SkipWhitespace();+ if (Pos >= text.Length) throw Error("Expecting value");+ object? value;+ switch (text[Pos])+ {+ case '"':+ Pos++;+ value = String();+ break;+ case '{':+ if (open.Count >= MaxDepth)+ throw new JsonException("maximum recursion depth exceeded while decoding a JSON object from a unicode string");+ Pos++;+ var map = new OrderedDictionary<string, object?>(StringComparer.Ordinal);+ SkipWhitespace();+ if (Pos < text.Length && text[Pos] == '}')+ {+ Pos++;+ value = map; // `{}` completes without ever opening a frame+ break;+ }+ open.Push(new Frame(map, ReadKey()));+ continue; // now parse that member's value+ case '[':+ if (open.Count >= MaxDepth)+ throw new JsonException("maximum recursion depth exceeded while decoding a JSON array from a unicode string");+ Pos++;+ var list = new List<object?>();+ SkipWhitespace();+ if (Pos < text.Length && text[Pos] == ']')+ {+ Pos++;+ value = list; // `[]` completes without ever opening a frame+ break;+ }+ open.Push(new Frame(list));+ continue; // now parse the first element+ case 'n' when Literal("null"):+ value = null;+ break;+ case 't' when Literal("true"):+ value = true;+ break;+ case 'f' when Literal("false"):+ value = false;+ break;+ case 'N' when Literal("NaN"):+ value = double.NaN;+ break;+ case 'I' when Literal("Infinity"):+ value = double.PositiveInfinity;+ break;+ case '-' when string.CompareOrdinal(text, Pos, "-Infinity", 0, 9) == 0:+ Pos += 9;+ value = double.NegativeInfinity;+ break;+ default:+ value = Number() ?? throw Error("Expecting value");+ break;+ }++ // The value is complete: hand it to the innermost open container, then read the delimiter that+ // follows. A container that closes becomes a value for ITS parent, so closing walks outward+ // through this same loop instead of returning through stack frames.+ while (true)+ {+ if (open.Count == 0) return value;+ var frame = open.Peek();+ frame.Add(value);+ SkipWhitespace();+ if (frame.IsObject)+ {+ if (Pos < text.Length && text[Pos] == '}')+ {+ Pos++;+ open.Pop();+ value = frame.Map;+ continue;+ }+ if (Pos >= text.Length || text[Pos] != ',') throw Error("Expecting ',' delimiter");+ Pos++;+ SkipWhitespace();+ frame.Key = ReadKey();+ break; // parse the next member's value+ }+ if (Pos < text.Length && text[Pos] == ']')+ {+ Pos++;+ open.Pop();+ value = frame.Items;+ continue;+ }+ if (Pos >= text.Length || text[Pos] != ',') throw Error("Expecting ',' delimiter");Pos++;- return Array(depth + 1);- case 'n' when Literal("null"): return null;- case 't' when Literal("true"): return true;- case 'f' when Literal("false"): return false;- case 'N' when Literal("NaN"): return double.NaN;- case 'I' when Literal("Infinity"): return double.PositiveInfinity;- case '-' when string.CompareOrdinal(text, Pos, "-Infinity", 0, 9) == 0: Pos += 9; return double.NegativeInfinity;- default:- var number = Number();- return number ?? throw Error("Expecting value");+ break; // parse the next element+ }}}+ /// <summary>The <c>"key":</c> of one object member, with the whitespace around the colon consumed.</summary>+ private string ReadKey()+ {+ if (Pos >= text.Length || text[Pos] != '"') throw Error("Expecting property name enclosed in double quotes");+ Pos++;+ var key = String();+ SkipWhitespace();+ if (Pos >= text.Length || text[Pos] != ':') throw Error("Expecting ':' delimiter");+ Pos++;+ return key;+ }+private bool Literal(string word){if (string.CompareOrdinal(text, Pos, word, 0, word.Length) != 0) return false;@@ -273,43 +494,5 @@ internal static class PythonJsonReaderreturn (char)value; // a high surrogate followed by \uDC00.. joins in the UTF-16 string, as Python joins the pair}- private object Object(int depth)- {- var map = new OrderedDictionary<string, object?>(StringComparer.Ordinal);- SkipWhitespace();- if (Pos < text.Length && text[Pos] == '}') { Pos++; return map; }- while (true)- {- if (Pos >= text.Length || text[Pos] != '"') throw Error("Expecting property name enclosed in double quotes");- Pos++;- var key = String();- SkipWhitespace();- if (Pos >= text.Length || text[Pos] != ':') throw Error("Expecting ':' delimiter");- Pos++;- SkipWhitespace();- map[key] = Value(depth);- SkipWhitespace();- if (Pos < text.Length && text[Pos] == '}') { Pos++; return map; }- if (Pos >= text.Length || text[Pos] != ',') throw Error("Expecting ',' delimiter");- Pos++;- SkipWhitespace();- }- }-- private object Array(int depth)- {- var list = new List<object?>();- SkipWhitespace();- if (Pos < text.Length && text[Pos] == ']') { Pos++; return list; }- while (true)- {- list.Add(Value(depth));- SkipWhitespace();- if (Pos < text.Length && text[Pos] == ']') { Pos++; return list; }- if (Pos >= text.Length || text[Pos] != ',') throw Error("Expecting ',' delimiter");- Pos++;- SkipWhitespace();- }- }}}diff --git a/laya-dotnet/tests/Laya.Tests/PythonJsonErrorTests.cs b/laya-dotnet/tests/Laya.Tests/PythonJsonErrorTests.csindex 60abec50..f426490d 100644--- a/laya-dotnet/tests/Laya.Tests/PythonJsonErrorTests.cs+++ b/laya-dotnet/tests/Laya.Tests/PythonJsonErrorTests.cs@@ -120,6 +120,37 @@ public sealed class PythonJsonErrorTestsAssert.Equal(levels, depth);}+ [Fact]+ public void DeepNestingNeedsNoStackProportionalToItsDepth()+ {+ // 2990 levels on a 256 KB thread. Neither the parse nor the re-emission may recurse, because the+ // caller's stack is the one thing this port cannot ask anyone to make bigger. Before both walks became+ // iterative this test killed the test host outright ("Stack overflow ... Abort trap: 6") -- which is+ // the defect itself: .NET cannot catch a stack overflow, so a document CPython loads had no way to be+ // reported as an error either.+ var text = Nested(2990, "1");+ Exception? caught = null;+ var thread = new Thread(+ () =>+ {+ try+ {+ using var doc = PythonJsonReader.ParseDocument(text, out _);+ if (doc.RootElement.GetProperty("cls_token").GetString() != "[CLS]")+ throw new InvalidOperationException("re-emitted payload lost a field");+ var depth = 0;+ var element = doc.RootElement.GetProperty("x");+ while (element.ValueKind == JsonValueKind.Array) { depth++; element = element[0]; }+ if (depth != 2990) throw new InvalidOperationException($"depth {depth} != 2990");+ }+ catch (Exception e) { caught = e; }+ },+ maxStackSize: 256 * 1024);+ thread.Start();+ thread.Join();+ Assert.Null(caught);+ }+[Fact]public void NestingPastPythonsLimitIsRejected() =>Assert.ThrowsAny<JsonException>(() => PythonJsonReader.Parse(new string((char)91, 2998) + new string((char)93, 2998)));