Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(dotnet): add typed questions, sequence builder and collator
kevin-gatimu1fa53c35c1a0
diff --git a/laya-dotnet/src/Laya/Collator.cs b/laya-dotnet/src/Laya/Collator.csnew file mode 100644index 00000000..7e422e98--- /dev/null+++ b/laya-dotnet/src/Laya/Collator.cs@@ -0,0 +1,121 @@+namespace Laya;++/// <summary>One question, tokenized and ready to be batched.</summary>+/// <param name="Ids">The token sequence.</param>+/// <param name="Markers">One position per option, pointing at that option's <c>[MASK]</c>.</param>+/// <param name="Type">Which question type, fed to the model's type embedding.</param>+public readonly record struct SequenceItem(int[] Ids, int[] Markers, QuestionType Type);++/// <summary>A padded batch, laid out as the row-major tensors the ONNX graph takes.</summary>+public sealed class CollatedBatch+{+ internal CollatedBatch(int count, int sequenceLength, int markerCount,+ long[] inputIds, long[] attentionMask, long[] markerPos,+ bool[] markerMask, long[] qtype, int[] markerCounts, int inputTokens)+ {+ Count = count;+ SequenceLength = sequenceLength;+ MarkerCount = markerCount;+ InputIds = inputIds;+ AttentionMask = attentionMask;+ MarkerPos = markerPos;+ MarkerMask = markerMask;+ QType = qtype;+ MarkerCounts = markerCounts;+ InputTokens = inputTokens;+ }++ /// <summary>Rows in the batch: one per question.</summary>+ public int Count { get; }++ /// <summary>Padded sequence length, the longest row.</summary>+ public int SequenceLength { get; }++ /// <summary>Padded marker-axis width.</summary>+ public int MarkerCount { get; }++ /// <summary><c>[Count, SequenceLength]</c>, padded with the pad id.</summary>+ public long[] InputIds { get; }++ /// <summary><c>[Count, SequenceLength]</c>, 1 on real tokens and 0 on padding.</summary>+ public long[] AttentionMask { get; }++ /// <summary><c>[Count, MarkerCount]</c>, 0 in padding columns.</summary>+ public long[] MarkerPos { get; }++ /// <summary><c>[Count, MarkerCount]</c>, false in padding columns.</summary>+ public bool[] MarkerMask { get; }++ /// <summary><c>[Count]</c>, the question type per row.</summary>+ public long[] QType { get; }++ /// <summary>The real (unpadded) option count per row — how many logits to read from that row.</summary>+ public int[] MarkerCounts { get; }++ /// <summary>Non-padding tokens across the batch, which is what Python reports as input tokens.</summary>+ public int InputTokens { get; }+}++/// <summary>Pads tokenized questions into one batch. Ports <c>collate_items</c> from <c>laya/common.py</c>.</summary>+public static class Collator+{+ /// <summary>+ /// The marker axis is never narrower than this.+ /// </summary>+ /// <remarks>+ /// The tracer baked <c>TopK k=2</c> into the exported graph: Python chooses the top-2 branch at+ /// runtime and the export only ever saw two or more markers, so a batch whose questions all+ /// have a single option makes TopK fail outright. Padding to two columns with+ /// <c>marker_mask = false</c> is exactly equivalent, not merely close: the model's+ /// <c>masked_fill</c> puts <c>-1e4</c> in the pad column, which underflows softmax to 0.0, so+ /// <c>p == [1.0, 0.0]</c> — precisely the <c>[top1, 0.0]</c> Python substitutes — and+ /// <c>k = marker_mask.sum().clamp(min=2) == 2</c> either way. Reading only+ /// <see cref="CollatedBatch.MarkerCounts"/> logits per row keeps the pad column out of the answer.+ /// </remarks>+ public const int MinMarkers = 2;++ /// <summary>Pad a batch of tokenized questions.</summary>+ public static CollatedBatch Collate(IReadOnlyList<SequenceItem> items, int padId)+ {+ ArgumentNullException.ThrowIfNull(items);+ if (items.Count == 0) throw new ArgumentException("nothing to collate", nameof(items));++ var n = items.Count;+ var length = items.Max(it => it.Ids.Length);+ var markerCount = Math.Max(MinMarkers, items.Max(it => it.Markers.Length));++ var ids = new long[n * length];+ if (padId != 0) Array.Fill(ids, padId);+ var attention = new long[n * length];+ var markerPos = new long[n * markerCount];+ var markerMask = new bool[n * markerCount];+ var qtype = new long[n];+ var markerCounts = new int[n];+ var inputTokens = 0;++ for (var i = 0; i < n; i++)+ {+ var item = items[i];+ var row = i * length;+ for (var j = 0; j < item.Ids.Length; j++)+ {+ ids[row + j] = item.Ids[j];+ attention[row + j] = 1;+ }+ inputTokens += item.Ids.Length;++ var mrow = i * markerCount;+ for (var j = 0; j < item.Markers.Length; j++)+ {+ markerPos[mrow + j] = item.Markers[j];+ markerMask[mrow + j] = true;+ }++ qtype[i] = (long)item.Type;+ markerCounts[i] = item.Markers.Length;+ }++ return new CollatedBatch(n, length, markerCount, ids, attention, markerPos, markerMask,+ qtype, markerCounts, inputTokens);+ }+}diff --git a/laya-dotnet/src/Laya/Questions.cs b/laya-dotnet/src/Laya/Questions.csnew file mode 100644index 00000000..a6e99a6c--- /dev/null+++ b/laya-dotnet/src/Laya/Questions.cs@@ -0,0 +1,205 @@+namespace Laya;++/// <summary>The three question types Laya answers. Values match Python's <c>QTYPES</c>.</summary>+public enum QuestionType+{+ /// <summary>Pick one label from a set of named options.</summary>+ Choice = 0,++ /// <summary>Place the state on an ordered scale; the answer is an expected level.</summary>+ Score = 1,++ /// <summary>Does the statement hold? The answer is a probability.</summary>+ Noul = 2,+}++/// <summary>+/// One typed question. Construct with <see cref="Choice(object, ValueTuple{string, object}[])"/>,+/// <see cref="Score(object, object[])"/> or <see cref="Noul(object, object, object)"/>.+/// </summary>+/// <remarks>+/// <para>+/// <c>Instructions</c> is <see cref="object"/> rather than <see cref="string"/> on purpose: Python+/// accepts a non-string and runs it through <c>json.dumps</c>, which escapes non-ASCII, so the+/// rendered prompt differs from the pass-through a string gets. Both paths are reproduced.+/// </para>+/// <para>+/// Criteria order is load-bearing. Python reads choice labels positionally+/// (<c>keys = list(q["crit"].keys())</c>), so the Nth option's probability is attached to the Nth+/// label. Every criteria collection here therefore preserves insertion order.+/// </para>+/// </remarks>+public abstract class Question+{+ internal Question(QuestionType type, object instructions)+ {+ ArgumentNullException.ThrowIfNull(instructions);+ Type = type;+ Instructions = instructions;+ }++ /// <summary>Which of the three question types this is.</summary>+ public QuestionType Type { get; }++ /// <summary>+ /// The prompt. A <see cref="string"/> is used verbatim; anything else is serialized with+ /// <see cref="PythonJson.Instructions"/>, matching Python's <c>_to_internal</c>.+ /// </summary>+ public object Instructions { get; }++ /// <summary>A choice question over ordered <c>(label, description)</c> options.</summary>+ /// <remarks>A null or empty description means "no description", exactly as in Python; note+ /// that <c>0</c> and <c>false</c> are real descriptions and are rendered.</remarks>+ public static ChoiceQuestion Choice(object instructions, params (string Label, object? Description)[] options)+ => new(instructions, options.Select(o => new KeyValuePair<string, object?>(o.Label, o.Description)));++ /// <summary>A choice question over bare labels, with no descriptions.</summary>+ public static ChoiceQuestion Choice(object instructions, params string[] labels)+ => new(instructions, labels.Select(l => new KeyValuePair<string, object?>(l, null)));++ /// <summary>A choice question from any ordered label/description sequence.</summary>+ public static ChoiceQuestion Choice(object instructions, IEnumerable<KeyValuePair<string, object?>> options)+ => new(instructions, options);++ /// <summary>A score question over ordered levels, lowest first.</summary>+ public static ScoreQuestion Score(object instructions, params object?[] levels)+ => new(instructions, levels);++ /// <summary>A score question over ordered levels, lowest first.</summary>+ public static ScoreQuestion Score(object instructions, IEnumerable<object?> levels)+ => new(instructions, levels);++ /// <summary>+ /// A yes/no question. Supplying <paramref name="ifFalse"/> / <paramref name="ifTrue"/> replaces+ /// the default "no, the statement does not hold" / "yes, the statement holds" wording.+ /// </summary>+ public static NoulQuestion Noul(object instructions, object? ifFalse = null, object? ifTrue = null)+ => new(instructions, ifFalse, ifTrue);+}++/// <summary>A <see cref="QuestionType.Choice"/> question: ordered, named options.</summary>+public sealed class ChoiceQuestion : Question+{+ internal ChoiceQuestion(object instructions, IEnumerable<KeyValuePair<string, object?>> options)+ : base(QuestionType.Choice, instructions)+ {+ ArgumentNullException.ThrowIfNull(options);+ var list = new List<KeyValuePair<string, object?>>();+ var seen = new HashSet<string>(StringComparer.Ordinal);+ foreach (var o in options)+ {+ ArgumentNullException.ThrowIfNull(o.Key, nameof(options));+ // Python would silently collapse duplicate keys into one option, changing the option+ // count and so every marker position. Refuse instead of answering a different question.+ if (!seen.Add(o.Key))+ throw new ArgumentException($"duplicate choice label '{o.Key}'", nameof(options));+ list.Add(o);+ }+ if (list.Count == 0)+ throw new ArgumentException("a choice question needs at least one option", nameof(options));+ Options = list;+ }++ /// <summary>The options in label order. Index N here is the label for logit N.</summary>+ public IReadOnlyList<KeyValuePair<string, object?>> Options { get; }++ /// <summary>The labels, in order.</summary>+ public IReadOnlyList<string> Labels => Options.Select(o => o.Key).ToList();+}++/// <summary>A <see cref="QuestionType.Score"/> question: an ordered scale, lowest level first.</summary>+public sealed class ScoreQuestion : Question+{+ internal ScoreQuestion(object instructions, IEnumerable<object?> levels)+ : base(QuestionType.Score, instructions)+ {+ ArgumentNullException.ThrowIfNull(levels);+ Levels = levels.ToList();+ if (Levels.Count == 0)+ throw new ArgumentException("a score question needs at least one level", nameof(levels));+ }++ /// <summary>The levels, lowest first. The answer is an expectation over these indices.</summary>+ public IReadOnlyList<object?> Levels { get; }+}++/// <summary>A <see cref="QuestionType.Noul"/> question: does the statement hold?</summary>+public sealed class NoulQuestion : Question+{+ internal NoulQuestion(object instructions, object? ifFalse, object? ifTrue)+ : base(QuestionType.Noul, instructions)+ {+ IfFalse = ifFalse;+ IfTrue = ifTrue;+ }++ /// <summary>Wording for the false option, or null for the default.</summary>+ public object? IfFalse { get; }++ /// <summary>Wording for the true option, or null for the default.</summary>+ public object? IfTrue { get; }+}++/// <summary>+/// An insertion-ordered set of question id to <see cref="Question"/>, answered in one batch.+/// </summary>+/// <remarks>+/// Deliberately not a <see cref="Dictionary{TKey,TValue}"/>: the batch row order must match the+/// order the caller wrote the questions in, because that is what Python's <c>dict</c> guarantees+/// and what the answer order is compared against.+/// </remarks>+public sealed class QuestionSet : IEnumerable<KeyValuePair<string, Question>>+{+ private readonly List<string> _order = [];+ private readonly Dictionary<string, Question> _byId = new(StringComparer.Ordinal);++ /// <summary>An empty set.</summary>+ public QuestionSet() { }++ /// <summary>A set seeded from an ordered sequence.</summary>+ public QuestionSet(IEnumerable<KeyValuePair<string, Question>> questions)+ {+ ArgumentNullException.ThrowIfNull(questions);+ foreach (var (id, q) in questions) this[id] = q;+ }++ /// <summary>How many questions are in the set.</summary>+ public int Count => _order.Count;++ /// <summary>The question ids, in insertion order.</summary>+ public IReadOnlyList<string> Ids => _order;++ /// <summary>Get or add a question by id. Re-assigning an existing id keeps its position.</summary>+ public Question this[string id]+ {+ get => _byId[id];+ set+ {+ ArgumentException.ThrowIfNullOrEmpty(id);+ ArgumentNullException.ThrowIfNull(value);+ if (!_byId.ContainsKey(id)) _order.Add(id);+ _byId[id] = value;+ }+ }++ /// <summary>Append a question. Throws if the id is already present.</summary>+ public void Add(string id, Question question)+ {+ ArgumentException.ThrowIfNullOrEmpty(id);+ ArgumentNullException.ThrowIfNull(question);+ if (!_byId.TryAdd(id, question))+ throw new ArgumentException($"duplicate question id '{id}'", nameof(id));+ _order.Add(id);+ }++ /// <summary>Whether the set contains this id.</summary>+ public bool ContainsId(string id) => _byId.ContainsKey(id);++ /// <inheritdoc/>+ public IEnumerator<KeyValuePair<string, Question>> GetEnumerator()+ {+ foreach (var id in _order) yield return new(id, _byId[id]);+ }++ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();+}diff --git a/laya-dotnet/src/Laya/SequenceBuilder.cs b/laya-dotnet/src/Laya/SequenceBuilder.csnew file mode 100644index 00000000..1c551d6d--- /dev/null+++ b/laya-dotnet/src/Laya/SequenceBuilder.cs@@ -0,0 +1,181 @@+using Laya.Tokenization;++namespace Laya;++/// <summary>+/// Builds the typed token sequence one question is answered from. Ports <c>build_sequence</c>,+/// <c>render_options</c> and <c>render_criterion</c> from <c>laya/common.py</c>.+/// </summary>+/// <remarks>+/// The layout is+/// <c>[CLS] <type> question: instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]</c>.+/// The scorer reads the hidden state at each <c>[MASK]</c>, so a marker position that is off by one+/// token attaches an option's probability to the wrong place in the sequence and the model answers+/// a different question without anything failing. Every offset here is therefore load-bearing.+/// </remarks>+public static class SequenceBuilder+{+ /// <summary>One option fragment is capped at this many tokens, before the marker is prepended.</summary>+ private const int MaxOptionTokens = 48;++ /// <summary>The head is never squeezed below this, even when the options have eaten the budget.</summary>+ private const int MinHeadTokens = 8;++ /// <summary>The Python type name, which appears verbatim in the prompt text.</summary>+ public static string TypeName(QuestionType type) => type switch+ {+ QuestionType.Choice => "choice",+ QuestionType.Score => "score",+ QuestionType.Noul => "noul",+ _ => throw new ArgumentOutOfRangeException(nameof(type), type, "unknown question type"),+ };++ /// <summary>+ /// The instruction text as it reaches the tokenizer: a string verbatim, anything else through+ /// the <c>json.dumps</c> dialect that escapes non-ASCII. Ports <c>Agent._to_internal</c>.+ /// </summary>+ public static string RenderInstructions(Question question)+ {+ ArgumentNullException.ThrowIfNull(question);+ return question.Instructions as string ?? PythonJson.Instructions(question.Instructions);+ }++ /// <summary>+ /// The option texts in label-index order. Index N is the option the Nth logit scores, so this+ /// order is what binds a probability to a label.+ /// </summary>+ public static IReadOnlyList<string> RenderOptions(Question question)+ {+ ArgumentNullException.ThrowIfNull(question);+ switch (question)+ {+ case ChoiceQuestion choice:+ var rendered = new List<string>(choice.Options.Count);+ foreach (var (label, description) in choice.Options)+ rendered.Add(IsBlankCriterion(description)+ ? label+ : $"{label}: {PythonJson.Criterion(description)}");+ return rendered;++ case ScoreQuestion score:+ var levels = new List<string>(score.Levels.Count);+ for (var i = 0; i < score.Levels.Count; i++)+ levels.Add($"level {i}: {PythonJson.Criterion(score.Levels[i])}");+ return levels;++ case NoulQuestion noul:+ return+ [+ "false: " + (IsBlankCriterion(noul.IfFalse)+ ? "no, the statement does not hold"+ : PythonJson.Criterion(noul.IfFalse)),+ "true: " + (IsBlankCriterion(noul.IfTrue)+ ? "yes, the statement holds"+ : PythonJson.Criterion(noul.IfTrue)),+ ];++ default:+ throw new ArgumentException($"unknown question kind {question.GetType().Name}", nameof(question));+ }+ }++ /// <summary>+ /// Whether a criterion value means "no description". Python tests <c>v is None or v == ""</c>,+ /// so only null and the empty string qualify: <c>0</c> and <c>false</c> are real descriptions+ /// and get rendered.+ /// </summary>+ private static bool IsBlankCriterion(object? value) => value is null || (value is string s && s.Length == 0);++ /// <summary>Build the token sequence and marker positions for one question.</summary>+ /// <param name="tokenizer">Must encode without special tokens.</param>+ /// <param name="state">The text, object or turn list being judged.</param>+ /// <param name="question">The question to build for.</param>+ /// <param name="maxLen">Total sequence budget.</param>+ /// <param name="headMaxLen">Budget for instructions plus option fragments.</param>+ /// <param name="truncateLeft">Keep the end of an over-long state rather than its start.</param>+ /// <returns>+ /// The token ids, and one marker position per option. Markers landing at or past+ /// <paramref name="maxLen"/> are dropped, so a short marker list means options did not fit —+ /// the caller must treat that as an error rather than answering a truncated question.+ /// </returns>+ public static (int[] Ids, int[] Markers) Build(+ ILayaTokenizer tokenizer, object? state, Question question,+ int maxLen, int headMaxLen, bool truncateLeft = false)+ {+ ArgumentNullException.ThrowIfNull(tokenizer);+ ArgumentNullException.ThrowIfNull(question);++ var mask = tokenizer.MaskToken;+ var options = RenderOptions(question);++ // A literal mask token anywhere in user input would forge a marker the scorer then reads,+ // so every fragment has it replaced with a space before encoding.+ var instructions = Scrub(RenderInstructions(question), mask);+ var headIds = tokenizer.Encode($"{TypeName(question.Type)} question: {instructions}").ToList();++ var optionIds = new List<List<int>>(options.Count);+ foreach (var option in options)+ {+ var encoded = tokenizer.Encode(" " + Scrub(option, mask));+ var take = Math.Min(MaxOptionTokens, encoded.Length);+ var fragment = new List<int>(take + 1) { tokenizer.MaskId };+ for (var i = 0; i < take; i++) fragment.Add(encoded[i]);+ optionIds.Add(fragment);+ }++ var optBudget = headMaxLen - optionIds.Sum(o => o.Count);+ if (optBudget < 16)+ {+ var per = Math.Max(4, FloorDiv(headMaxLen - 16, Math.Max(1, optionIds.Count)));+ for (var i = 0; i < optionIds.Count; i++)+ if (optionIds[i].Count > per)+ optionIds[i] = optionIds[i][..per];+ optBudget = headMaxLen - optionIds.Sum(o => o.Count);+ }++ var headTake = Math.Max(MinHeadTokens, optBudget);+ if (headIds.Count > headTake) headIds = headIds[..headTake];++ var ids = new List<int>(maxLen) { tokenizer.ClsId };+ ids.AddRange(headIds);+ ids.Add(tokenizer.SepId);++ var markers = new List<int>(optionIds.Count);+ foreach (var fragment in optionIds)+ {+ markers.Add(ids.Count);+ ids.AddRange(fragment);+ }+ ids.Add(tokenizer.SepId);++ var room = Math.Max(0, maxLen - ids.Count - 1);+ var stateIds = tokenizer.Encode(Scrub(PythonJson.State(state), mask));+ ids.AddRange(SliceState(stateIds, room, truncateLeft));+ ids.Add(tokenizer.SepId);++ if (ids.Count > maxLen) ids = ids[..maxLen];+ markers.RemoveAll(m => m >= maxLen);+ return (ids.ToArray(), markers.ToArray());+ }++ private static string Scrub(string text, string maskToken) =>+ text.Replace(maskToken, " ", StringComparison.Ordinal);++ private static IEnumerable<int> SliceState(int[] stateIds, int room, bool truncateLeft)+ {+ if (!truncateLeft) return stateIds.Take(room);+ // Matches 0.3.21's `state_ids[max(0, len(state_ids) - room):]`. 0.3.6 wrote `st[-room:]`,+ // which for room == 0 is Python's `st[0:]` and so kept the whole state instead of none of+ // it; `Skip(Math.Max(0, len - room))` has no such quirk and correctly returns empty when+ // room == 0. `Agent._encode_state` sets `truncate_left = isinstance(state, list)`, so this+ // path is reachable whenever a caller's state is a list of conversation turns.+ return stateIds.Skip(Math.Max(0, stateIds.Length - room));+ }++ /// <summary>Python's <c>//</c>, which floors rather than truncating toward zero.</summary>+ private static int FloorDiv(int a, int b)+ {+ var q = a / b;+ return a % b != 0 && (a < 0) != (b < 0) ? q - 1 : q;+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/CollatorTests.cs b/laya-dotnet/tests/Laya.Tests/CollatorTests.csnew file mode 100644index 00000000..14f2b6f7--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/CollatorTests.cs@@ -0,0 +1,130 @@+namespace Laya.Tests;++/// <summary>Tier 1: batch padding, which needs no artifacts.</summary>+public sealed class CollatorTests+{+ private static SequenceItem Item(int length, int markers, QuestionType type = QuestionType.Choice) =>+ new([.. Enumerable.Range(10, length)], [.. Enumerable.Range(1, markers)], type);++ [Fact]+ public void PadsToTheLongestRowAndMarksAttentionAccordingly()+ {+ var batch = Collator.Collate([Item(5, 2), Item(3, 2)], padId: 0);++ Assert.Equal(2, batch.Count);+ Assert.Equal(5, batch.SequenceLength);+ Assert.Equal([1, 1, 1, 1, 1, 1, 1, 1, 0, 0], batch.AttentionMask);+ // The short row's tail is the pad id, and attention is 0 there.+ Assert.Equal(0, batch.InputIds[8]);+ Assert.Equal(0, batch.InputIds[9]);+ }++ [Fact]+ public void FillsWithTheGivenPadIdRatherThanZero()+ {+ var batch = Collator.Collate([Item(3, 2), Item(1, 2)], padId: 77);+ Assert.Equal(77, batch.InputIds[4]);+ Assert.Equal(77, batch.InputIds[5]);+ Assert.Equal(0, batch.AttentionMask[4]);+ }++ [Fact]+ public void CountsOnlyRealTokensAsInputTokens()+ {+ var batch = Collator.Collate([Item(5, 2), Item(3, 2)], padId: 0);+ Assert.Equal(8, batch.InputTokens);+ }++ [Fact]+ public void WidensTheMarkerAxisToTwoForSingleOptionBatches()+ {+ // The exported graph has TopK k=2 baked in: a one-column marker axis makes it fail outright.+ var batch = Collator.Collate([Item(6, 1)], padId: 0);++ Assert.Equal(Collator.MinMarkers, batch.MarkerCount);+ Assert.Equal([1, 0], batch.MarkerPos);+ Assert.Equal([true, false], batch.MarkerMask);+ // The real option count is kept separately, so the pad column never reaches an answer.+ Assert.Equal([1], batch.MarkerCounts);+ }++ [Fact]+ public void PadsTheMarkerAxisToTheWidestQuestion()+ {+ var batch = Collator.Collate([Item(6, 4), Item(6, 2)], padId: 0);++ Assert.Equal(4, batch.MarkerCount);+ Assert.Equal([1, 2, 3, 4, 1, 2, 0, 0], batch.MarkerPos);+ Assert.Equal([true, true, true, true, true, true, false, false], batch.MarkerMask);+ Assert.Equal([4, 2], batch.MarkerCounts);+ }++ [Fact]+ public void CarriesTheQuestionTypePerRowUsingPythonsNumbering()+ {+ var batch = Collator.Collate(+ [Item(4, 2, QuestionType.Noul), Item(4, 2, QuestionType.Choice), Item(4, 2, QuestionType.Score)],+ padId: 0);++ Assert.Equal([2, 0, 1], batch.QType);+ }++ [Fact]+ public void RefusesAnEmptyBatch() =>+ Assert.Throws<ArgumentException>(() => Collator.Collate([], padId: 0));++ [Theory]+ [MemberData(nameof(GoldenData.SuccessCases), MemberType = typeof(GoldenData))]+ public void ReproducesTheRecordedBatchTensors(GoldenCaseInfo info)+ {+ using var doc = GoldenData.Load(info);+ var root = doc.RootElement;+ var expected = root.GetProperty("batch");++ // Rebuild the batch from the recorded per-question ids, so this checks the collator alone+ // and stays meaningful without a tokenizer present.+ var items = new List<SequenceItem>();+ foreach (var q in root.GetProperty("per_question").EnumerateArray())+ items.Add(new SequenceItem(+ GoldenData.Ints(q.GetProperty("input_ids")),+ GoldenData.Ints(q.GetProperty("markers")),+ (QuestionType)q.GetProperty("qtype").GetInt32()));++ var batch = Collator.Collate(items, padId: GoldenData.Meta.GetProperty("special_tokens")+ .GetProperty("pad_id").GetInt32());++ AssertMatrix(expected.GetProperty("input_ids"), batch.InputIds, batch.SequenceLength);+ AssertMatrix(expected.GetProperty("attention_mask"), batch.AttentionMask, batch.SequenceLength);+ AssertMatrix(expected.GetProperty("marker_pos"), batch.MarkerPos, batch.MarkerCount);+ Assert.Equal([.. expected.GetProperty("qtype").EnumerateArray().Select(e => e.GetInt64())], batch.QType);++ var expectedMask = expected.GetProperty("marker_mask");+ Assert.Equal(expectedMask.GetArrayLength(), batch.Count);+ for (var r = 0; r < batch.Count; r++)+ {+ var row = expectedMask[r];+ for (var c = 0; c < batch.MarkerCount; c++)+ Assert.Equal(row[c].GetBoolean(), batch.MarkerMask[r * batch.MarkerCount + c]);+ }++ Assert.Equal(root.GetProperty("result").GetProperty("usage").GetProperty("input_tokens").GetInt32(),+ batch.InputTokens);+ // marker_pos above already pinned the padded width; this checks the flag the dump script+ // set, so a future golden that stops exercising the TopK edge case is noticed.+ Assert.Equal(root.GetProperty("marker_axis_padded").GetBoolean(),+ batch.MarkerCount > items.Max(i => i.Markers.Length));+ }++ private static void AssertMatrix(System.Text.Json.JsonElement expected, long[] actual, int cols)+ {+ var rows = expected.GetArrayLength();+ Assert.Equal(rows * cols, actual.Length);+ for (var r = 0; r < rows; r++)+ {+ var row = expected[r];+ Assert.Equal(cols, row.GetArrayLength());+ for (var c = 0; c < cols; c++)+ Assert.Equal(row[c].GetInt64(), actual[r * cols + c]);+ }+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.cs b/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.csnew file mode 100644index 00000000..85214317--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.cs@@ -0,0 +1,253 @@+namespace Laya.Tests;++/// <summary>+/// Tier 1: sequence layout and the option budget, using <see cref="StubTokenizer"/> so no artifacts+/// are needed. Real token ids are checked in <see cref="TokenizerParityTests"/>.+/// </summary>+public sealed class SequenceBuilderTests+{+ private readonly StubTokenizer _tok = new();++ [Fact]+ public void LaysOutClsHeadSepOptionsSepStateSep()+ {+ var q = Question.Choice("pick one", ("a", null), ("b", null));+ var (ids, markers) = SequenceBuilder.Build(_tok, "state text", q, maxLen: 64, headMaxLen: 32);++ Assert.Equal(_tok.ClsId, ids[0]);+ Assert.Equal(_tok.SepId, ids[^1]);+ // Each marker is a mask token, and the two options are adjacent fragments.+ Assert.Equal(2, markers.Length);+ Assert.All(markers, m => Assert.Equal(_tok.MaskId, ids[m]));+ Assert.True(markers[1] > markers[0]);+ // A SEP closes the head, immediately before the first marker.+ Assert.Equal(_tok.SepId, ids[markers[0] - 1]);+ }++ [Fact]+ public void MarkerPositionsPointAtConsecutiveOptionFragments()+ {+ // One word per option keeps the arithmetic checkable: mask + 1 word = 2 ids per fragment.+ var q = Question.Choice("h", ("alpha", null), ("beta", null), ("gamma", null));+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 64, headMaxLen: 32);++ Assert.Equal(3, markers.Length);+ Assert.Equal(markers[0] + 2, markers[1]);+ Assert.Equal(markers[1] + 2, markers[2]);+ Assert.Equal(_tok.SepId, ids[markers[2] + 2]);+ }++ [Fact]+ public void PutsTheTypeNameInTheHeadText()+ {+ SequenceBuilder.Build(_tok, "s", Question.Noul("does it hold"), maxLen: 64, headMaxLen: 32);+ Assert.Equal("noul question: does it hold", _tok.Encoded[0]);+ }++ [Theory]+ [InlineData(QuestionType.Choice, "choice")]+ [InlineData(QuestionType.Score, "score")]+ [InlineData(QuestionType.Noul, "noul")]+ public void TypeNamesMatchPython(QuestionType type, string expected) =>+ Assert.Equal(expected, SequenceBuilder.TypeName(type));++ [Fact]+ public void OptionFragmentsAreEncodedWithALeadingSpace()+ {+ SequenceBuilder.Build(_tok, "s", Question.Choice("h", ("a", null)), maxLen: 64, headMaxLen: 32);+ // Python encodes " " + option, which the Metaspace pre-tokenizer turns into a word boundary.+ Assert.Contains(" a", _tok.Encoded);+ }++ [Fact]+ public void RendersOptionsTheWayPythonDoes()+ {+ Assert.Equal(["a: first", "b"],+ SequenceBuilder.RenderOptions(Question.Choice("h", ("a", "first"), ("b", null))));++ Assert.Equal(["level 0: low", "level 1: high"],+ SequenceBuilder.RenderOptions(Question.Score("h", "low", "high")));++ Assert.Equal(["false: no, the statement does not hold", "true: yes, the statement holds"],+ SequenceBuilder.RenderOptions(Question.Noul("h")));++ Assert.Equal(["false: all fine", "true: on fire"],+ SequenceBuilder.RenderOptions(Question.Noul("h", ifFalse: "all fine", ifTrue: "on fire")));+ }++ [Fact]+ public void TreatsOnlyNullAndEmptyStringAsAMissingDescription()+ {+ // Python tests `v is None or v == ""`, so 0 and false are real descriptions.+ Assert.Equal(["zero: 0", "no: false", "blank", "none"],+ SequenceBuilder.RenderOptions(Question.Choice("h",+ ("zero", 0L), ("no", false), ("blank", ""), ("none", null))));+ }++ [Fact]+ public void ScrubsAMaskTokenOutOfEveryFragment()+ {+ // A literal <mask> in user input would forge a marker the scorer then reads as an option.+ var q = Question.Choice($"is this {_tok.MaskToken} safe", ($"a{_tok.MaskToken}b", null));+ SequenceBuilder.Build(_tok, $"state {_tok.MaskToken} here", q, maxLen: 64, headMaxLen: 32);++ Assert.All(_tok.Encoded, text => Assert.DoesNotContain(_tok.MaskToken, text));+ }++ [Fact]+ public void TruncatesLongOptionsToFortyEightTokens()+ {+ var words = string.Join(' ', Enumerable.Range(0, 120).Select(i => $"w{i}"));+ var q = Question.Choice("h", ("opt", words));+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 512, headMaxLen: 256);++ // The only fragment is mask + at most 48 option tokens, then the closing SEP. "opt: " adds+ // two words, so 48 is the cap that bites rather than the option running out.+ Assert.Single(markers);+ var fragmentEnd = Array.IndexOf(ids, _tok.SepId, markers[0]);+ Assert.Equal(markers[0] + 49, fragmentEnd);+ }++ [Fact]+ public void ShrinksOptionsWhenTheyWouldEatTheWholeHeadBudget()+ {+ // 20 options of ~10 tokens each against a 96-token head budget forces the+ // `opt_budget < 16` branch, which caps every fragment at `(head_max_len - 16) // n`.+ var options = Enumerable.Range(0, 20)+ .Select(i => ($"opt{i}", (object?)string.Join(' ', Enumerable.Range(0, 10).Select(j => $"d{i}_{j}"))))+ .ToArray();+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", Question.Choice("head words here", options),+ maxLen: 512, headMaxLen: 96);++ Assert.Equal(20, markers.Length);+ var per = Math.Max(4, (96 - 16) / 20); // 4+ for (var i = 1; i < markers.Length; i++)+ Assert.True(markers[i] - markers[i - 1] <= per,+ $"fragment {i - 1} is {markers[i] - markers[i - 1]} tokens, above the {per}-token cap");+ Assert.Equal(_tok.ClsId, ids[0]);+ }++ [Fact]+ public void KeepsAtLeastEightHeadTokensWhenOptionsDominate()+ {+ var options = Enumerable.Range(0, 30)+ .Select(i => ($"o{i}", (object?)"a b c d e f g h"))+ .ToArray();+ var (ids, markers) = SequenceBuilder.Build(_tok, "s",+ Question.Choice("one two three four five six seven eight nine ten", options),+ maxLen: 512, headMaxLen: 64);++ // CLS + head + SEP, so the first marker cannot be earlier than position 2.+ Assert.True(markers[0] >= 2);+ Assert.Equal(_tok.SepId, ids[markers[0] - 1]);+ }++ [Fact]+ public void TruncatesTheStateToFitMaxLen()+ {+ var state = string.Join(' ', Enumerable.Range(0, 500).Select(i => $"s{i}"));+ var (ids, markers) = SequenceBuilder.Build(_tok, state, Question.Noul("h"), maxLen: 64, headMaxLen: 32);++ Assert.Equal(64, ids.Length);+ Assert.Equal(_tok.SepId, ids[^1]);+ Assert.Equal(2, markers.Length);+ }++ [Fact]+ public void KeepsTheStateTailWhenTruncatingLeft()+ {+ var state = string.Join(' ', Enumerable.Range(0, 200).Select(i => $"s{i}"));+ var head = SequenceBuilder.Build(_tok, state, Question.Noul("h"), 64, 32, truncateLeft: false);+ var tail = SequenceBuilder.Build(_tok, state, Question.Noul("h"), 64, 32, truncateLeft: true);++ Assert.Equal(head.Ids.Length, tail.Ids.Length);+ Assert.NotEqual(head.Ids, tail.Ids);+ // Both keep the same prefix up to the state, and differ from there on.+ Assert.Equal(head.Markers, tail.Markers);+ }++ [Fact]+ public void LeftTruncationWithNoRoomDropsTheWholeStateRatherThanKeepingIt()+ {+ // 0.3.21's `state_ids[max(0, len - room):]` correctly returns empty when room == 0.+ // 0.3.6's `st[-room:]` had a quirk here (Python's `x[-0:]` is `x[0:]`, the whole list) —+ // reproduced faithfully until now, since `truncate_left` was previously unreachable from+ // Predict. Isolate the "no room left" boundary without hardcoding tokenizer counts: probe+ // the exact CLS+head+options+SEP prefix length with a one-token state, then rebuild at+ // exactly that maxLen (room == 0) with a long state.+ var q = Question.Noul("h");+ var probe = SequenceBuilder.Build(_tok, "x", q, maxLen: 10_000, headMaxLen: 32);+ var prefixLen = probe.Ids.Length - 2; // minus the one state token and the closing SEP++ var longState = string.Join(' ', Enumerable.Range(0, 50).Select(i => $"s{i}"));+ var (ids, _) = SequenceBuilder.Build(_tok, longState, q,+ maxLen: prefixLen + 1, headMaxLen: 32, truncateLeft: true);++ // Fixed: no state tokens fit, so the sequence ends prefix + SEP, exactly at maxLen.+ // The old bug would instead have appended the whole 50-word state before the final+ // maxLen clamp cut it back down to size, landing a state token (not SEP) at this position.+ Assert.Equal(prefixLen + 1, ids.Length);+ Assert.Equal(_tok.SepId, ids[^1]);+ }++ [Theory]+ [InlineData(true)]+ [InlineData(false)]+ public void ListStatesAreDetectedForAutomaticLeftTruncation(bool expectListLike)+ {+ // Mirrors Agent._encode_state: `truncate_left = isinstance(state, list)`. LayaEngine.Predict+ // derives its truncateLeft argument from PythonJson.IsListState, so this is the source of+ // truth callers rely on for "is this state list-shaped conversation turns".+ object state = expectListLike+ ? new List<object?> { "turn 1", "turn 2" }+ : "a plain string state";++ Assert.Equal(expectListLike, PythonJson.IsListState(state));+ }++ [Fact]+ public void DictAndStringStatesAreNotListLike()+ {+ Assert.False(PythonJson.IsListState("a string"));+ Assert.False(PythonJson.IsListState(new Dictionary<string, object?> { ["k"] = "v" }));+ Assert.False(PythonJson.IsListState(null));+ Assert.True(PythonJson.IsListState(new object?[] { "a", "b" }));+ }++ [Fact]+ public void DropsMarkersThatWouldLandPastMaxLen()+ {+ // 40 options cannot fit a 32-token sequence, so markers are lost. Predict must treat a short+ // marker list as an error rather than answering a question missing options.+ var options = Enumerable.Range(0, 40).Select(i => ($"o{i}", (object?)"desc")).ToArray();+ var q = Question.Choice("h", options);+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 32, headMaxLen: 256);++ Assert.Equal(32, ids.Length);+ Assert.True(markers.Length < options.Length);+ Assert.All(markers, m => Assert.True(m < 32));+ }++ [Fact]+ public void RendersNonStringInstructionsWithRealUnicodeNotEscapes()+ {+ // 0.3.21's `_to_internal` switched to json.dumps(ins, ensure_ascii=False): real Unicode+ // glyphs reach the tokenizer instead of literal \uXXXX escape text.+ var q = Question.Noul(new OrderedDictionary<string, object?>(StringComparer.Ordinal)+ {+ ["hint"] = "Müller",+ });+ var rendered = SequenceBuilder.RenderInstructions(q);++ Assert.Contains("ü", rendered);+ Assert.DoesNotContain("\\u", rendered);+ Assert.Equal("{\"hint\": \"Müller\"}", rendered);+ }++ [Fact]+ public void PassesStringInstructionsThroughUntouched()+ {+ var q = Question.Noul("Müller & co <urgent>");+ Assert.Equal("Müller & co <urgent>", SequenceBuilder.RenderInstructions(q));+ }+}