Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(dotnet): add typed questions, sequence builder and collator

kevin-gatimu1fa53c35c1a0
diff --git a/laya-dotnet/src/Laya/Collator.cs b/laya-dotnet/src/Laya/Collator.cs
new file mode 100644
index 00000000..7e422e98
--- /dev/null
+++ b/laya-dotnet/src/Laya/Collator.cs
@@ -0,0 +1,121 @@
+namespace Laya;
+
+/// <summary>One question, tokenized and ready to be batched.</summary>
+/// <param name="Ids">The token sequence.</param>
+/// <param name="Markers">One position per option, pointing at that option's <c>[MASK]</c>.</param>
+/// <param name="Type">Which question type, fed to the model's type embedding.</param>
+public readonly record struct SequenceItem(int[] Ids, int[] Markers, QuestionType Type);
+
+/// <summary>A padded batch, laid out as the row-major tensors the ONNX graph takes.</summary>
+public sealed class CollatedBatch
+{
+ internal CollatedBatch(int count, int sequenceLength, int markerCount,
+ long[] inputIds, long[] attentionMask, long[] markerPos,
+ bool[] markerMask, long[] qtype, int[] markerCounts, int inputTokens)
+ {
+ Count = count;
+ SequenceLength = sequenceLength;
+ MarkerCount = markerCount;
+ InputIds = inputIds;
+ AttentionMask = attentionMask;
+ MarkerPos = markerPos;
+ MarkerMask = markerMask;
+ QType = qtype;
+ MarkerCounts = markerCounts;
+ InputTokens = inputTokens;
+ }
+
+ /// <summary>Rows in the batch: one per question.</summary>
+ public int Count { get; }
+
+ /// <summary>Padded sequence length, the longest row.</summary>
+ public int SequenceLength { get; }
+
+ /// <summary>Padded marker-axis width.</summary>
+ public int MarkerCount { get; }
+
+ /// <summary><c>[Count, SequenceLength]</c>, padded with the pad id.</summary>
+ public long[] InputIds { get; }
+
+ /// <summary><c>[Count, SequenceLength]</c>, 1 on real tokens and 0 on padding.</summary>
+ public long[] AttentionMask { get; }
+
+ /// <summary><c>[Count, MarkerCount]</c>, 0 in padding columns.</summary>
+ public long[] MarkerPos { get; }
+
+ /// <summary><c>[Count, MarkerCount]</c>, false in padding columns.</summary>
+ public bool[] MarkerMask { get; }
+
+ /// <summary><c>[Count]</c>, the question type per row.</summary>
+ public long[] QType { get; }
+
+ /// <summary>The real (unpadded) option count per row — how many logits to read from that row.</summary>
+ public int[] MarkerCounts { get; }
+
+ /// <summary>Non-padding tokens across the batch, which is what Python reports as input tokens.</summary>
+ public int InputTokens { get; }
+}
+
+/// <summary>Pads tokenized questions into one batch. Ports <c>collate_items</c> from <c>laya/common.py</c>.</summary>
+public static class Collator
+{
+ /// <summary>
+ /// The marker axis is never narrower than this.
+ /// </summary>
+ /// <remarks>
+ /// The tracer baked <c>TopK k=2</c> into the exported graph: Python chooses the top-2 branch at
+ /// runtime and the export only ever saw two or more markers, so a batch whose questions all
+ /// have a single option makes TopK fail outright. Padding to two columns with
+ /// <c>marker_mask = false</c> is exactly equivalent, not merely close: the model's
+ /// <c>masked_fill</c> puts <c>-1e4</c> in the pad column, which underflows softmax to 0.0, so
+ /// <c>p == [1.0, 0.0]</c> — precisely the <c>[top1, 0.0]</c> Python substitutes — and
+ /// <c>k = marker_mask.sum().clamp(min=2) == 2</c> either way. Reading only
+ /// <see cref="CollatedBatch.MarkerCounts"/> logits per row keeps the pad column out of the answer.
+ /// </remarks>
+ public const int MinMarkers = 2;
+
+ /// <summary>Pad a batch of tokenized questions.</summary>
+ public static CollatedBatch Collate(IReadOnlyList<SequenceItem> items, int padId)
+ {
+ ArgumentNullException.ThrowIfNull(items);
+ if (items.Count == 0) throw new ArgumentException("nothing to collate", nameof(items));
+
+ var n = items.Count;
+ var length = items.Max(it => it.Ids.Length);
+ var markerCount = Math.Max(MinMarkers, items.Max(it => it.Markers.Length));
+
+ var ids = new long[n * length];
+ if (padId != 0) Array.Fill(ids, padId);
+ var attention = new long[n * length];
+ var markerPos = new long[n * markerCount];
+ var markerMask = new bool[n * markerCount];
+ var qtype = new long[n];
+ var markerCounts = new int[n];
+ var inputTokens = 0;
+
+ for (var i = 0; i < n; i++)
+ {
+ var item = items[i];
+ var row = i * length;
+ for (var j = 0; j < item.Ids.Length; j++)
+ {
+ ids[row + j] = item.Ids[j];
+ attention[row + j] = 1;
+ }
+ inputTokens += item.Ids.Length;
+
+ var mrow = i * markerCount;
+ for (var j = 0; j < item.Markers.Length; j++)
+ {
+ markerPos[mrow + j] = item.Markers[j];
+ markerMask[mrow + j] = true;
+ }
+
+ qtype[i] = (long)item.Type;
+ markerCounts[i] = item.Markers.Length;
+ }
+
+ return new CollatedBatch(n, length, markerCount, ids, attention, markerPos, markerMask,
+ qtype, markerCounts, inputTokens);
+ }
+}
diff --git a/laya-dotnet/src/Laya/Questions.cs b/laya-dotnet/src/Laya/Questions.cs
new file mode 100644
index 00000000..a6e99a6c
--- /dev/null
+++ b/laya-dotnet/src/Laya/Questions.cs
@@ -0,0 +1,205 @@
+namespace Laya;
+
+/// <summary>The three question types Laya answers. Values match Python's <c>QTYPES</c>.</summary>
+public enum QuestionType
+{
+ /// <summary>Pick one label from a set of named options.</summary>
+ Choice = 0,
+
+ /// <summary>Place the state on an ordered scale; the answer is an expected level.</summary>
+ Score = 1,
+
+ /// <summary>Does the statement hold? The answer is a probability.</summary>
+ Noul = 2,
+}
+
+/// <summary>
+/// One typed question. Construct with <see cref="Choice(object, ValueTuple{string, object}[])"/>,
+/// <see cref="Score(object, object[])"/> or <see cref="Noul(object, object, object)"/>.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <c>Instructions</c> is <see cref="object"/> rather than <see cref="string"/> on purpose: Python
+/// accepts a non-string and runs it through <c>json.dumps</c>, which escapes non-ASCII, so the
+/// rendered prompt differs from the pass-through a string gets. Both paths are reproduced.
+/// </para>
+/// <para>
+/// Criteria order is load-bearing. Python reads choice labels positionally
+/// (<c>keys = list(q["crit"].keys())</c>), so the Nth option's probability is attached to the Nth
+/// label. Every criteria collection here therefore preserves insertion order.
+/// </para>
+/// </remarks>
+public abstract class Question
+{
+ internal Question(QuestionType type, object instructions)
+ {
+ ArgumentNullException.ThrowIfNull(instructions);
+ Type = type;
+ Instructions = instructions;
+ }
+
+ /// <summary>Which of the three question types this is.</summary>
+ public QuestionType Type { get; }
+
+ /// <summary>
+ /// The prompt. A <see cref="string"/> is used verbatim; anything else is serialized with
+ /// <see cref="PythonJson.Instructions"/>, matching Python's <c>_to_internal</c>.
+ /// </summary>
+ public object Instructions { get; }
+
+ /// <summary>A choice question over ordered <c>(label, description)</c> options.</summary>
+ /// <remarks>A null or empty description means "no description", exactly as in Python; note
+ /// that <c>0</c> and <c>false</c> are real descriptions and are rendered.</remarks>
+ public static ChoiceQuestion Choice(object instructions, params (string Label, object? Description)[] options)
+ => new(instructions, options.Select(o => new KeyValuePair<string, object?>(o.Label, o.Description)));
+
+ /// <summary>A choice question over bare labels, with no descriptions.</summary>
+ public static ChoiceQuestion Choice(object instructions, params string[] labels)
+ => new(instructions, labels.Select(l => new KeyValuePair<string, object?>(l, null)));
+
+ /// <summary>A choice question from any ordered label/description sequence.</summary>
+ public static ChoiceQuestion Choice(object instructions, IEnumerable<KeyValuePair<string, object?>> options)
+ => new(instructions, options);
+
+ /// <summary>A score question over ordered levels, lowest first.</summary>
+ public static ScoreQuestion Score(object instructions, params object?[] levels)
+ => new(instructions, levels);
+
+ /// <summary>A score question over ordered levels, lowest first.</summary>
+ public static ScoreQuestion Score(object instructions, IEnumerable<object?> levels)
+ => new(instructions, levels);
+
+ /// <summary>
+ /// A yes/no question. Supplying <paramref name="ifFalse"/> / <paramref name="ifTrue"/> replaces
+ /// the default "no, the statement does not hold" / "yes, the statement holds" wording.
+ /// </summary>
+ public static NoulQuestion Noul(object instructions, object? ifFalse = null, object? ifTrue = null)
+ => new(instructions, ifFalse, ifTrue);
+}
+
+/// <summary>A <see cref="QuestionType.Choice"/> question: ordered, named options.</summary>
+public sealed class ChoiceQuestion : Question
+{
+ internal ChoiceQuestion(object instructions, IEnumerable<KeyValuePair<string, object?>> options)
+ : base(QuestionType.Choice, instructions)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ var list = new List<KeyValuePair<string, object?>>();
+ var seen = new HashSet<string>(StringComparer.Ordinal);
+ foreach (var o in options)
+ {
+ ArgumentNullException.ThrowIfNull(o.Key, nameof(options));
+ // Python would silently collapse duplicate keys into one option, changing the option
+ // count and so every marker position. Refuse instead of answering a different question.
+ if (!seen.Add(o.Key))
+ throw new ArgumentException($"duplicate choice label '{o.Key}'", nameof(options));
+ list.Add(o);
+ }
+ if (list.Count == 0)
+ throw new ArgumentException("a choice question needs at least one option", nameof(options));
+ Options = list;
+ }
+
+ /// <summary>The options in label order. Index N here is the label for logit N.</summary>
+ public IReadOnlyList<KeyValuePair<string, object?>> Options { get; }
+
+ /// <summary>The labels, in order.</summary>
+ public IReadOnlyList<string> Labels => Options.Select(o => o.Key).ToList();
+}
+
+/// <summary>A <see cref="QuestionType.Score"/> question: an ordered scale, lowest level first.</summary>
+public sealed class ScoreQuestion : Question
+{
+ internal ScoreQuestion(object instructions, IEnumerable<object?> levels)
+ : base(QuestionType.Score, instructions)
+ {
+ ArgumentNullException.ThrowIfNull(levels);
+ Levels = levels.ToList();
+ if (Levels.Count == 0)
+ throw new ArgumentException("a score question needs at least one level", nameof(levels));
+ }
+
+ /// <summary>The levels, lowest first. The answer is an expectation over these indices.</summary>
+ public IReadOnlyList<object?> Levels { get; }
+}
+
+/// <summary>A <see cref="QuestionType.Noul"/> question: does the statement hold?</summary>
+public sealed class NoulQuestion : Question
+{
+ internal NoulQuestion(object instructions, object? ifFalse, object? ifTrue)
+ : base(QuestionType.Noul, instructions)
+ {
+ IfFalse = ifFalse;
+ IfTrue = ifTrue;
+ }
+
+ /// <summary>Wording for the false option, or null for the default.</summary>
+ public object? IfFalse { get; }
+
+ /// <summary>Wording for the true option, or null for the default.</summary>
+ public object? IfTrue { get; }
+}
+
+/// <summary>
+/// An insertion-ordered set of question id to <see cref="Question"/>, answered in one batch.
+/// </summary>
+/// <remarks>
+/// Deliberately not a <see cref="Dictionary{TKey,TValue}"/>: the batch row order must match the
+/// order the caller wrote the questions in, because that is what Python's <c>dict</c> guarantees
+/// and what the answer order is compared against.
+/// </remarks>
+public sealed class QuestionSet : IEnumerable<KeyValuePair<string, Question>>
+{
+ private readonly List<string> _order = [];
+ private readonly Dictionary<string, Question> _byId = new(StringComparer.Ordinal);
+
+ /// <summary>An empty set.</summary>
+ public QuestionSet() { }
+
+ /// <summary>A set seeded from an ordered sequence.</summary>
+ public QuestionSet(IEnumerable<KeyValuePair<string, Question>> questions)
+ {
+ ArgumentNullException.ThrowIfNull(questions);
+ foreach (var (id, q) in questions) this[id] = q;
+ }
+
+ /// <summary>How many questions are in the set.</summary>
+ public int Count => _order.Count;
+
+ /// <summary>The question ids, in insertion order.</summary>
+ public IReadOnlyList<string> Ids => _order;
+
+ /// <summary>Get or add a question by id. Re-assigning an existing id keeps its position.</summary>
+ public Question this[string id]
+ {
+ get => _byId[id];
+ set
+ {
+ ArgumentException.ThrowIfNullOrEmpty(id);
+ ArgumentNullException.ThrowIfNull(value);
+ if (!_byId.ContainsKey(id)) _order.Add(id);
+ _byId[id] = value;
+ }
+ }
+
+ /// <summary>Append a question. Throws if the id is already present.</summary>
+ public void Add(string id, Question question)
+ {
+ ArgumentException.ThrowIfNullOrEmpty(id);
+ ArgumentNullException.ThrowIfNull(question);
+ if (!_byId.TryAdd(id, question))
+ throw new ArgumentException($"duplicate question id '{id}'", nameof(id));
+ _order.Add(id);
+ }
+
+ /// <summary>Whether the set contains this id.</summary>
+ public bool ContainsId(string id) => _byId.ContainsKey(id);
+
+ /// <inheritdoc/>
+ public IEnumerator<KeyValuePair<string, Question>> GetEnumerator()
+ {
+ foreach (var id in _order) yield return new(id, _byId[id]);
+ }
+
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
+}
diff --git a/laya-dotnet/src/Laya/SequenceBuilder.cs b/laya-dotnet/src/Laya/SequenceBuilder.cs
new file mode 100644
index 00000000..1c551d6d
--- /dev/null
+++ b/laya-dotnet/src/Laya/SequenceBuilder.cs
@@ -0,0 +1,181 @@
+using Laya.Tokenization;
+
+namespace Laya;
+
+/// <summary>
+/// Builds the typed token sequence one question is answered from. Ports <c>build_sequence</c>,
+/// <c>render_options</c> and <c>render_criterion</c> from <c>laya/common.py</c>.
+/// </summary>
+/// <remarks>
+/// The layout is
+/// <c>[CLS] &lt;type&gt; question: instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]</c>.
+/// The scorer reads the hidden state at each <c>[MASK]</c>, so a marker position that is off by one
+/// token attaches an option's probability to the wrong place in the sequence and the model answers
+/// a different question without anything failing. Every offset here is therefore load-bearing.
+/// </remarks>
+public static class SequenceBuilder
+{
+ /// <summary>One option fragment is capped at this many tokens, before the marker is prepended.</summary>
+ private const int MaxOptionTokens = 48;
+
+ /// <summary>The head is never squeezed below this, even when the options have eaten the budget.</summary>
+ private const int MinHeadTokens = 8;
+
+ /// <summary>The Python type name, which appears verbatim in the prompt text.</summary>
+ public static string TypeName(QuestionType type) => type switch
+ {
+ QuestionType.Choice => "choice",
+ QuestionType.Score => "score",
+ QuestionType.Noul => "noul",
+ _ => throw new ArgumentOutOfRangeException(nameof(type), type, "unknown question type"),
+ };
+
+ /// <summary>
+ /// The instruction text as it reaches the tokenizer: a string verbatim, anything else through
+ /// the <c>json.dumps</c> dialect that escapes non-ASCII. Ports <c>Agent._to_internal</c>.
+ /// </summary>
+ public static string RenderInstructions(Question question)
+ {
+ ArgumentNullException.ThrowIfNull(question);
+ return question.Instructions as string ?? PythonJson.Instructions(question.Instructions);
+ }
+
+ /// <summary>
+ /// The option texts in label-index order. Index N is the option the Nth logit scores, so this
+ /// order is what binds a probability to a label.
+ /// </summary>
+ public static IReadOnlyList<string> RenderOptions(Question question)
+ {
+ ArgumentNullException.ThrowIfNull(question);
+ switch (question)
+ {
+ case ChoiceQuestion choice:
+ var rendered = new List<string>(choice.Options.Count);
+ foreach (var (label, description) in choice.Options)
+ rendered.Add(IsBlankCriterion(description)
+ ? label
+ : $"{label}: {PythonJson.Criterion(description)}");
+ return rendered;
+
+ case ScoreQuestion score:
+ var levels = new List<string>(score.Levels.Count);
+ for (var i = 0; i < score.Levels.Count; i++)
+ levels.Add($"level {i}: {PythonJson.Criterion(score.Levels[i])}");
+ return levels;
+
+ case NoulQuestion noul:
+ return
+ [
+ "false: " + (IsBlankCriterion(noul.IfFalse)
+ ? "no, the statement does not hold"
+ : PythonJson.Criterion(noul.IfFalse)),
+ "true: " + (IsBlankCriterion(noul.IfTrue)
+ ? "yes, the statement holds"
+ : PythonJson.Criterion(noul.IfTrue)),
+ ];
+
+ default:
+ throw new ArgumentException($"unknown question kind {question.GetType().Name}", nameof(question));
+ }
+ }
+
+ /// <summary>
+ /// Whether a criterion value means "no description". Python tests <c>v is None or v == ""</c>,
+ /// so only null and the empty string qualify: <c>0</c> and <c>false</c> are real descriptions
+ /// and get rendered.
+ /// </summary>
+ private static bool IsBlankCriterion(object? value) => value is null || (value is string s && s.Length == 0);
+
+ /// <summary>Build the token sequence and marker positions for one question.</summary>
+ /// <param name="tokenizer">Must encode without special tokens.</param>
+ /// <param name="state">The text, object or turn list being judged.</param>
+ /// <param name="question">The question to build for.</param>
+ /// <param name="maxLen">Total sequence budget.</param>
+ /// <param name="headMaxLen">Budget for instructions plus option fragments.</param>
+ /// <param name="truncateLeft">Keep the end of an over-long state rather than its start.</param>
+ /// <returns>
+ /// The token ids, and one marker position per option. Markers landing at or past
+ /// <paramref name="maxLen"/> are dropped, so a short marker list means options did not fit —
+ /// the caller must treat that as an error rather than answering a truncated question.
+ /// </returns>
+ public static (int[] Ids, int[] Markers) Build(
+ ILayaTokenizer tokenizer, object? state, Question question,
+ int maxLen, int headMaxLen, bool truncateLeft = false)
+ {
+ ArgumentNullException.ThrowIfNull(tokenizer);
+ ArgumentNullException.ThrowIfNull(question);
+
+ var mask = tokenizer.MaskToken;
+ var options = RenderOptions(question);
+
+ // A literal mask token anywhere in user input would forge a marker the scorer then reads,
+ // so every fragment has it replaced with a space before encoding.
+ var instructions = Scrub(RenderInstructions(question), mask);
+ var headIds = tokenizer.Encode($"{TypeName(question.Type)} question: {instructions}").ToList();
+
+ var optionIds = new List<List<int>>(options.Count);
+ foreach (var option in options)
+ {
+ var encoded = tokenizer.Encode(" " + Scrub(option, mask));
+ var take = Math.Min(MaxOptionTokens, encoded.Length);
+ var fragment = new List<int>(take + 1) { tokenizer.MaskId };
+ for (var i = 0; i < take; i++) fragment.Add(encoded[i]);
+ optionIds.Add(fragment);
+ }
+
+ var optBudget = headMaxLen - optionIds.Sum(o => o.Count);
+ if (optBudget < 16)
+ {
+ var per = Math.Max(4, FloorDiv(headMaxLen - 16, Math.Max(1, optionIds.Count)));
+ for (var i = 0; i < optionIds.Count; i++)
+ if (optionIds[i].Count > per)
+ optionIds[i] = optionIds[i][..per];
+ optBudget = headMaxLen - optionIds.Sum(o => o.Count);
+ }
+
+ var headTake = Math.Max(MinHeadTokens, optBudget);
+ if (headIds.Count > headTake) headIds = headIds[..headTake];
+
+ var ids = new List<int>(maxLen) { tokenizer.ClsId };
+ ids.AddRange(headIds);
+ ids.Add(tokenizer.SepId);
+
+ var markers = new List<int>(optionIds.Count);
+ foreach (var fragment in optionIds)
+ {
+ markers.Add(ids.Count);
+ ids.AddRange(fragment);
+ }
+ ids.Add(tokenizer.SepId);
+
+ var room = Math.Max(0, maxLen - ids.Count - 1);
+ var stateIds = tokenizer.Encode(Scrub(PythonJson.State(state), mask));
+ ids.AddRange(SliceState(stateIds, room, truncateLeft));
+ ids.Add(tokenizer.SepId);
+
+ if (ids.Count > maxLen) ids = ids[..maxLen];
+ markers.RemoveAll(m => m >= maxLen);
+ return (ids.ToArray(), markers.ToArray());
+ }
+
+ private static string Scrub(string text, string maskToken) =>
+ text.Replace(maskToken, " ", StringComparison.Ordinal);
+
+ private static IEnumerable<int> SliceState(int[] stateIds, int room, bool truncateLeft)
+ {
+ if (!truncateLeft) return stateIds.Take(room);
+ // Matches 0.3.21's `state_ids[max(0, len(state_ids) - room):]`. 0.3.6 wrote `st[-room:]`,
+ // which for room == 0 is Python's `st[0:]` and so kept the whole state instead of none of
+ // it; `Skip(Math.Max(0, len - room))` has no such quirk and correctly returns empty when
+ // room == 0. `Agent._encode_state` sets `truncate_left = isinstance(state, list)`, so this
+ // path is reachable whenever a caller's state is a list of conversation turns.
+ return stateIds.Skip(Math.Max(0, stateIds.Length - room));
+ }
+
+ /// <summary>Python's <c>//</c>, which floors rather than truncating toward zero.</summary>
+ private static int FloorDiv(int a, int b)
+ {
+ var q = a / b;
+ return a % b != 0 && (a < 0) != (b < 0) ? q - 1 : q;
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/CollatorTests.cs b/laya-dotnet/tests/Laya.Tests/CollatorTests.cs
new file mode 100644
index 00000000..14f2b6f7
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/CollatorTests.cs
@@ -0,0 +1,130 @@
+namespace Laya.Tests;
+
+/// <summary>Tier 1: batch padding, which needs no artifacts.</summary>
+public sealed class CollatorTests
+{
+ private static SequenceItem Item(int length, int markers, QuestionType type = QuestionType.Choice) =>
+ new([.. Enumerable.Range(10, length)], [.. Enumerable.Range(1, markers)], type);
+
+ [Fact]
+ public void PadsToTheLongestRowAndMarksAttentionAccordingly()
+ {
+ var batch = Collator.Collate([Item(5, 2), Item(3, 2)], padId: 0);
+
+ Assert.Equal(2, batch.Count);
+ Assert.Equal(5, batch.SequenceLength);
+ Assert.Equal([1, 1, 1, 1, 1, 1, 1, 1, 0, 0], batch.AttentionMask);
+ // The short row's tail is the pad id, and attention is 0 there.
+ Assert.Equal(0, batch.InputIds[8]);
+ Assert.Equal(0, batch.InputIds[9]);
+ }
+
+ [Fact]
+ public void FillsWithTheGivenPadIdRatherThanZero()
+ {
+ var batch = Collator.Collate([Item(3, 2), Item(1, 2)], padId: 77);
+ Assert.Equal(77, batch.InputIds[4]);
+ Assert.Equal(77, batch.InputIds[5]);
+ Assert.Equal(0, batch.AttentionMask[4]);
+ }
+
+ [Fact]
+ public void CountsOnlyRealTokensAsInputTokens()
+ {
+ var batch = Collator.Collate([Item(5, 2), Item(3, 2)], padId: 0);
+ Assert.Equal(8, batch.InputTokens);
+ }
+
+ [Fact]
+ public void WidensTheMarkerAxisToTwoForSingleOptionBatches()
+ {
+ // The exported graph has TopK k=2 baked in: a one-column marker axis makes it fail outright.
+ var batch = Collator.Collate([Item(6, 1)], padId: 0);
+
+ Assert.Equal(Collator.MinMarkers, batch.MarkerCount);
+ Assert.Equal([1, 0], batch.MarkerPos);
+ Assert.Equal([true, false], batch.MarkerMask);
+ // The real option count is kept separately, so the pad column never reaches an answer.
+ Assert.Equal([1], batch.MarkerCounts);
+ }
+
+ [Fact]
+ public void PadsTheMarkerAxisToTheWidestQuestion()
+ {
+ var batch = Collator.Collate([Item(6, 4), Item(6, 2)], padId: 0);
+
+ Assert.Equal(4, batch.MarkerCount);
+ Assert.Equal([1, 2, 3, 4, 1, 2, 0, 0], batch.MarkerPos);
+ Assert.Equal([true, true, true, true, true, true, false, false], batch.MarkerMask);
+ Assert.Equal([4, 2], batch.MarkerCounts);
+ }
+
+ [Fact]
+ public void CarriesTheQuestionTypePerRowUsingPythonsNumbering()
+ {
+ var batch = Collator.Collate(
+ [Item(4, 2, QuestionType.Noul), Item(4, 2, QuestionType.Choice), Item(4, 2, QuestionType.Score)],
+ padId: 0);
+
+ Assert.Equal([2, 0, 1], batch.QType);
+ }
+
+ [Fact]
+ public void RefusesAnEmptyBatch() =>
+ Assert.Throws<ArgumentException>(() => Collator.Collate([], padId: 0));
+
+ [Theory]
+ [MemberData(nameof(GoldenData.SuccessCases), MemberType = typeof(GoldenData))]
+ public void ReproducesTheRecordedBatchTensors(GoldenCaseInfo info)
+ {
+ using var doc = GoldenData.Load(info);
+ var root = doc.RootElement;
+ var expected = root.GetProperty("batch");
+
+ // Rebuild the batch from the recorded per-question ids, so this checks the collator alone
+ // and stays meaningful without a tokenizer present.
+ var items = new List<SequenceItem>();
+ foreach (var q in root.GetProperty("per_question").EnumerateArray())
+ items.Add(new SequenceItem(
+ GoldenData.Ints(q.GetProperty("input_ids")),
+ GoldenData.Ints(q.GetProperty("markers")),
+ (QuestionType)q.GetProperty("qtype").GetInt32()));
+
+ var batch = Collator.Collate(items, padId: GoldenData.Meta.GetProperty("special_tokens")
+ .GetProperty("pad_id").GetInt32());
+
+ AssertMatrix(expected.GetProperty("input_ids"), batch.InputIds, batch.SequenceLength);
+ AssertMatrix(expected.GetProperty("attention_mask"), batch.AttentionMask, batch.SequenceLength);
+ AssertMatrix(expected.GetProperty("marker_pos"), batch.MarkerPos, batch.MarkerCount);
+ Assert.Equal([.. expected.GetProperty("qtype").EnumerateArray().Select(e => e.GetInt64())], batch.QType);
+
+ var expectedMask = expected.GetProperty("marker_mask");
+ Assert.Equal(expectedMask.GetArrayLength(), batch.Count);
+ for (var r = 0; r < batch.Count; r++)
+ {
+ var row = expectedMask[r];
+ for (var c = 0; c < batch.MarkerCount; c++)
+ Assert.Equal(row[c].GetBoolean(), batch.MarkerMask[r * batch.MarkerCount + c]);
+ }
+
+ Assert.Equal(root.GetProperty("result").GetProperty("usage").GetProperty("input_tokens").GetInt32(),
+ batch.InputTokens);
+ // marker_pos above already pinned the padded width; this checks the flag the dump script
+ // set, so a future golden that stops exercising the TopK edge case is noticed.
+ Assert.Equal(root.GetProperty("marker_axis_padded").GetBoolean(),
+ batch.MarkerCount > items.Max(i => i.Markers.Length));
+ }
+
+ private static void AssertMatrix(System.Text.Json.JsonElement expected, long[] actual, int cols)
+ {
+ var rows = expected.GetArrayLength();
+ Assert.Equal(rows * cols, actual.Length);
+ for (var r = 0; r < rows; r++)
+ {
+ var row = expected[r];
+ Assert.Equal(cols, row.GetArrayLength());
+ for (var c = 0; c < cols; c++)
+ Assert.Equal(row[c].GetInt64(), actual[r * cols + c]);
+ }
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.cs b/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.cs
new file mode 100644
index 00000000..85214317
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/SequenceBuilderTests.cs
@@ -0,0 +1,253 @@
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 1: sequence layout and the option budget, using <see cref="StubTokenizer"/> so no artifacts
+/// are needed. Real token ids are checked in <see cref="TokenizerParityTests"/>.
+/// </summary>
+public sealed class SequenceBuilderTests
+{
+ private readonly StubTokenizer _tok = new();
+
+ [Fact]
+ public void LaysOutClsHeadSepOptionsSepStateSep()
+ {
+ var q = Question.Choice("pick one", ("a", null), ("b", null));
+ var (ids, markers) = SequenceBuilder.Build(_tok, "state text", q, maxLen: 64, headMaxLen: 32);
+
+ Assert.Equal(_tok.ClsId, ids[0]);
+ Assert.Equal(_tok.SepId, ids[^1]);
+ // Each marker is a mask token, and the two options are adjacent fragments.
+ Assert.Equal(2, markers.Length);
+ Assert.All(markers, m => Assert.Equal(_tok.MaskId, ids[m]));
+ Assert.True(markers[1] > markers[0]);
+ // A SEP closes the head, immediately before the first marker.
+ Assert.Equal(_tok.SepId, ids[markers[0] - 1]);
+ }
+
+ [Fact]
+ public void MarkerPositionsPointAtConsecutiveOptionFragments()
+ {
+ // One word per option keeps the arithmetic checkable: mask + 1 word = 2 ids per fragment.
+ var q = Question.Choice("h", ("alpha", null), ("beta", null), ("gamma", null));
+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 64, headMaxLen: 32);
+
+ Assert.Equal(3, markers.Length);
+ Assert.Equal(markers[0] + 2, markers[1]);
+ Assert.Equal(markers[1] + 2, markers[2]);
+ Assert.Equal(_tok.SepId, ids[markers[2] + 2]);
+ }
+
+ [Fact]
+ public void PutsTheTypeNameInTheHeadText()
+ {
+ SequenceBuilder.Build(_tok, "s", Question.Noul("does it hold"), maxLen: 64, headMaxLen: 32);
+ Assert.Equal("noul question: does it hold", _tok.Encoded[0]);
+ }
+
+ [Theory]
+ [InlineData(QuestionType.Choice, "choice")]
+ [InlineData(QuestionType.Score, "score")]
+ [InlineData(QuestionType.Noul, "noul")]
+ public void TypeNamesMatchPython(QuestionType type, string expected) =>
+ Assert.Equal(expected, SequenceBuilder.TypeName(type));
+
+ [Fact]
+ public void OptionFragmentsAreEncodedWithALeadingSpace()
+ {
+ SequenceBuilder.Build(_tok, "s", Question.Choice("h", ("a", null)), maxLen: 64, headMaxLen: 32);
+ // Python encodes " " + option, which the Metaspace pre-tokenizer turns into a word boundary.
+ Assert.Contains(" a", _tok.Encoded);
+ }
+
+ [Fact]
+ public void RendersOptionsTheWayPythonDoes()
+ {
+ Assert.Equal(["a: first", "b"],
+ SequenceBuilder.RenderOptions(Question.Choice("h", ("a", "first"), ("b", null))));
+
+ Assert.Equal(["level 0: low", "level 1: high"],
+ SequenceBuilder.RenderOptions(Question.Score("h", "low", "high")));
+
+ Assert.Equal(["false: no, the statement does not hold", "true: yes, the statement holds"],
+ SequenceBuilder.RenderOptions(Question.Noul("h")));
+
+ Assert.Equal(["false: all fine", "true: on fire"],
+ SequenceBuilder.RenderOptions(Question.Noul("h", ifFalse: "all fine", ifTrue: "on fire")));
+ }
+
+ [Fact]
+ public void TreatsOnlyNullAndEmptyStringAsAMissingDescription()
+ {
+ // Python tests `v is None or v == ""`, so 0 and false are real descriptions.
+ Assert.Equal(["zero: 0", "no: false", "blank", "none"],
+ SequenceBuilder.RenderOptions(Question.Choice("h",
+ ("zero", 0L), ("no", false), ("blank", ""), ("none", null))));
+ }
+
+ [Fact]
+ public void ScrubsAMaskTokenOutOfEveryFragment()
+ {
+ // A literal <mask> in user input would forge a marker the scorer then reads as an option.
+ var q = Question.Choice($"is this {_tok.MaskToken} safe", ($"a{_tok.MaskToken}b", null));
+ SequenceBuilder.Build(_tok, $"state {_tok.MaskToken} here", q, maxLen: 64, headMaxLen: 32);
+
+ Assert.All(_tok.Encoded, text => Assert.DoesNotContain(_tok.MaskToken, text));
+ }
+
+ [Fact]
+ public void TruncatesLongOptionsToFortyEightTokens()
+ {
+ var words = string.Join(' ', Enumerable.Range(0, 120).Select(i => $"w{i}"));
+ var q = Question.Choice("h", ("opt", words));
+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 512, headMaxLen: 256);
+
+ // The only fragment is mask + at most 48 option tokens, then the closing SEP. "opt: " adds
+ // two words, so 48 is the cap that bites rather than the option running out.
+ Assert.Single(markers);
+ var fragmentEnd = Array.IndexOf(ids, _tok.SepId, markers[0]);
+ Assert.Equal(markers[0] + 49, fragmentEnd);
+ }
+
+ [Fact]
+ public void ShrinksOptionsWhenTheyWouldEatTheWholeHeadBudget()
+ {
+ // 20 options of ~10 tokens each against a 96-token head budget forces the
+ // `opt_budget < 16` branch, which caps every fragment at `(head_max_len - 16) // n`.
+ var options = Enumerable.Range(0, 20)
+ .Select(i => ($"opt{i}", (object?)string.Join(' ', Enumerable.Range(0, 10).Select(j => $"d{i}_{j}"))))
+ .ToArray();
+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", Question.Choice("head words here", options),
+ maxLen: 512, headMaxLen: 96);
+
+ Assert.Equal(20, markers.Length);
+ var per = Math.Max(4, (96 - 16) / 20); // 4
+ for (var i = 1; i < markers.Length; i++)
+ Assert.True(markers[i] - markers[i - 1] <= per,
+ $"fragment {i - 1} is {markers[i] - markers[i - 1]} tokens, above the {per}-token cap");
+ Assert.Equal(_tok.ClsId, ids[0]);
+ }
+
+ [Fact]
+ public void KeepsAtLeastEightHeadTokensWhenOptionsDominate()
+ {
+ var options = Enumerable.Range(0, 30)
+ .Select(i => ($"o{i}", (object?)"a b c d e f g h"))
+ .ToArray();
+ var (ids, markers) = SequenceBuilder.Build(_tok, "s",
+ Question.Choice("one two three four five six seven eight nine ten", options),
+ maxLen: 512, headMaxLen: 64);
+
+ // CLS + head + SEP, so the first marker cannot be earlier than position 2.
+ Assert.True(markers[0] >= 2);
+ Assert.Equal(_tok.SepId, ids[markers[0] - 1]);
+ }
+
+ [Fact]
+ public void TruncatesTheStateToFitMaxLen()
+ {
+ var state = string.Join(' ', Enumerable.Range(0, 500).Select(i => $"s{i}"));
+ var (ids, markers) = SequenceBuilder.Build(_tok, state, Question.Noul("h"), maxLen: 64, headMaxLen: 32);
+
+ Assert.Equal(64, ids.Length);
+ Assert.Equal(_tok.SepId, ids[^1]);
+ Assert.Equal(2, markers.Length);
+ }
+
+ [Fact]
+ public void KeepsTheStateTailWhenTruncatingLeft()
+ {
+ var state = string.Join(' ', Enumerable.Range(0, 200).Select(i => $"s{i}"));
+ var head = SequenceBuilder.Build(_tok, state, Question.Noul("h"), 64, 32, truncateLeft: false);
+ var tail = SequenceBuilder.Build(_tok, state, Question.Noul("h"), 64, 32, truncateLeft: true);
+
+ Assert.Equal(head.Ids.Length, tail.Ids.Length);
+ Assert.NotEqual(head.Ids, tail.Ids);
+ // Both keep the same prefix up to the state, and differ from there on.
+ Assert.Equal(head.Markers, tail.Markers);
+ }
+
+ [Fact]
+ public void LeftTruncationWithNoRoomDropsTheWholeStateRatherThanKeepingIt()
+ {
+ // 0.3.21's `state_ids[max(0, len - room):]` correctly returns empty when room == 0.
+ // 0.3.6's `st[-room:]` had a quirk here (Python's `x[-0:]` is `x[0:]`, the whole list) —
+ // reproduced faithfully until now, since `truncate_left` was previously unreachable from
+ // Predict. Isolate the "no room left" boundary without hardcoding tokenizer counts: probe
+ // the exact CLS+head+options+SEP prefix length with a one-token state, then rebuild at
+ // exactly that maxLen (room == 0) with a long state.
+ var q = Question.Noul("h");
+ var probe = SequenceBuilder.Build(_tok, "x", q, maxLen: 10_000, headMaxLen: 32);
+ var prefixLen = probe.Ids.Length - 2; // minus the one state token and the closing SEP
+
+ var longState = string.Join(' ', Enumerable.Range(0, 50).Select(i => $"s{i}"));
+ var (ids, _) = SequenceBuilder.Build(_tok, longState, q,
+ maxLen: prefixLen + 1, headMaxLen: 32, truncateLeft: true);
+
+ // Fixed: no state tokens fit, so the sequence ends prefix + SEP, exactly at maxLen.
+ // The old bug would instead have appended the whole 50-word state before the final
+ // maxLen clamp cut it back down to size, landing a state token (not SEP) at this position.
+ Assert.Equal(prefixLen + 1, ids.Length);
+ Assert.Equal(_tok.SepId, ids[^1]);
+ }
+
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void ListStatesAreDetectedForAutomaticLeftTruncation(bool expectListLike)
+ {
+ // Mirrors Agent._encode_state: `truncate_left = isinstance(state, list)`. LayaEngine.Predict
+ // derives its truncateLeft argument from PythonJson.IsListState, so this is the source of
+ // truth callers rely on for "is this state list-shaped conversation turns".
+ object state = expectListLike
+ ? new List<object?> { "turn 1", "turn 2" }
+ : "a plain string state";
+
+ Assert.Equal(expectListLike, PythonJson.IsListState(state));
+ }
+
+ [Fact]
+ public void DictAndStringStatesAreNotListLike()
+ {
+ Assert.False(PythonJson.IsListState("a string"));
+ Assert.False(PythonJson.IsListState(new Dictionary<string, object?> { ["k"] = "v" }));
+ Assert.False(PythonJson.IsListState(null));
+ Assert.True(PythonJson.IsListState(new object?[] { "a", "b" }));
+ }
+
+ [Fact]
+ public void DropsMarkersThatWouldLandPastMaxLen()
+ {
+ // 40 options cannot fit a 32-token sequence, so markers are lost. Predict must treat a short
+ // marker list as an error rather than answering a question missing options.
+ var options = Enumerable.Range(0, 40).Select(i => ($"o{i}", (object?)"desc")).ToArray();
+ var q = Question.Choice("h", options);
+ var (ids, markers) = SequenceBuilder.Build(_tok, "s", q, maxLen: 32, headMaxLen: 256);
+
+ Assert.Equal(32, ids.Length);
+ Assert.True(markers.Length < options.Length);
+ Assert.All(markers, m => Assert.True(m < 32));
+ }
+
+ [Fact]
+ public void RendersNonStringInstructionsWithRealUnicodeNotEscapes()
+ {
+ // 0.3.21's `_to_internal` switched to json.dumps(ins, ensure_ascii=False): real Unicode
+ // glyphs reach the tokenizer instead of literal \uXXXX escape text.
+ var q = Question.Noul(new OrderedDictionary<string, object?>(StringComparer.Ordinal)
+ {
+ ["hint"] = "Müller",
+ });
+ var rendered = SequenceBuilder.RenderInstructions(q);
+
+ Assert.Contains("ü", rendered);
+ Assert.DoesNotContain("\\u", rendered);
+ Assert.Equal("{\"hint\": \"Müller\"}", rendered);
+ }
+
+ [Fact]
+ public void PassesStringInstructionsThroughUntouched()
+ {
+ var q = Question.Noul("Müller & co <urgent>");
+ Assert.Equal("Müller & co <urgent>", SequenceBuilder.RenderInstructions(q));
+ }
+}