Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

38.1 KB
Raw Blame
using System.Runtime.CompilerServices;
using Laya.Tokenization;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;

namespace Laya;

/// <summary>
/// Answers typed questions about a piece of state in one forward pass. The .NET counterpart of
/// Python's <c>laya.Agent</c>: this library is a .NET port of the Laya Python SDK (the
/// <c>laya</c> package on PyPI), run on ONNX Runtime instead of PyTorch.
/// </summary>
/// <remarks>
/// <para>
/// Loading the model is expensive and the session is reusable, so create one engine and keep it:
/// it is safe to share across threads and works well as a singleton.
/// </para>
/// <para>
/// Every question in a <see cref="QuestionSet"/> is answered by a single batched inference call,
/// matching Python's single forward pass — asking five questions costs roughly what asking one does.
/// </para>
/// <para>
/// Loads either ONNX artifact layout (<see cref="LayaArtifactLayout"/>). For the fused layout,
/// one <see cref="InferenceSession"/> runs the whole forward pass. For the split layout (the
/// laya-ts exporter's output), two sessions run in sequence: the encoder produces
/// <c>last_hidden_state</c>, which is fed straight into the head. Both cases go through
/// <see cref="Run"/>, so batching, calibration and <see cref="Predict(object?, QuestionSet)"/> do not know which
/// layout is loaded.
/// </para>
/// </remarks>
public sealed class LayaEngine : IDisposable, ILayaLongLangPredictor, ILayaBatchPredictor, ILayaCallPredictor, ILayaLangTemperatures
{
    private const string ModelName = "laya-rl-agent";

    // Exactly one of _fusedSession or (_encoderSession, _headSession) is non-null, matching
    // Artifacts.Layout.
    private readonly InferenceSession? _fusedSession;
    private readonly InferenceSession? _encoderSession;
    private readonly InferenceSession? _headSession;
    private readonly HfTokenizer _tokenizer;
    private readonly bool _ownsTokenizer;
    private readonly Lock _tokenizerGate = new();
    private readonly IReadOnlyDictionary<string, ResolvedLangTemperature> _langTemperatures;
    // The temperatures the checkpoint answers with: its own, or a calibration file's (which replaces both), and the file's
    // binning map (null: none, so answer_confidence is the temperature-scaled one).
    private readonly IReadOnlyList<double> _temperature;
    private readonly IReadOnlyDictionary<string, double> _temperatureByOptions;
    private readonly IReadOnlyDictionary<string, BinningEntry>? _binningMap;
    // 0 = live, 1 = disposed; int so Interlocked.Exchange can operate on it.
    private int _disposed;

    private LayaEngine(InferenceSession? fusedSession, InferenceSession? encoderSession,
                       InferenceSession? headSession, HfTokenizer tokenizer, bool ownsTokenizer,
                       LayaConfig config, ModelArtifacts artifacts,
                       IReadOnlyDictionary<string, ResolvedLangTemperature> langTemperatures,
                       CalibrationPayload? calibration)
    {
        _fusedSession = fusedSession;
        _encoderSession = encoderSession;
        _headSession = headSession;
        _tokenizer = tokenizer;
        _ownsTokenizer = ownsTokenizer;
        Config = config;
        Artifacts = artifacts;
        _langTemperatures = langTemperatures;
        _temperature = calibration?.Temperature ?? config.Temperature;
        _temperatureByOptions = calibration?.TemperatureByOptions ?? config.TemperatureByOptions;
        // Python tests the map's truth, so an empty one is no map at all.
        _binningMap = calibration?.BinningMap is { Count: > 0 } map ? map : null;
    }

    /// <summary>The loaded checkpoint's configuration.</summary>
    public LayaConfig Config { get; }

    /// <summary>Where the loaded artifacts came from.</summary>
    public ModelArtifacts Artifacts { get; }

    /// <summary>
    /// Whether <see cref="LayaOptions.LangTemperatures"/> named at least one language. Python's
    /// <c>agent.lang_temperatures</c> being non-empty; a router splits a batch by language only for such an engine.
    /// </summary>
    public bool HasLangTemperatures => _langTemperatures.Count > 0;

    /// <summary>
    /// Which checkpoint variant was loaded, or <see langword="null"/> when loaded from an
    /// explicit directory (via <see cref="LayaOptions.ModelDirectory"/> or the
    /// <c>LAYA_ONNX_DIR</c> environment variable) where no checkpoint was inferred.
    /// </summary>
    public LayaCheckpoint? Checkpoint => Artifacts.Checkpoint;

    /// <summary>
    /// Load an engine, resolving the artifact directory from <paramref name="options"/>, the
    /// <c>LAYA_ONNX_DIR</c> environment variable, or the local cache.
    /// </summary>
    /// <exception cref="InvalidOperationException">
    /// When the loaded config's encoder contradicts the expected encoder for the requested
    /// <see cref="LayaOptions.Checkpoint"/> — indicates the wrong artifact was loaded.
    /// </exception>
    /// <exception cref="ArgumentException">
    /// When <see cref="LayaOptions.LangTemperatures"/> holds a language whose temperature is not three values, with
    /// Python's <c>ValueError</c> text.
    /// </exception>
    /// <exception cref="InvalidDataException">
    /// When <see cref="LayaOptions.Calibration"/> names a file Python's <c>load_calibration</c> refuses, with its exception type and text.
    /// </exception>
    [WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
    public static LayaEngine Create(LayaOptions? options = null)
    {
        options ??= new LayaOptions();
        var artifacts = ModelArtifacts.Resolve(options);
        // onnx_agent.py:249: resolved against the checkpoint's own temperature list, between reading it and warning about it.
        IReadOnlyDictionary<string, ResolvedLangTemperature> langTemperatures = new Dictionary<string, ResolvedLangTemperature>();
        var config = LayaConfig.Load(artifacts.ConfigPath, loaded =>
            langTemperatures = LangTemperatures.Resolve(options.LangTemperatures, loaded.TemperatureSource, loaded.Temperature));

        // When the checkpoint was resolved via a checkpoint-aware path (LAYA_ONNX_ROOT, cache,
        // or download), validate that the artifact's encoder agrees. A mismatch here means the
        // directory name and the contents disagree — something that happens when a user manually
        // copies or renames a checkpoint directory.
        if (artifacts.Checkpoint is { } expectedCheckpoint)
            ValidateEncoder(config.Encoder, expectedCheckpoint, artifacts.Directory);

        // using disposes sessionOptions on both success and failure; ORT copies the options at
        // CreateSession time so they are not needed after the InferenceSession constructor returns.
        using var sessionOptions = BuildSessionOptions(options);
        InferenceSession? fusedSession = null;
        InferenceSession? encoderSession = null;
        InferenceSession? headSession = null;
        HfTokenizer? tokenizer = null;
        try
        {
            if (artifacts.Layout == LayaArtifactLayout.Fused)
            {
                // The path must point at model.onnx inside its own directory: this checkpoint is
                // in ONNX external-data format and the runtime resolves the ~1.3 GB
                // model.onnx.data sidecar relative to the graph file.
                fusedSession = new InferenceSession(artifacts.ModelPath!, sessionOptions);
            }
            else
            {
                encoderSession = new InferenceSession(artifacts.EncoderPath!, sessionOptions);
                headSession = new InferenceSession(artifacts.HeadPath!, sessionOptions);
            }

            tokenizer = new HfTokenizer(artifacts.TokenizerPath);

            // onnx_agent.py:263: last, after the tokenizer, the session, the checkpoint's temperatures and
            // the language table, so a broken checkpoint reports its own error before a bad calibration file.
            CalibrationPayload? calibration = null;
            if (!string.IsNullOrEmpty(options.Calibration))
                calibration = CalibrationFile.Load(options.Calibration);

            return new LayaEngine(fusedSession, encoderSession, headSession,
                tokenizer, ownsTokenizer: true, config, artifacts, langTemperatures, calibration);
        }
        catch
        {
            tokenizer?.Dispose();
            fusedSession?.Dispose();
            encoderSession?.Dispose();
            headSession?.Dispose();
            throw;
        }
    }

    /// <summary>Load an engine from an artifact directory containing either ONNX layout.</summary>
    [WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
    public static LayaEngine FromDirectory(string directory) =>
        Create(new LayaOptions { ModelDirectory = directory });

    private static void ValidateEncoder(string? actualEncoder, LayaCheckpoint checkpoint, string dir)
    {
        // Multilingual uses mmBERT; English and TypedDecisions both use ModernBERT-large.
        // We can only detect a family mismatch (mmBERT vs. ModernBERT), not distinguish English
        // from TypedDecisions, since both share the same encoder string.
        var expectModernBert = checkpoint != LayaCheckpoint.Multilingual;
        var gotModernBert    = actualEncoder is not null
                               && actualEncoder.Contains("ModernBERT", StringComparison.OrdinalIgnoreCase);

        if (expectModernBert != gotModernBert)
        {
            var expected = checkpoint == LayaCheckpoint.Multilingual
                ? "jhu-clsp/mmBERT-base"
                : "answerdotai/ModernBERT-large";
            throw new InvalidOperationException(
                $"Checkpoint '{checkpoint}' expects encoder '{expected}' but the artifact at "
                + $"'{dir}' declares encoder '{actualEncoder ?? "(null)"}'. "
                + "The artifact directory may contain the wrong checkpoint.");
        }
    }

    private static SessionOptions BuildSessionOptions(LayaOptions options)
    {
        var so = new SessionOptions();
        try
        {
            if (options.IntraOpThreads is { } intra) so.IntraOpNumThreads = intra;
            if (options.InterOpThreads is { } inter) so.InterOpNumThreads = inter;
            switch (options.ExecutionProvider)
            {
                case LayaExecutionProvider.Cpu:
                    break;
                case LayaExecutionProvider.Cuda:
                    so.AppendExecutionProvider_CUDA();
                    break;
                case LayaExecutionProvider.DirectMl:
                    so.AppendExecutionProvider_DML();
                    break;
                default:
                    throw new ArgumentOutOfRangeException(nameof(options), options.ExecutionProvider,
                        "unknown execution provider");
            }
            return so;
        }
        catch
        {
            so.Dispose();
            throw;
        }
    }

    /// <summary>Answer every question in <paramref name="questions"/> about <paramref name="state"/>.</summary>
    /// <param name="state">
    /// A <see cref="string"/> is used verbatim; anything else is serialized the way Python's
    /// <c>serialize_state</c> does, so an object or a list of conversation turns both work.
    /// </param>
    /// <param name="questions">The questions to answer, in the order they should be returned.</param>
    /// <exception cref="ArgumentNullException">If <paramref name="state"/> is null, as Python's <c>TypeError</c>.</exception>
    /// <exception cref="ArgumentException">
    /// If a question's options cannot fit the head budget, rather than answering a silently
    /// truncated question.
    /// </exception>
    public LayaResult Predict(object? state, QuestionSet questions) => PredictOne(state, questions, default);

    private LayaResult PredictOne(object? state, QuestionSet questions, CallSettings call)
    {
        StateGuard.NotNull(state);
        ArgumentNullException.ThrowIfNull(questions);
        ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
        // Python's system_one is predict_batch([state]): no questions is an empty answer set, not an error,
        // and the usage carries the same truncation diagnostics.
        return PredictStates([state], questions, diagnostics: true, call)[0];
    }

    /// <summary>The per-call arguments of Python's <c>system_one</c> / <c>predict_batch</c> that reach the decode.</summary>
    private readonly record struct CallSettings(string? Lang = null, int? MaxLen = null, int? HeadMaxLen = null);

    /// <summary>
    /// <see cref="Predict(object?, QuestionSet)"/> with per-call options. Ports the <c>min_confidence</c>
    /// argument of Python's <c>system_one</c>: with <see cref="LayaPredictOptions.MinConfidence"/> set, every answer
    /// reports <see cref="Answer.Abstention"/> and <see cref="Answer.AbstentionThreshold"/>, and carries
    /// <see cref="Answer.LowConfidence"/> when it falls below the threshold. The threshold is validated before
    /// anything else, as in Python, where it is the first thing <c>predict_batch</c> checks. Also ports the
    /// <c>lang</c>, <c>max_len</c> and <c>head_max_len</c> arguments: see <see cref="LayaPredictOptions.Lang"/>,
    /// <see cref="LayaPredictOptions.MaxLen"/> and <see cref="LayaPredictOptions.HeadMaxLen"/>.
    /// </summary>
    /// <exception cref="ArgumentException">The threshold is invalid, with Python's <c>ValueError</c> text; or a token
    /// budget leaves a question's options no room (the same error as a checkpoint whose own budget is too small).</exception>
    public LayaResult Predict(object? state, QuestionSet questions, LayaPredictOptions? options)
    {
        var gate = options?.MinConfidence?.Check();
        var result = PredictOne(state, questions, new(options?.Lang, options?.MaxLen, options?.HeadMaxLen));
        return gate is { } g ? ConfidenceGate.Apply(result, g) : result;
    }

    /// <summary>
    /// Scan a long state in overlapping token windows. Ports Python ONNXAgent.predict_long:
    /// noul selects the greatest P(true); choice and score select the greatest answer confidence.
    /// Ties select the first window. A state within the room the questions leave (or within an
    /// explicit <paramref name="window"/>) takes the normal prediction path.
    /// </summary>
    /// <param name="state">Text, a JSON-shaped object, or a chronological conversation list; not null.</param>
    /// <param name="questions">Questions in declaration order.</param>
    /// <param name="window">State tokens per window; nonpositive or null uses the checkpoint default.
    /// Capped at the smallest state room left by these questions.</param>
    /// <param name="stride">Token step; nonpositive or null uses half the effective window.
    /// A step past the requested window is rejected; a valid step past a clamped window is reduced.</param>
    /// <param name="aggregate">Only "auto" is supported.</param>
    /// <param name="batchSize">Maximum windows per forward pass; nonpositive or null uses Python's automatic cap.</param>
    /// <returns>The deciding window's answers and summed usage, including the number of windows.</returns>
    /// <exception cref="ArgumentException">Unsupported aggregation, a gap-causing stride, or no state room.</exception>
    [WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
    public LayaResult PredictLong(object state, QuestionSet questions, int? window = null, int? stride = null,
                                  string aggregate = "auto", int? batchSize = null) =>
        PredictLongCore(state, questions, window, stride, aggregate, batchSize, lang: null);

    /// <summary>
    /// <see cref="PredictLong(object, QuestionSet, int?, int?, string, int?)"/> with the per-call language of
    /// Python's <c>predict_long(lang=...)</c>, applied to every window (and to the single pass a short state takes);
    /// see <see cref="LayaLongOptions"/>. <see langword="null"/> options are the defaults of the other overload.
    /// </summary>
    [WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
    public LayaResult PredictLong(object state, QuestionSet questions, LayaLongOptions? options) =>
        PredictLongCore(state, questions, options?.Window, options?.Stride, options?.Aggregate ?? "auto",
                        options?.BatchSize, options?.Lang);

    private LayaResult PredictLongCore(object state, QuestionSet questions, int? window, int? stride,
                                       string aggregate, int? batchSize, string? lang)
    {
        StateGuard.NotNull(state);
        ArgumentNullException.ThrowIfNull(questions);
        ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
        if (aggregate != "auto")
            throw new ArgumentException("predict_long: only aggregate='auto' is supported", nameof(aggregate));

        int budget, step, room;
        int[] stateIds;
        lock (_tokenizerGate)
        {
            (budget, step, room) = LongPrediction.Budget(
                _tokenizer, questions, Config.MaxLen, Config.HeadMaxLen, window, stride);
            stateIds = _tokenizer.Encode(PythonJson.State(state).Replace(_tokenizer.MaskToken, " ", StringComparison.Ordinal));
        }
        // Predict reads the whole room the questions leave, so the default window must not
        // force a scan there; an explicit window still bounds the single pass (Python 3e9bda30).
        if (stateIds.Length <= (window is > 0 ? budget : room))
        {
            var single = PredictStates([state], questions, diagnostics: true, new(lang))[0];
            return new LayaResult(ModelName, [.. single.Ids],
                single.Ids.ToDictionary(id => id, id => single[id], StringComparer.Ordinal),
                single.Usage with { Windows = 1 });
        }
        var spans = LongPrediction.Spans(stateIds.Length, budget, step);
        var windows = new List<object?>(spans.Count);
        lock (_tokenizerGate)
            foreach (var span in spans)
                windows.Add(_tokenizer.Decode(stateIds[span.TokenStart..span.TokenEnd]));
        var cap = LongPrediction.BatchCap(spans.Count, budget,
            Math.Max(64, Config.MaxLen - Config.HeadMaxLen - 8), batchSize) ?? spans.Count;
        var results = new List<LayaResult>(spans.Count);
        for (var i = 0; i < windows.Count; i += cap)
            results.AddRange(PredictStates(windows.GetRange(i, Math.Min(cap, windows.Count - i)),
                                           questions, diagnostics: true, new(lang)));
        return LongPrediction.Aggregate(questions, results, spans);
    }

    /// <summary>
    /// Evaluate the same <paramref name="questions"/> over many states, sharing forward passes. Ports
    /// Python <c>ONNXAgent.predict_batch</c> (onnx_agent.py:342): one result per state in input order,
    /// each identical in shape to what a single-state pass over the same states would produce, with
    /// the diagnostic usage fields (state tokens, truncation, collapsed options) that Python's ONNX
    /// path reports.
    /// </summary>
    /// <param name="states">The states; a null element is refused, an empty list returns an empty list.</param>
    /// <param name="questions">The questions answered about every state. With no questions each state
    /// gets an empty answer set and zero usage, as in Python.</param>
    /// <param name="options">Batch size and ordering; see <see cref="LayaBatchOptions"/>.</param>
    /// <exception cref="ArgumentException">A null state, an invalid <see cref="LayaBatchOptions.MinConfidence"/>,
    /// or a question whose options cannot fit the head budget.</exception>
    public IReadOnlyList<LayaResult> PredictBatch(
        IReadOnlyList<object?> states, QuestionSet questions, LayaBatchOptions? options = null)
    {
        // onnx_agent.py:386: the threshold is validated before the states or the questions are looked at.
        var gate = options?.MinConfidence?.Check();
        var results = PredictBatchUngated(states, questions, options);
        return gate is { } g ? [.. results.Select(r => ConfidenceGate.Apply(r, g))] : results;
    }

    private IReadOnlyList<LayaResult> PredictBatchUngated(
        IReadOnlyList<object?> states, QuestionSet questions, LayaBatchOptions? options)
    {
        ArgumentNullException.ThrowIfNull(states);
        ArgumentNullException.ThrowIfNull(questions);
        ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
        // onnx_agent.py:398-414: the surviving states are validated before anything runs, and a
        // missing state would otherwise be answered as the literal text "null".
        var list = states.ToArray();
        foreach (var state in list) StateGuard.NotNull(state);
        // onnx_agent.py:415: an empty list answers [] without looking at the questions.
        if (list.Length == 0) return [];
        var call = new CallSettings(options?.Lang, options?.MaxLen, options?.HeadMaxLen);
        if (questions.Count == 0) return PredictStates(list, questions, diagnostics: true, call);
        return BatchPlanning.Execute<EncodedState, LayaResult>(
            list, options?.BatchSize, options?.SortByLength ?? false,
            part => EncodeStates(part, questions, call),
            encoded => encoded.Items.Max(item => item.Ids.Length),
            chunk => InferStates(chunk, questions, diagnostics: true, call.Lang));
    }

    private sealed record EncodedState(SequenceItem[] Items, SequenceBuilder.SequenceDetails[] Details);

    private IReadOnlyList<LayaResult> PredictStates(
        IReadOnlyList<object?> states, QuestionSet questions, bool diagnostics, CallSettings call)
    {
        if (questions.Count == 0)
            return states.Select(_ => new LayaResult(ModelName, [], new(StringComparer.Ordinal), new Usage(0, 0))).ToArray();
        return InferStates(EncodeStates(states, questions, call), questions, diagnostics, call.Lang);
    }

    private IReadOnlyList<EncodedState> EncodeStates(IReadOnlyList<object?> states, QuestionSet questions, CallSettings call)
    {
        var ids = questions.Ids;
        // onnx_agent.py:807: the call's budgets, else the checkpoint's own.
        var maxLen = call.MaxLen ?? Config.MaxLen;
        var headMaxLen = call.HeadMaxLen ?? Config.HeadMaxLen;
        var encoded = new List<EncodedState>(states.Count);

        // A chronological conversation list is serialized newest-last, so right-truncation
        // (the default) would silently drop the newest turn. Matches Agent._encode_state's
        // `truncate_left = isinstance(state, list)`: only a list-shaped state truncates from the
        // left; string and dict states keep the existing right-truncation.

        // The tokenizer wraps a native handle whose thread safety is not documented, so
        // tokenization is serialized. Inference is not: it is the expensive part and
        // InferenceSession.Run is thread-safe.
        lock (_tokenizerGate)
        {
            foreach (var state in states)
            {
                var items = new List<SequenceItem>(ids.Count);
                var details = new List<SequenceBuilder.SequenceDetails>(ids.Count);
                var truncateLeft = PythonJson.IsListState(state);
                var stateIds = _tokenizer.Encode(
                    PythonJson.State(state).Replace(_tokenizer.MaskToken, " ", StringComparison.Ordinal));
                foreach (var id in ids)
                {
                    var question = questions[id];
                    var built = SequenceBuilder.BuildDetailed(
                        _tokenizer, state, question, maxLen, headMaxLen, truncateLeft, stateIds);
                    var (seq, markers) = (built.Ids, built.Markers);

                    // A dropped marker would score an unrelated token at position 0.
                    if (markers.Length != SequenceBuilder.RenderOptions(question).Count)
                        throw new ArgumentException(
                            $"question '{id}': only {markers.Length} of its {SequenceBuilder.RenderOptions(question).Count} "
                            + $"option markers fit in max_len={maxLen} with head_max_len={headMaxLen} "
                            + "spent on the question; lower head_max_len, raise max_len, or use fewer options", nameof(questions));

                    items.Add(new SequenceItem(seq, markers, question.Type));
                    details.Add(built);
                }
                encoded.Add(new EncodedState([.. items], [.. details]));
            }
        }
        return encoded;
    }

    private IReadOnlyList<LayaResult> InferStates(
        IReadOnlyList<EncodedState> encoded, QuestionSet questions, bool diagnostics, string? lang)
    {
        var ids = questions.Ids;
        var states = encoded;
        var items = encoded.SelectMany(e => e.Items).ToList();
        var details = encoded.SelectMany(e => e.Details).ToList();

        var batch = Collator.Collate(items, _tokenizer.PadId);
        var (logits, actLogits) = Run(batch);

        // n_act is the act_logits column count, derived from the tensor's dims[1] rather than
        // hardcoded so the code stays correct if the graph is re-exported with a wider action head.
        var nAct = actLogits.Length / batch.Count;

        var results = new List<LayaResult>(states.Count);
        for (var stateIndex = 0; stateIndex < states.Count; stateIndex++)
        {
            var answers = new Dictionary<string, Answer>(questions.Count, StringComparer.Ordinal);
            var offset = stateIndex * ids.Count;
            for (var q = 0; q < ids.Count; q++)
            {
                var row = offset + q;
                var question = questions[ids[q]];
                var k = batch.MarkerCounts[row];
                var rowLogits = logits.AsSpan(row * batch.MarkerCount, batch.MarkerCount);
                var action = new ActionInfo(
                    Calibration.Round4(Calibration.Softmax(actLogits.AsSpan(row * nAct, nAct))[0]));
                answers[ids[q]] = DecodeRow(question, rowLogits, k, _temperature, _temperatureByOptions,
                    LangTemperatures.Find(_langTemperatures, lang), _binningMap, action);
            }
            var stateDetails = details.GetRange(offset, ids.Count);
            var usage = new Usage(stateDetails.Sum(d => d.Ids.Length), 0);
            if (diagnostics)
            {
                var dropped = stateDetails.Max(d => d.StateTokensDropped);
                var collapsed = new Dictionary<string, OptionUsage>(StringComparer.Ordinal);
                for (var q = 0; q < ids.Count; q++)
                    if (stateDetails[q].Options.Distinct < stateDetails[q].Options.Total)
                        collapsed[ids[q]] = stateDetails[q].Options;
                usage = usage with
                {
                    StateTokens = stateDetails[0].StateTokens,
                    StateTokensDropped = dropped,
                    Truncated = dropped > 0 ? 1 : 0,
                    TruncatedQuestions = ids.Where((_, q) => stateDetails[q].StateTokensDropped > 0).ToArray(),
                    Options = collapsed.Count == 0 ? null : collapsed,
                };
            }
            results.Add(new LayaResult(ModelName, [.. ids], answers, usage));
        }
        return results;
    }

    /// <summary>
    /// One row of <c>_decode_answers</c> (<c>onnx_agent.py:725-756</c>): the temperature (<paramref name="langTemperature"/>, when
    /// the call's language is one the table holds, replaces both the bucket map and the per-type temperature, with no fallback to
    /// either), the softmax, and the answer. <paramref name="binningMap"/> recalibrates <c>answer_confidence</c> alone, on the
    /// unrounded <c>max(p)</c> and before the rounding, and is skipped under a language override.
    /// </summary>
    internal static Answer DecodeRow(
        Question question, ReadOnlySpan<float> rowLogits, int k,
        IReadOnlyList<double> temperature, IReadOnlyDictionary<string, double> temperatureByOptions,
        ResolvedLangTemperature? langTemperature, IReadOnlyDictionary<string, BinningEntry>? binningMap, ActionInfo action)
    {
        var bucket = Calibration.TempBucket(question.Type, k);
        var scale = Calibration.ResolveTemperature(temperature, temperatureByOptions, question.Type, k);
        if (langTemperature is not null)
            scale = langTemperature.TemperatureByOptions.TryGetValue(bucket, out var byBucket)
                ? byBucket
                : langTemperature.Temperature[(int)question.Type];
        var p = Calibration.Probabilities(rowLogits, k, scale);
        var answerConfidence = Calibration.AnswerConfidence(p, k);
        if (binningMap is not null && langTemperature is null)
            answerConfidence = CalibrationFile.ApplyBinning(answerConfidence, bucket, binningMap);
        return BuildAnswer(question, p, k, action, Calibration.Round4(answerConfidence));
    }

    private static Answer BuildAnswer(Question question, double[] p, int k, ActionInfo action, double answerConfidence)
    {
        var confidence = Calibration.Round4(Calibration.ConfidenceFromProbs(p, k));
        switch (question)
        {
            case ChoiceQuestion choice:
            {
                var labels = choice.Labels;
                var probabilities = new List<KeyValuePair<string, double>>(k);
                // Python zips labels against probabilities, so a mismatch would silently drop the
                // tail rather than misalign; take the shorter for the same reason.
                for (var i = 0; i < Math.Min(k, labels.Count); i++)
                    probabilities.Add(new(labels[i], Calibration.Round4(p[i])));
                return new ChoiceAnswer(labels[Calibration.ArgMax(p)], probabilities,
                                        confidence, answerConfidence, action);
            }

            case ScoreQuestion score:
            {
                var probabilities = new double[p.Length];
                for (var i = 0; i < p.Length; i++) probabilities[i] = Calibration.Round4(p[i]);
                return new ScoreAnswer(Calibration.Round4(Calibration.ExpectedScore(p)),
                                       score.Levels.Select(PythonJson.Criterion).Cast<object?>().ToArray(),
                                       probabilities, confidence, answerConfidence, action);
            }

            default:
            {
                // Noul reports the true-branch probability, and its confidence is the distance from
                // a coin flip rather than the entropy measure the other two types use. Over two
                // options max(p_true, 1 - p_true) equals max(p), so answerConfidence agrees here —
                // matching the Python comment on this exact case in `_decode_answers`.
                var pTrue = p[1];
                return new NoulAnswer(Calibration.Round4(pTrue),
                                      Calibration.Round4(Math.Max(pTrue, 1.0 - pTrue)),
                                      answerConfidence, action);
            }
        }
    }

    /// <summary>One forward pass over a collated batch, returning the raw graph outputs.</summary>
    /// <remarks>
    /// Internal rather than private so the test suite can drive the graph directly — the shape sweep
    /// and the recorded-logits comparison both need the outputs before calibration, and sharing this
    /// session keeps them from loading a second copy of the weights.
    /// </remarks>
    internal (float[] Logits, float[] ActLogits) Run(CollatedBatch batch) =>
        Artifacts.Layout == LayaArtifactLayout.Fused ? RunFused(batch) : RunSplit(batch);

    private (float[] Logits, float[] ActLogits) RunFused(CollatedBatch batch)
    {
        var session = _fusedSession!;
        var inputs = new List<NamedOnnxValue>(5);

        if (session.InputMetadata.ContainsKey("input_ids"))
            inputs.Add(LongInput(session, "input_ids", batch.InputIds, batch.Count, batch.SequenceLength));
        if (session.InputMetadata.ContainsKey("attention_mask"))
            inputs.Add(LongInput(session, "attention_mask", batch.AttentionMask, batch.Count, batch.SequenceLength));
        if (session.InputMetadata.ContainsKey("marker_pos"))
            inputs.Add(LongInput(session, "marker_pos", batch.MarkerPos, batch.Count, batch.MarkerCount));
        if (session.InputMetadata.ContainsKey("marker_mask"))
            inputs.Add(MaskInput(session, "marker_mask", batch.MarkerMask, batch.Count, batch.MarkerCount));
        if (session.InputMetadata.ContainsKey("qtype"))
            inputs.Add(QTypeInput(session, batch.QType, batch.Count));

        using var results = session.Run(inputs);
        return (Output(results, "logits"), Output(results, "act_logits"));
    }

    private (float[] Logits, float[] ActLogits) RunSplit(CollatedBatch batch)
    {
        var encoder = _encoderSession!;
        var head = _headSession!;

        var encoderInputs = new List<NamedOnnxValue>(2);
        if (encoder.InputMetadata.ContainsKey("input_ids"))
            encoderInputs.Add(LongInput(encoder, "input_ids", batch.InputIds, batch.Count, batch.SequenceLength));
        if (encoder.InputMetadata.ContainsKey("attention_mask"))
            encoderInputs.Add(LongInput(encoder, "attention_mask", batch.AttentionMask, batch.Count, batch.SequenceLength));

        // Both Run calls happen inside this one using block: headResults is built and consumed
        // while encoderResults (and the last_hidden_state tensor it owns) is still alive, so the
        // head reads the encoder's output tensor directly with no extra copy.
        using var encoderResults = encoder.Run(encoderInputs);
        var hiddenState = OutputTensor(encoderResults, "last_hidden_state");

        var headInputs = new List<NamedOnnxValue>(5)
        {
            NamedOnnxValue.CreateFromTensor("hidden_states", hiddenState),
        };
        if (head.InputMetadata.ContainsKey("marker_pos"))
            headInputs.Add(LongInput(head, "marker_pos", batch.MarkerPos, batch.Count, batch.MarkerCount));
        if (head.InputMetadata.ContainsKey("marker_mask"))
            headInputs.Add(MaskInput(head, "marker_mask", batch.MarkerMask, batch.Count, batch.MarkerCount));
        if (head.InputMetadata.ContainsKey("qtype"))
            headInputs.Add(QTypeInput(head, batch.QType, batch.Count));
        if (head.InputMetadata.ContainsKey("attention_mask"))
            headInputs.Add(LongInput(head, "attention_mask", batch.AttentionMask, batch.Count, batch.SequenceLength));

        using var headResults = head.Run(headInputs);
        return (Output(headResults, "logits"), Output(headResults, "act_logits"));
    }

    // ── input/output helpers shared by both layouts ────────────────────────────

    /// <summary>
    /// Builds an integer input tensor, matching whichever of int64/int32 the session declares
    /// for <paramref name="name"/> (mmBERT- and ModernBERT-family exports both use int64, but the
    /// declared dtype — not a hardcoded assumption — decides which one is fed).
    /// </summary>
    private static NamedOnnxValue LongInput(InferenceSession session, string name, long[] data, params int[] shape)
    {
        var elementType = session.InputMetadata[name].ElementType;
        if (elementType == typeof(long))
            return NamedOnnxValue.CreateFromTensor(name, new DenseTensor<long>(data, shape));
        if (elementType == typeof(int))
            return NamedOnnxValue.CreateFromTensor(name, new DenseTensor<int>(Array.ConvertAll(data, x => (int)x), shape));

        throw new InvalidOperationException(
            $"input '{name}' is declared as {elementType} in the loaded graph; expected int64 or int32.");
    }

    /// <summary>
    /// Builds a boolean-mask input tensor, matching whichever of bool/float32 the session
    /// declares for <paramref name="name"/>.
    /// </summary>
    private static NamedOnnxValue MaskInput(InferenceSession session, string name, bool[] data, params int[] shape)
    {
        var elementType = session.InputMetadata[name].ElementType;
        if (elementType == typeof(bool))
            return NamedOnnxValue.CreateFromTensor(name, new DenseTensor<bool>(data, shape));
        if (elementType == typeof(float))
        {
            var floats = new float[data.Length];
            for (var i = 0; i < data.Length; i++) floats[i] = data[i] ? 1f : 0f;
            return NamedOnnxValue.CreateFromTensor(name, new DenseTensor<float>(floats, shape));
        }

        throw new InvalidOperationException(
            $"input '{name}' is declared as {elementType} in the loaded graph; expected bool or float32.");
    }

    /// <summary>
    /// Builds the <c>qtype</c> input, reading its declared rank from session metadata rather than
    /// hardcoding it per layout: shape <c>[batch]</c> when the graph declares rank 1 (today's
    /// fused export), or <c>[batch, 1]</c> when it declares rank 2 (the split export's head).
    /// </summary>
    private static NamedOnnxValue QTypeInput(InferenceSession session, long[] data, int batchCount)
    {
        var rank = session.InputMetadata["qtype"].Dimensions.Length;
        var shape = rank >= 2 ? new[] { batchCount, 1 } : new[] { batchCount };
        return LongInput(session, "qtype", data, shape);
    }

    private static float[] Output(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results, string name)
    {
        foreach (var value in results)
            if (string.Equals(value.Name, name, StringComparison.Ordinal))
                return value.AsEnumerable<float>().ToArray();
        throw new InvalidOperationException($"the model produced no '{name}' output");
    }

    private static Tensor<float> OutputTensor(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results, string name)
    {
        foreach (var value in results)
            if (string.Equals(value.Name, name, StringComparison.Ordinal))
                return value.AsTensor<float>();
        throw new InvalidOperationException($"the encoder produced no '{name}' output");
    }

    /// <inheritdoc/>
    public void Dispose()
    {
        // Interlocked.Exchange is the standard pattern for a one-shot teardown: it atomically
        // sets the flag and returns the old value, so the cleanup runs exactly once even when
        // multiple threads race to dispose.
        if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
        if (_ownsTokenizer && _tokenizer is IDisposable disposable) disposable.Dispose();
        _fusedSession?.Dispose();
        _encoderSession?.Dispose();
        _headSession?.Dispose();
    }
}