Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

51.3 KB
Raw Blame
//! Answers typed questions about a piece of state in one forward pass. Ports the inference path
//! of Python's `laya.Agent` (`laya/agent.py`).
//!
//! Loading the model is expensive and the session is reusable, so create one engine and keep it:
//! it is safe to share across threads (`Arc<LayaEngine>`) and works well as a singleton. Every
//! question in a [`QuestionSet`] is answered by a single batched inference call, matching
//! Python's single forward pass — asking five questions costs roughly what asking one does.
//!
//! ## `Session::run` needs `&mut self`
//!
//! `ort` 2.x's `Session::run` takes `&mut self`, so the ONNX session is held in a [`Mutex`] here
//! so [`LayaEngine`] itself can stay lock-free everywhere else and still be `Send + Sync`.
//! Tokenization needs no such lock: `tokenizers::Tokenizer::encode` takes `&self`.

use crate::answers::{
    ActionInfo, Answer, ChoiceAnswer, LayaResult, NoulAnswer, ScanUsage, ScoreAnswer, StateUsage,
    Usage,
};
use crate::batch_prediction::{self, BatchOptions};
use crate::calibration::Calibration;
use crate::collator::{CollatedBatch, Collator, SequenceItem};
use crate::confidence::apply_confidence_gate;
use crate::error::{LayaError, Result};
use crate::lang_temperatures::LangTemperatures;
use crate::laya_config::{Deferred, LayaConfig};
use crate::laya_options::{LayaCheckpoint, LayaExecutionProvider, LayaOptions};
use crate::long_prediction::{self, LongOptions};
use crate::model_artifacts::{ModelArtifacts, ModelPaths};
use crate::predict_options::PredictOptions;
use crate::questions::{Question, QuestionSet};
use crate::sequence_builder::SequenceBuilder;
use crate::tokenization::{HfTokenizer, LayaTokenizer};
use indexmap::IndexMap;
use ort::session::Session;
use ort::session::builder::SessionBuilder;
use ort::value::{DynValue, Tensor, TensorElementType, ValueType};
use serde::Serialize;
use serde_json::Value;
use std::collections::HashMap;
use std::path::Path;
use std::sync::Mutex;

const MODEL_NAME: &str = "laya-rl-agent";

/// The raw graph outputs of one forward pass, before calibration: the logits and action logits
/// tuple the engine's internal run step produces.
///
/// Public (rather than crate-private) only so the integration test suite — a separate crate that
/// links against this one — can drive the ONNX graph directly for the shape sweep and the raw
/// logits comparison, without loading a second copy of the ~1.3 GB weights just to reach a
/// crate-private API. `#[doc(hidden)]` keeps it out of the published API surface; Rust has no
/// visibility scoped to "this crate's own tests", so a hidden `pub` item is the closest match
/// without a whole extra `test-internals` feature.
#[doc(hidden)]
#[derive(Debug, Clone)]
pub struct LayaOutput {
    /// Flattened `[batch.count, batch.marker_count]` raw scores, one per option marker.
    pub logits: Vec<f32>,
    /// Flattened `[batch.count, n_act]` raw action-head scores.
    pub act_logits: Vec<f32>,
}

/// One state's tokenized question rows, plus what its truncation report says, ready to collate.
struct EncodedState {
    items: Vec<SequenceItem>,
    usage: StateUsage,
}

impl EncodedState {
    /// The longest question row, the key `sort_by_length` orders by.
    fn longest_row(&self) -> usize {
        self.items
            .iter()
            .map(|item| item.ids.len())
            .max()
            .unwrap_or(0)
    }
}

/// Python's `TypeError` for a `None` state: scoring it would answer a decision about the text
/// `"null"`, byte-identical to passing `"null"` and at full confidence.
pub(crate) fn refuse_null_states(states: &[Value]) -> Result<()> {
    if states.iter().any(Value::is_null) {
        return Err(LayaError::InvalidState(
            "state must not be None; pass a string, dict, or list".to_string(),
        ));
    }
    Ok(())
}

fn question_ids(questions: &QuestionSet) -> Vec<String> {
    questions.ids().map(str::to_string).collect()
}

/// The loaded ONNX session(s) for whichever [`crate::model_artifacts::ModelLayout`] the artifact
/// directory used. `Fused` holds the single graph; `Split` holds the encoder and head graphs that
/// [`LayaEngine::run`] chains together, feeding the encoder's `last_hidden_state` output into the
/// head as its `hidden_states` input.
enum ModelSessions {
    Fused(Session),
    Split { encoder: Session, head: Session },
}

/// Answers typed questions about a piece of state in one forward pass. See the module docs.
pub struct LayaEngine {
    sessions: Mutex<ModelSessions>,
    tokenizer: Box<dyn LayaTokenizer>,
    config: LayaConfig,
    artifacts: ModelArtifacts,
    /// What Python only discovers at prediction time (a `temperature` list shorter than three).
    deferred: Deferred,
    /// Python's `self.lang_temperatures`: the resolved per-language overrides.
    lang_temperatures: LangTemperatures,
}

/// What one call may change about how its states are scored: Python's `lang`, `max_len` and
/// `head_max_len` arguments, the budgets already resolved against the checkpoint's own.
#[derive(Clone, Copy)]
struct CallParams<'a> {
    lang: Option<&'a str>,
    max_len: i64,
    head_max_len: i64,
}

impl LayaEngine {
    /// Loads an engine, resolving the artifact directory from `options` (see
    /// [`ModelArtifacts::resolve`] for the full resolution order: an explicit directory,
    /// `LAYA_ONNX_DIR`, `LAYA_ONNX_ROOT`, the local cache, then an opt-in download).
    ///
    /// Returns [`LayaError::CheckpointMismatch`] when the loaded config's encoder contradicts the
    /// requested [`LayaOptions::checkpoint`] — this only happens when the artifact was resolved
    /// through a checkpoint-aware path (`LAYA_ONNX_ROOT`, cache, or download); an explicit
    /// `model_directory` is trusted as-is.
    #[track_caller]
    pub fn create(options: LayaOptions) -> Result<Self> {
        let artifacts = ModelArtifacts::resolve(&options)?;
        let (config, mut deferred) =
            LayaConfig::load_deferred(&artifacts.config_path, &options.lang_temperatures)?;
        let lang_temperatures = std::mem::take(&mut deferred.lang_temperatures);

        if let Some(expected_checkpoint) = artifacts.checkpoint {
            validate_encoder(
                config.encoder.as_deref(),
                expected_checkpoint,
                &artifacts.directory,
            )?;
        }

        let sessions = match &artifacts.model {
            ModelPaths::Fused { model_path } => {
                ModelSessions::Fused(build_session(&options, model_path)?)
            }
            ModelPaths::Split {
                encoder_path,
                head_path,
            } => ModelSessions::Split {
                encoder: build_session(&options, encoder_path)?,
                head: build_session(&options, head_path)?,
            },
        };
        let tokenizer = HfTokenizer::from_file_with_config(
            &artifacts.tokenizer_path,
            artifacts.tokenizer_config_path.as_deref(),
        )?;

        Ok(LayaEngine {
            sessions: Mutex::new(sessions),
            tokenizer: Box::new(tokenizer),
            config,
            artifacts,
            deferred,
            lang_temperatures,
        })
    }

    /// Loads an engine from an artifact directory containing `model.onnx` directly, skipping
    /// every other resolution step.
    #[track_caller]
    pub fn from_directory(directory: impl AsRef<Path>) -> Result<Self> {
        Self::create(LayaOptions {
            model_directory: Some(directory.as_ref().to_path_buf()),
            ..Default::default()
        })
    }

    /// The loaded checkpoint's configuration.
    pub fn config(&self) -> &LayaConfig {
        &self.config
    }

    /// The resolved per-language temperatures ([`LayaOptions::lang_temperatures`]); empty when
    /// none were given.
    pub fn lang_temperatures(&self) -> &LangTemperatures {
        &self.lang_temperatures
    }

    /// Where the loaded artifacts came from.
    pub fn artifacts(&self) -> &ModelArtifacts {
        &self.artifacts
    }

    /// Which checkpoint variant was loaded, or `None` when loaded from an explicit directory
    /// where no checkpoint was inferred.
    pub fn checkpoint(&self) -> Option<LayaCheckpoint> {
        self.artifacts.checkpoint
    }

    /// Answer every question in `questions` about `state`.
    ///
    /// `state` accepts anything convertible to a [`serde_json::Value`] — a `&str`/`String` is used
    /// verbatim, anything else is serialized the way Python's `serialize_state` does (see
    /// [`crate::python_json::PythonJson::state`]). For a type that only implements
    /// [`serde::Serialize`] (rather than `Into<Value>`), use [`Self::predict_serialize`] instead.
    ///
    /// Like Python's `system_one`, this is a one-state [`Self::predict_batch`]: a `null` state is
    /// [`LayaError::InvalidState`], and no questions give an empty result with zero usage.
    ///
    /// Returns [`LayaError::NoMarkers`] if a question's options cannot fit within
    /// `Config::head_max_len`, rather than silently scoring fewer options than it was asked about.
    pub fn predict(&self, state: impl Into<Value>, questions: &QuestionSet) -> Result<LayaResult> {
        self.predict_value(state.into(), questions, &PredictOptions::default())
    }

    /// [`Self::predict`] with per-call [`PredictOptions`]. Ports `system_one(min_confidence=...)`:
    /// with `options.min_confidence` set, every answer reports its gate state (see
    /// [`Answer::abstention`]); a threshold Python's `check_min_confidence` rejects is
    /// [`LayaError::InvalidMinConfidence`], checked before the state is, as in `predict_batch`.
    pub fn predict_with_options(
        &self,
        state: impl Into<Value>,
        questions: &QuestionSet,
        options: &PredictOptions,
    ) -> Result<LayaResult> {
        self.predict_value(state.into(), questions, options)
    }

    /// [`Self::predict`], for a state that implements [`serde::Serialize`] but not
    /// `Into<serde_json::Value>` (e.g. a `#[derive(Serialize)]` struct).
    pub fn predict_serialize(
        &self,
        state: &impl Serialize,
        questions: &QuestionSet,
    ) -> Result<LayaResult> {
        let value = serde_json::to_value(state)?;
        self.predict_value(value, questions, &PredictOptions::default())
    }

    /// Python's `system_one` is `predict_batch([state], ...)`, so this is a one-state batch: a
    /// `null` state is refused and no questions give an empty result.
    fn predict_value(
        &self,
        state: Value,
        questions: &QuestionSet,
        options: &PredictOptions,
    ) -> Result<LayaResult> {
        let min_confidence = options.min_confidence.as_ref();
        if let Some(min_confidence) = min_confidence {
            min_confidence.validate()?;
        }
        let states = [state];
        refuse_null_states(&states)?;
        let params = self.call_params(
            options.lang.as_deref(),
            options.max_len,
            options.head_max_len,
        );
        let mut results = self.predict_states(&states, questions, params)?;
        apply_confidence_gate(&mut results, min_confidence);
        results
            .pop()
            .ok_or_else(|| LayaError::Onnx("internal error: no result for the state".to_string()))
    }

    /// Scan a state longer than one sequence in overlapping token windows and keep, per question,
    /// the most confident window. Ports `ONNXAgent.predict_long` with `aggregate="auto"`:
    ///
    /// * `noul`: the window with the greatest P(true);
    /// * `choice` and `score`: the window with the greatest `answer_confidence`;
    /// * ties go to the first window.
    ///
    /// The returned probabilities are the deciding window's, **not** a calibrated number for the
    /// whole document. Each answer's [`Answer::window`] names that window, and
    /// [`LayaResult::scan_usage`] / [`LayaResult::state_usage`] sum the scan.
    ///
    /// A state the questions leave room for (or, with an explicit [`LongOptions::window`], a state
    /// that fits that window) takes the ordinary [`Self::predict`] pass: no scan, one window, no
    /// window attribution. Otherwise each window is decoded back to text (special tokens kept, as
    /// Python's default `decode`) and scored as an ordinary state, so a window's tokens may differ
    /// slightly from the original after re-encoding, exactly as in Python. Windows go through the
    /// model together unless [`LongOptions::batch_size`] or the automatic cap chunks them.
    ///
    /// Per-call hooks are not part of this port, and, as in Python, `predict_long` takes no
    /// `min_confidence`, `max_len` or `head_max_len`; [`LongOptions::lang`] selects the
    /// per-language temperatures for every window. Errors: [`LayaError::UnsupportedAggregate`],
    /// [`LayaError::InvalidWindow`] (no room for the state, or a stride that leaves gaps), plus
    /// everything [`Self::predict`] returns.
    #[track_caller]
    pub fn predict_long(
        &self,
        state: impl Into<Value>,
        questions: &QuestionSet,
        options: &LongOptions,
    ) -> Result<LayaResult> {
        self.predict_long_value(state.into(), questions, options)
    }

    /// [`Self::predict_long`], for a state that implements [`serde::Serialize`] but not
    /// `Into<serde_json::Value>`.
    #[track_caller]
    pub fn predict_long_serialize(
        &self,
        state: &impl Serialize,
        questions: &QuestionSet,
        options: &LongOptions,
    ) -> Result<LayaResult> {
        let value = serde_json::to_value(state)?;
        self.predict_long_value(value, questions, options)
    }

    #[track_caller]
    fn predict_long_value(
        &self,
        state: Value,
        questions: &QuestionSet,
        options: &LongOptions,
    ) -> Result<LayaResult> {
        // Python checks the state before the aggregate mode.
        refuse_null_states(std::slice::from_ref(&state))?;
        options.check_aggregate()?;
        let tokenizer = self.tokenizer.as_ref();
        let explicit_window = options.window.unwrap_or(0);
        let (budget, warnings) = long_prediction::window_budget_warned(
            tokenizer,
            questions,
            self.config.max_len,
            self.config.head_max_len,
            explicit_window,
            options.stride.unwrap_or(0),
        )?;
        long_prediction::emit_warnings(&warnings, std::panic::Location::caller());
        let state_ids = SequenceBuilder::encode_state(tokenizer, &state)?;
        let params = self.call_params(options.lang.as_deref(), None, None);

        // `predict` reads the whole room the questions leave, so the default window must not force
        // a scan there; an explicit window still bounds the single pass (Python 3e9bda30).
        let one_pass = if explicit_window > 0 {
            budget.window
        } else {
            budget.room
        };
        if state_ids.len() <= one_pass {
            let single = self.predict_states(&[state], questions, params)?.remove(0);
            let truncated_windows = single.state_usage().is_some_and(|u| u.truncated) as usize;
            return Ok(single.with_scan_usage(ScanUsage {
                windows: 1,
                truncated_windows,
            }));
        }

        let spans = long_prediction::spans(state_ids.len(), budget.window, budget.stride);
        let windows = spans
            .iter()
            .map(|span| {
                tokenizer
                    .decode(&state_ids[span.token_start..span.token_end])
                    .map(Value::String)
            })
            .collect::<Result<Vec<Value>>>()?;
        let cap = long_prediction::batch_cap(
            spans.len(),
            budget.window,
            long_prediction::default_window(self.config.max_len, self.config.head_max_len),
            options.batch_size.unwrap_or(0),
        )
        .unwrap_or(spans.len());
        let mut results = Vec::with_capacity(spans.len());
        for chunk in windows.chunks(cap) {
            results.extend(self.predict_states(chunk, questions, params)?);
        }
        Ok(long_prediction::aggregate(
            MODEL_NAME, questions, &results, &spans,
        ))
    }

    /// Answer every question about every state, `options.batch_size` states per forward pass.
    /// Ports `ONNXAgent.predict_batch`: the same questions are scored over each state, and the
    /// result for state `i` is the one `predict` gives it, in input order.
    ///
    /// * `batch_size` of `None` or `0` sends every state in one pass; see [`BatchOptions`].
    /// * `sort_by_length` groups similarly sized states inside windows of eight batches, and only
    ///   with a `batch_size` above one and below the number of states.
    /// * No states give an empty list. No questions give one empty result per state, with no
    ///   state usage, as Python does.
    ///
    /// * `min_confidence` gates every answer of every result, batch-wide; see [`BatchOptions`].
    ///
    /// A threshold Python's `check_min_confidence` rejects is [`LayaError::InvalidMinConfidence`],
    /// checked first. A `null` state is refused with [`LayaError::InvalidState`] (Python's
    /// `TypeError`: scoring it would answer a decision about the text `"null"`). Everything else
    /// [`Self::predict`] returns applies per state, and the first error fails the whole call.
    pub fn predict_batch(
        &self,
        states: &[Value],
        questions: &QuestionSet,
        options: &BatchOptions,
    ) -> Result<Vec<LayaResult>> {
        let min_confidence = options.min_confidence.as_ref();
        if let Some(min_confidence) = min_confidence {
            min_confidence.validate()?;
        }
        refuse_null_states(states)?;
        let mut results = self.predict_batch_ungated(states, questions, options)?;
        apply_confidence_gate(&mut results, min_confidence);
        Ok(results)
    }

    fn predict_batch_ungated(
        &self,
        states: &[Value],
        questions: &QuestionSet,
        options: &BatchOptions,
    ) -> Result<Vec<LayaResult>> {
        if states.is_empty() {
            return Ok(Vec::new());
        }
        let ids = question_ids(questions);
        if ids.is_empty() {
            return Ok(self.empty_results(states.len()));
        }
        let params = self.call_params(
            options.lang.as_deref(),
            options.max_len,
            options.head_max_len,
        );
        batch_prediction::run(
            states.len(),
            options,
            |range| {
                states[range]
                    .iter()
                    .map(|state| self.encode_state(state, &ids, questions, params))
                    .collect()
            },
            EncodedState::longest_row,
            |batch| self.infer_encoded(batch, &ids, questions, params),
        )
    }

    /// [`Self::predict_batch`], for states that implement [`serde::Serialize`] but not
    /// `Into<serde_json::Value>`.
    pub fn predict_batch_serialize<S: Serialize>(
        &self,
        states: &[S],
        questions: &QuestionSet,
        options: &BatchOptions,
    ) -> Result<Vec<LayaResult>> {
        let values = states
            .iter()
            .map(serde_json::to_value)
            .collect::<std::result::Result<Vec<Value>, _>>()?;
        self.predict_batch(&values, questions, options)
    }

    /// Scores every question against every state in one collate and one forward pass, returning
    /// one result (with its own state usage) per state, in order. With no questions each result is
    /// empty and carries no state usage. `predict` is this with one state.
    fn predict_states(
        &self,
        states: &[Value],
        questions: &QuestionSet,
        params: CallParams<'_>,
    ) -> Result<Vec<LayaResult>> {
        let ids = question_ids(questions);
        if ids.is_empty() {
            return Ok(self.empty_results(states.len()));
        }
        let encoded = states
            .iter()
            .map(|state| self.encode_state(state, &ids, questions, params))
            .collect::<Result<Vec<_>>>()?;
        self.infer_encoded(encoded, &ids, questions, params)
    }

    /// `max_len = self.cfg.get("max_len", 512) if max_len is None else max_len`, and the same for
    /// `head_max_len`: a per-call value is used as given, whatever it is.
    fn call_params<'a>(
        &self,
        lang: Option<&'a str>,
        max_len: Option<i64>,
        head_max_len: Option<i64>,
    ) -> CallParams<'a> {
        CallParams {
            lang,
            max_len: max_len.unwrap_or_else(|| SequenceBuilder::signed(self.config.max_len)),
            head_max_len: head_max_len
                .unwrap_or_else(|| SequenceBuilder::signed(self.config.head_max_len)),
        }
    }

    /// The result for a state when there are no questions to answer.
    fn empty_results(&self, count: usize) -> Vec<LayaResult> {
        (0..count)
            .map(|_| LayaResult::new(MODEL_NAME, Vec::new(), HashMap::new(), Usage::default()))
            .collect()
    }

    /// One state's question rows and the truncation report for them, tokenizing the state for
    /// each question. Ports `ONNXAgent._encode_state`.
    fn encode_state(
        &self,
        state: &Value,
        ids: &[String],
        questions: &QuestionSet,
        params: CallParams<'_>,
    ) -> Result<EncodedState> {
        let mut state_usage = StateUsage {
            state_tokens: 0,
            state_tokens_dropped: 0,
            truncated: false,
            truncated_questions: Vec::new(),
            options: IndexMap::new(),
        };

        // A conversation-turn list is serialized newest-last (see `PythonJson::state`), so the
        // default right-truncation of an over-long state would silently drop the newest turn.
        // Truncate from the left for lists so the most recent turn survives instead. Ports
        // `Agent._encode_state`'s `truncate_left = isinstance(state, list)` (laya/agent.py,
        // laya 0.3.21) — 0.3.6 never set this, so every state was right-truncated regardless
        // of shape.
        let truncate_left = matches!(state, Value::Array(_));

        let mut items = Vec::with_capacity(ids.len());
        // The tokenizer here needs no lock: tokenizers::Tokenizer::encode takes &self.
        for id in ids {
            let question = &questions[id.as_str()];
            let (seq, markers, stats) = SequenceBuilder::build_with_overrides(
                self.tokenizer.as_ref(),
                state,
                question,
                params.max_len,
                params.head_max_len,
                truncate_left,
            )?;
            state_usage.state_tokens = stats.tokens;
            state_usage.state_tokens_dropped = state_usage.state_tokens_dropped.max(stats.dropped);
            if stats.dropped > 0 {
                state_usage.truncated = true;
                state_usage.truncated_questions.push(id.clone());
            }
            if let Some(options) = stats.options {
                state_usage.options.insert(id.clone(), options);
            }

            // A marker dropped for landing past max_len means an option is not in the
            // sequence at all, so its logit would score whatever token happens to sit at
            // position 0.
            let expected_markers = SequenceBuilder::render_options(question).len();
            if markers.len() != expected_markers {
                return Err(LayaError::NoMarkers {
                    id: id.clone(),
                    head_max_len: params.head_max_len,
                    actual_markers: markers.len(),
                    expected_markers,
                    max_len: params.max_len,
                });
            }

            items.push(SequenceItem::new(seq, markers, question.question_type()));
        }
        Ok(EncodedState {
            items,
            usage: state_usage,
        })
    }

    /// Collates the encoded states' rows into one batch, runs one forward pass and decodes one
    /// result per state, in the order given. `ids` must be non-empty.
    fn infer_encoded(
        &self,
        encoded: Vec<EncodedState>,
        ids: &[String],
        questions: &QuestionSet,
        params: CallParams<'_>,
    ) -> Result<Vec<LayaResult>> {
        let state_count = encoded.len();
        let mut items = Vec::with_capacity(ids.len() * state_count);
        let mut state_usages = Vec::with_capacity(state_count);
        for state in encoded {
            items.extend(state.items);
            state_usages.push(state.usage);
        }

        let batch = Collator::collate(&items, self.tokenizer.special().pad_id)?;
        let output = self.run(&batch)?;

        // n_act is the act_logits column count, derived from the flat output length rather than
        // hardcoded so this stays correct if the graph is re-exported with a wider action head.
        let n_act = output.act_logits.len() / batch.count;

        let mut results = Vec::with_capacity(state_count);
        for (state_index, state_usage) in state_usages.into_iter().enumerate() {
            let offset = state_index * ids.len();
            let mut answers = HashMap::with_capacity(ids.len());
            for (q, id) in ids.iter().enumerate() {
                let row = offset + q;
                let question = &questions[id.as_str()];
                let k = batch.marker_counts[row];
                // `self.temperature_by_options.get(bucket, self.temperature[qt])` evaluates the
                // default first, so a missing list entry fails even when the bucket is found.
                if self
                    .deferred
                    .short_temperature
                    .is_some_and(|len| question.question_type() as usize >= len)
                {
                    return Err(LayaError::InvalidConfig(
                        "IndexError: list index out of range".to_string(),
                    ));
                }
                // A language in the table replaces both the bucket map and the per-type
                // temperature, with no fallback to the checkpoint's bucket map; there is no
                // histogram binning in this port to skip for it.
                let temperature = self
                    .lang_temperatures
                    .temperature_for(params.lang, question.question_type(), k)
                    .unwrap_or_else(|| {
                        Calibration::resolve_temperature(&self.config, question.question_type(), k)
                    });
                let row_start = row * batch.marker_count;
                let row_logits = &output.logits[row_start..row_start + batch.marker_count];
                let p = Calibration::probabilities(row_logits, k, temperature);

                let act_start = row * n_act;
                let act_row = &output.act_logits[act_start..act_start + n_act];
                let action = ActionInfo {
                    act_probability: Calibration::round4(Calibration::softmax(act_row)[0]),
                };

                answers.insert(id.clone(), build_answer(question, &p, action));
            }

            // This state's own non-padding tokens; with one state it equals `batch.input_tokens`.
            let input_tokens: usize = items[offset..offset + ids.len()]
                .iter()
                .map(|item| item.ids.len())
                .sum();
            results.push(
                LayaResult::new(
                    MODEL_NAME,
                    ids.to_vec(),
                    answers,
                    Usage {
                        input_tokens: input_tokens as u32,
                        output_tokens: 0,
                    },
                )
                .with_state_usage(state_usage),
            );
        }
        Ok(results)
    }

    /// One forward pass over a collated batch, returning the raw graph outputs. See the
    /// [`LayaOutput`] docs for why this is `pub` rather than crate-private.
    ///
    /// For the fused layout this is a single `session.run`. For the split layout, the encoder
    /// runs first over `input_ids`/`attention_mask`, and its `last_hidden_state` output is fed
    /// into the head session's `hidden_states` input alongside `marker_pos`/`marker_mask`/
    /// `qtype`/`attention_mask` — one copy of the hidden-state tensor is unavoidable (it must
    /// outlive the encoder's `SessionOutputs` borrow, which is tied to the encoder session's
    /// exclusive borrow, itself released before the head session can be borrowed mutably from the
    /// same lock), but no other tensor is copied more than once.
    #[doc(hidden)]
    pub fn run(&self, batch: &CollatedBatch) -> Result<LayaOutput> {
        // Session::run needs &mut self (see the module docs), so the lock is held for both the
        // run(s) below and the output extraction; tensor construction and calibration in the
        // caller happen outside it.
        let mut sessions = self.sessions.lock().expect("onnx session mutex poisoned");
        match &mut *sessions {
            ModelSessions::Fused(session) => {
                let inputs = build_inputs(session, batch, None)?;
                let outputs = session.run(inputs).map_err(onnx_err)?;
                Ok(LayaOutput {
                    logits: extract_f32(&outputs, "logits")?,
                    act_logits: extract_f32(&outputs, "act_logits")?,
                })
            }
            ModelSessions::Split { encoder, head } => {
                let encoder_inputs = build_inputs(encoder, batch, None)?;
                let encoder_outputs = encoder.run(encoder_inputs).map_err(onnx_err)?;
                let hidden_states = extract_hidden_states(&encoder_outputs, "last_hidden_state")?;
                drop(encoder_outputs);

                let head_inputs = build_inputs(head, batch, Some(hidden_states))?;
                let head_outputs = head.run(head_inputs).map_err(onnx_err)?;
                Ok(LayaOutput {
                    logits: extract_f32(&head_outputs, "logits")?,
                    act_logits: extract_f32(&head_outputs, "act_logits")?,
                })
            }
        }
    }
}

// ── generic, metadata-driven input binding (shared by both layouts) ─────────────────────────

/// The encoder's `last_hidden_state` output, extracted to owned data so it can outlive the
/// encoder's `SessionOutputs` borrow and be fed into the head session as its `hidden_states`
/// input. One copy (the `try_extract_tensor` call in [`extract_hidden_states`]); building the
/// head's input tensor from it below does not copy again.
struct HiddenStates {
    shape: Vec<i64>,
    data: Vec<f32>,
}

impl HiddenStates {
    fn into_dyn_value(self, ty: TensorElementType) -> Result<DynValue> {
        match ty {
            TensorElementType::Float32 => Ok(Tensor::from_array((self.shape, self.data))
                .map_err(onnx_err)?
                .into_dyn()),
            other => Err(LayaError::Onnx(format!(
                "the head model declares 'hidden_states' as {other:?}, but the encoder produces \
                 Float32"
            ))),
        }
    }
}

fn extract_hidden_states(
    outputs: &ort::session::SessionOutputs,
    name: &str,
) -> Result<HiddenStates> {
    let value = outputs
        .get(name)
        .ok_or_else(|| LayaError::Onnx(format!("the encoder produced no '{name}' output")))?;
    let (shape, data) = value.try_extract_tensor::<f32>().map_err(onnx_err)?;
    Ok(HiddenStates {
        shape: shape.to_vec(),
        data: data.to_vec(),
    })
}

/// Builds an integer-sourced tensor (`input_ids`, `attention_mask`, `marker_pos`, `qtype`),
/// casting `data` to whichever element type the session declared.
fn int_tensor(shape: Vec<i64>, data: &[i64], ty: TensorElementType) -> Result<DynValue> {
    match ty {
        TensorElementType::Int64 => Ok(Tensor::from_array((shape, data.to_vec()))
            .map_err(onnx_err)?
            .into_dyn()),
        TensorElementType::Int32 => {
            let data: Vec<i32> = data.iter().map(|&v| v as i32).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        TensorElementType::Float32 => {
            let data: Vec<f32> = data.iter().map(|&v| v as f32).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        TensorElementType::Bool => {
            let data: Vec<bool> = data.iter().map(|&v| v != 0).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        other => Err(LayaError::Onnx(format!(
            "unsupported ONNX element type {other:?} for an integer-valued input"
        ))),
    }
}

/// Builds a boolean-sourced tensor (`marker_mask`), casting to whichever element type the session
/// declared.
fn bool_tensor(shape: Vec<i64>, data: &[bool], ty: TensorElementType) -> Result<DynValue> {
    match ty {
        TensorElementType::Bool => Ok(Tensor::from_array((shape, data.to_vec()))
            .map_err(onnx_err)?
            .into_dyn()),
        TensorElementType::Int64 => {
            let data: Vec<i64> = data.iter().map(|&b| b as i64).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        TensorElementType::Int32 => {
            let data: Vec<i32> = data.iter().map(|&b| b as i32).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        TensorElementType::Float32 => {
            let data: Vec<f32> = data.iter().map(|&b| if b { 1.0 } else { 0.0 }).collect();
            Ok(Tensor::from_array((shape, data))
                .map_err(onnx_err)?
                .into_dyn())
        }
        other => Err(LayaError::Onnx(format!(
            "unsupported ONNX element type {other:?} for a boolean-valued input"
        ))),
    }
}

/// Produces the value for one declared input by name, or `None` if `name` is not one this engine
/// knows how to provide (which [`build_inputs`] turns into a clear error rather than silently
/// skipping it). `qtype`'s shape follows the session's own declared rank: `[batch, 1]` when the
/// graph declares it rank 2 (the split layout's head, whose exporter's dummy input is `[[0]] *
/// batch`), `[batch]` otherwise (the fused layout) — decided from metadata, not hardcoded per
/// layout, so either graph shape is honored even if a future export changes it.
fn value_for_input(
    name: &str,
    ty: TensorElementType,
    rank: usize,
    batch: &CollatedBatch,
    hidden_states: &mut Option<HiddenStates>,
) -> Result<Option<DynValue>> {
    let count = batch.count as i64;
    match name {
        "input_ids" => Ok(Some(int_tensor(
            vec![count, batch.sequence_length as i64],
            &batch.input_ids,
            ty,
        )?)),
        "attention_mask" => Ok(Some(int_tensor(
            vec![count, batch.sequence_length as i64],
            &batch.attention_mask,
            ty,
        )?)),
        "marker_pos" => Ok(Some(int_tensor(
            vec![count, batch.marker_count as i64],
            &batch.marker_pos,
            ty,
        )?)),
        "marker_mask" => Ok(Some(bool_tensor(
            vec![count, batch.marker_count as i64],
            &batch.marker_mask,
            ty,
        )?)),
        "qtype" => {
            let shape = if rank >= 2 {
                vec![count, 1]
            } else {
                vec![count]
            };
            Ok(Some(int_tensor(shape, &batch.qtype, ty)?))
        }
        "hidden_states" => {
            let hidden = hidden_states.take().ok_or_else(|| {
                LayaError::Onnx(
                    "the head model declares a 'hidden_states' input, but no encoder output was \
                     available to provide it"
                        .to_string(),
                )
            })?;
            Ok(Some(hidden.into_dyn_value(ty)?))
        }
        _ => Ok(None),
    }
}

/// Binds only the inputs `session` actually declares (per item 3 of the design: never a fixed,
/// hardcoded list), reading each one's declared dtype and rank from session metadata so the
/// tensor built for it matches exactly what the graph expects. `hidden_states` is consumed (via
/// [`Option::take`]) the one time a declared input is actually named `hidden_states`, so the
/// extracted encoder output is moved into its tensor rather than copied again.
fn build_inputs(
    session: &Session,
    batch: &CollatedBatch,
    hidden_states: Option<HiddenStates>,
) -> Result<Vec<(String, DynValue)>> {
    let mut hidden_states = hidden_states;
    let mut inputs = Vec::with_capacity(session.inputs().len());
    for outlet in session.inputs() {
        let name = outlet.name();
        let (ty, rank) = match outlet.dtype() {
            ValueType::Tensor { ty, shape, .. } => (*ty, shape.len()),
            other => {
                return Err(LayaError::Onnx(format!(
                    "input '{name}' has unsupported (non-tensor) type {other:?}"
                )));
            }
        };
        match value_for_input(name, ty, rank, batch, &mut hidden_states)? {
            Some(value) => inputs.push((name.to_string(), value)),
            None => {
                return Err(LayaError::Onnx(format!(
                    "the model declares an input '{name}' that this engine does not know how to \
                     provide"
                )));
            }
        }
    }
    Ok(inputs)
}

// SAFETY-free: every field is itself Send + Sync (Mutex<ModelSessions> — Session is Send + Sync
// and so is the enum wrapping one or two of them —, Box<dyn LayaTokenizer> whose trait requires
// Send + Sync, and plain owned data in LayaConfig / ModelArtifacts), so this is
// the ordinary auto-derived impl, not an unsafe opt-in. Asserted here so a future field addition
// that breaks it fails to compile at this obvious spot rather than at some faraway `Arc<LayaEngine>`
// call site.
const _: fn() = || {
    fn assert_send_sync<T: Send + Sync>() {}
    assert_send_sync::<LayaEngine>();
};

fn build_answer(question: &Question, p: &[f64], action: ActionInfo) -> Answer {
    let k = p.len();
    let confidence = Calibration::round4(Calibration::confidence_from_probs(p, k));
    // Calibrated on every question type alike (`max(p)`), unlike `confidence` above. Ports
    // `answer_confidence` from `laya/common.py` (laya 0.3.21); see `Answer::answer_confidence`.
    let answer_confidence = Calibration::round4(Calibration::answer_confidence(p, k));

    match question {
        Question::Choice(choice) => {
            let labels: Vec<&str> = choice.labels().collect();
            // Python zips labels against probabilities, so a mismatch would silently drop the
            // tail rather than misalign; take the shorter for the same reason.
            let n = k.min(labels.len());
            let mut probabilities = IndexMap::with_capacity(n);
            for (label, &prob) in labels.iter().zip(p.iter()).take(n) {
                probabilities.insert((*label).to_string(), Calibration::round4(prob));
            }
            let chosen = labels
                .get(Calibration::arg_max(p))
                .copied()
                .unwrap_or_default()
                .to_string();
            Answer::Choice(ChoiceAnswer::new(
                chosen,
                probabilities,
                confidence,
                answer_confidence,
                action,
            ))
        }

        Question::Score(score) => {
            let probabilities: Vec<f64> = p.iter().copied().map(Calibration::round4).collect();
            let expected = Calibration::round4(Calibration::expected_score(p));
            Answer::Score(ScoreAnswer::new(
                expected,
                score
                    .levels()
                    .iter()
                    .map(|level| Value::String(crate::python_json::PythonJson::criterion(level)))
                    .collect(),
                probabilities,
                confidence,
                answer_confidence,
                action,
            ))
        }

        Question::Noul(_) => {
            // Noul reports the true-branch probability, and its confidence is the distance from a
            // coin flip rather than the entropy measure the other two types use. Over two options
            // max(p_true, 1 - p_true) equals max(p), so `answer_confidence` agrees with
            // `confidence` here even though the two differ for choice and score.
            let p_true = p[1];
            Answer::Noul(NoulAnswer::new(
                Calibration::round4(p_true),
                Calibration::round4(p_true.max(1.0 - p_true)),
                answer_confidence,
                action,
            ))
        }
    }
}

fn validate_encoder(
    actual_encoder: Option<&str>,
    checkpoint: LayaCheckpoint,
    dir: &Path,
) -> Result<()> {
    // Multilingual uses mmBERT; English and TypedDecisions both use ModernBERT-large. This can
    // only detect a family mismatch (mmBERT vs. ModernBERT), not distinguish English from
    // TypedDecisions, since both share the same encoder string.
    let expect_modern_bert = checkpoint != LayaCheckpoint::Multilingual;
    let got_modern_bert = actual_encoder
        .map(|e| e.to_ascii_lowercase().contains("modernbert"))
        .unwrap_or(false);

    if expect_modern_bert != got_modern_bert {
        let expected = if checkpoint == LayaCheckpoint::Multilingual {
            "jhu-clsp/mmBERT-base"
        } else {
            "answerdotai/ModernBERT-large"
        };
        return Err(LayaError::CheckpointMismatch(format!(
            "Checkpoint '{checkpoint:?}' expects encoder '{expected}' but the artifact at '{}' \
             declares encoder '{}'. The artifact directory may contain the wrong checkpoint.",
            dir.display(),
            actual_encoder.unwrap_or("(null)")
        )));
    }
    Ok(())
}

fn build_session(options: &LayaOptions, model_path: &Path) -> Result<Session> {
    let mut builder = Session::builder().map_err(onnx_err)?;
    if let Some(intra) = options.intra_op_threads {
        builder = builder
            .with_intra_threads(intra as usize)
            .map_err(onnx_err)?;
    }
    if let Some(inter) = options.inter_op_threads {
        builder = builder
            .with_inter_threads(inter as usize)
            .map_err(onnx_err)?;
    }
    let mut builder = apply_execution_provider(builder, options.execution_provider)?;

    // The path must point at model.onnx inside its own directory: this checkpoint is in ONNX
    // external-data format and the runtime resolves the ~1.29 GB model.onnx.data sidecar relative
    // to the graph file.
    builder.commit_from_file(model_path).map_err(onnx_err)
}

fn apply_execution_provider(
    builder: SessionBuilder,
    provider: LayaExecutionProvider,
) -> Result<SessionBuilder> {
    match provider {
        LayaExecutionProvider::Cpu => Ok(builder),

        LayaExecutionProvider::Cuda => {
            #[cfg(feature = "cuda")]
            {
                builder
                    .with_execution_providers([ort::ep::CUDA::default().build()])
                    .map_err(onnx_err)
            }
            #[cfg(not(feature = "cuda"))]
            {
                let _ = builder;
                Err(LayaError::Onnx(
                    "LayaExecutionProvider::Cuda was requested, but this crate was built \
                     without the `cuda` feature."
                        .to_string(),
                ))
            }
        }

        LayaExecutionProvider::DirectMl => {
            #[cfg(feature = "directml")]
            {
                builder
                    .with_execution_providers([ort::ep::DirectML::default().build()])
                    .map_err(onnx_err)
            }
            #[cfg(not(feature = "directml"))]
            {
                let _ = builder;
                Err(LayaError::Onnx(
                    "LayaExecutionProvider::DirectMl was requested, but this crate was built \
                     without the `directml` feature."
                        .to_string(),
                ))
            }
        }
    }
}

fn extract_f32(outputs: &ort::session::SessionOutputs, name: &str) -> Result<Vec<f32>> {
    let value = outputs
        .get(name)
        .ok_or_else(|| LayaError::Onnx(format!("the model produced no '{name}' output")))?;
    let (_, data) = value.try_extract_tensor::<f32>().map_err(onnx_err)?;
    Ok(data.to_vec())
}

fn onnx_err(e: impl std::fmt::Display) -> LayaError {
    LayaError::Onnx(e.to_string())
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn a_null_state_is_refused_with_pythons_text() {
        assert!(refuse_null_states(&[json!("a"), json!([null]), json!({})]).is_ok());
        let error = refuse_null_states(&[json!("a"), Value::Null]).unwrap_err();
        assert!(matches!(error, LayaError::InvalidState(_)), "{error:?}");
        assert_eq!(
            error.to_string(),
            "state must not be None; pass a string, dict, or list"
        );
    }

    #[test]
    fn score_legends_render_structured_levels_as_strings() {
        let question = Question::score(
            "severity",
            [
                json!({"label": "info", "cvss": 0.0}),
                json!({"label": "high", "cvss": 7.5}),
            ],
        )
        .unwrap();
        let Answer::Score(answer) = build_answer(
            &question,
            &[0.25, 0.75],
            ActionInfo {
                act_probability: 1.0,
            },
        ) else {
            panic!("score question must produce a score answer");
        };
        assert_eq!(
            answer.legend,
            vec![
                json!("{\"label\": \"info\", \"cvss\": 0.0}"),
                json!("{\"label\": \"high\", \"cvss\": 7.5}"),
            ],
        );
    }

    /// A tiny two-row batch, just enough shape/data to exercise `value_for_input` without a real
    /// ONNX session.
    fn tiny_batch() -> CollatedBatch {
        CollatedBatch {
            count: 2,
            sequence_length: 3,
            marker_count: 2,
            input_ids: vec![1, 2, 3, 4, 5, 6],
            attention_mask: vec![1, 1, 1, 1, 1, 0],
            marker_pos: vec![0, 1, 0, 1],
            marker_mask: vec![true, true, true, false],
            qtype: vec![7, 9],
            marker_counts: vec![2, 1],
            input_tokens: 5,
        }
    }

    fn tensor_shape(value: &DynValue) -> Vec<i64> {
        value
            .try_extract_tensor::<i64>()
            .map(|(shape, _)| shape.to_vec())
            .or_else(|_| {
                value
                    .try_extract_tensor::<i32>()
                    .map(|(shape, _)| shape.to_vec())
            })
            .expect("qtype should extract as an integer tensor")
    }

    #[test]
    fn qtype_rank_one_declared_yields_a_one_dimensional_shape() {
        let batch = tiny_batch();
        let mut hidden = None;
        let value = value_for_input("qtype", TensorElementType::Int64, 1, &batch, &mut hidden)
            .expect("qtype should be handled")
            .expect("qtype should produce a value");
        assert_eq!(tensor_shape(&value), vec![2]);
    }

    #[test]
    fn qtype_rank_two_declared_yields_a_batch_by_one_shape() {
        // The split layout's head declares qtype as rank 2 ([B, 1]), matching the exporter's
        // dummy input `[[0]] * batch`.
        let batch = tiny_batch();
        let mut hidden = None;
        let value = value_for_input("qtype", TensorElementType::Int64, 2, &batch, &mut hidden)
            .expect("qtype should be handled")
            .expect("qtype should produce a value");
        assert_eq!(tensor_shape(&value), vec![2, 1]);
    }

    #[test]
    fn qtype_rank_selection_is_independent_of_element_type() {
        // Rank drives the shape regardless of which integer dtype the graph declares.
        let batch = tiny_batch();
        let mut hidden = None;
        let value = value_for_input("qtype", TensorElementType::Int32, 2, &batch, &mut hidden)
            .expect("qtype should be handled")
            .expect("qtype should produce a value");
        assert_eq!(tensor_shape(&value), vec![2, 1]);
    }

    #[test]
    fn hidden_states_input_is_taken_exactly_once() {
        let batch = tiny_batch();
        let mut hidden = Some(HiddenStates {
            shape: vec![2, 3, 4],
            data: vec![0.0; 24],
        });
        let value = value_for_input(
            "hidden_states",
            TensorElementType::Float32,
            3,
            &batch,
            &mut hidden,
        )
        .expect("hidden_states should be handled")
        .expect("hidden_states should produce a value");
        assert!(
            hidden.is_none(),
            "hidden_states should be consumed by take()"
        );
        let (shape, _) = value
            .try_extract_tensor::<f32>()
            .expect("should extract as f32");
        assert_eq!(shape.to_vec(), vec![2, 3, 4]);
    }

    #[test]
    fn hidden_states_input_without_an_encoder_output_is_a_clear_error() {
        let batch = tiny_batch();
        let mut hidden = None;
        let err = value_for_input(
            "hidden_states",
            TensorElementType::Float32,
            3,
            &batch,
            &mut hidden,
        )
        .expect_err("must fail: no encoder output was ever provided");
        assert!(matches!(err, LayaError::Onnx(_)));
    }

    #[test]
    fn build_answer_reports_answer_confidence_alongside_confidence_for_choice() {
        // A skewed three-option distribution: answer_confidence is just the top probability,
        // while confidence is the normalized-entropy measure, so the two must differ here.
        let question = Question::choice(
            "h",
            [("a", Value::Null), ("b", Value::Null), ("c", Value::Null)],
        )
        .unwrap();
        let p = [0.7, 0.2, 0.1];
        let action = ActionInfo {
            act_probability: 1.0,
        };
        let answer = build_answer(&question, &p, action);

        assert_eq!(0.7, answer.answer_confidence());
        assert_ne!(answer.answer_confidence(), answer.confidence());
    }

    #[test]
    fn build_answer_answer_confidence_agrees_with_confidence_for_noul() {
        // Over two options max(p_true, 1 - p_true) equals max(p), so the two quantities coincide
        // for noul even though they differ for choice and score.
        let question = Question::noul("h", Value::Null, Value::Null);
        let p = [0.3, 0.7];
        let action = ActionInfo {
            act_probability: 1.0,
        };
        let answer = build_answer(&question, &p, action);

        assert_eq!(answer.answer_confidence(), answer.confidence());
        assert_eq!(0.7, answer.answer_confidence());
    }

    #[test]
    fn truncate_left_is_derived_from_the_state_shape() {
        // Ports `Agent._encode_state`'s `truncate_left = isinstance(state, list)` (laya 0.3.21):
        // a JSON array state truncates from the left, anything else from the right.
        assert!(matches!(json!([1, 2, 3]), Value::Array(_)));
        assert!(!matches!(json!({"a": 1}), Value::Array(_)));
        assert!(!matches!(json!("state text"), Value::Array(_)));
        assert!(!matches!(Value::Null, Value::Array(_)));
    }

    #[test]
    fn an_unknown_input_name_is_reported_as_none_not_an_error() {
        // `build_inputs` is the one that turns a `None` into a clear error; `value_for_input`
        // itself just reports "I don't know this name".
        let batch = tiny_batch();
        let mut hidden = None;
        let result = value_for_input(
            "some_future_input",
            TensorElementType::Int64,
            1,
            &batch,
            &mut hidden,
        )
        .expect("unknown names are not themselves an error");
        assert!(result.is_none());
    }
}