Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

28.2 KB
Raw Blame
use ort::session::Session;
use std::{fmt, path::Path};

use crate::error::{MlError, MlResult};

mod coreml_cache;
mod golden_test;
pub use golden_test::tooling as golden_tooling;
mod providers;
mod tensor;
mod webgpu;

pub(crate) fn set_webgpu_enabled(enabled: bool) {
    webgpu::set_enabled(enabled);
}

pub(crate) use ort::session::Session as SessionHandle;
pub(crate) use providers::ExecutionMode;
use tensor::run_golden_tensor;
pub(crate) use tensor::{
    BorrowedFloatTensor, FloatTensorData, PreparedF32Input, run_f32, run_i32_f32,
    with_prepared_float_output,
};

use providers::{ExecutionProvider, ProviderPlan};

#[derive(Clone, Debug)]
pub(crate) struct GpuOptions {
    pub(crate) subgraphs: bool,
    #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
    pub(crate) prefer_nhwc: bool,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum AccelerationValidation {
    GoldenRequired,
    Unvalidated,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct ProviderUsage {
    pub(crate) coreml: bool,
    pub(crate) webgpu: bool,
}

impl ProviderUsage {
    fn from_provider(provider: ExecutionProvider) -> Self {
        Self {
            coreml: provider == ExecutionProvider::CoreMl,
            webgpu: provider == ExecutionProvider::WebGpu,
        }
    }

    pub(crate) fn merge(self, other: Self) -> Self {
        Self {
            coreml: self.coreml || other.coreml,
            webgpu: self.webgpu || other.webgpu,
        }
    }
}

#[derive(Debug)]
pub(crate) struct OnnxSession {
    model_path: String,
    model_namespace: String,
    mode: ExecutionMode,
    validation: AccelerationValidation,
    gpu_options: Option<GpuOptions>,
    provider_plan: Option<ProviderPlan>,
    session: Option<(Session, ExecutionProvider)>,
    first_run_canary: Option<webgpu::ArmedCanary>,
    deferred_first_run_canary: bool,
}

impl OnnxSession {
    pub(crate) fn new(model_path: &str, model_namespace: &str, mode: ExecutionMode) -> Self {
        Self {
            model_path: model_path.to_string(),
            model_namespace: model_namespace.to_string(),
            mode,
            validation: AccelerationValidation::GoldenRequired,
            gpu_options: None,
            provider_plan: None,
            session: None,
            first_run_canary: None,
            deferred_first_run_canary: false,
        }
    }

    // Only for models whose output is not indexed and so cannot
    // silently poison stored data.
    pub(crate) fn with_unvalidated_acceleration(mut self) -> Self {
        self.validation = AccelerationValidation::Unvalidated;
        self
    }

    pub(crate) fn with_gpu_options(mut self, options: GpuOptions) -> Self {
        self.gpu_options = Some(options);
        self
    }

    pub(crate) fn model_path(&self) -> &str {
        &self.model_path
    }

    pub(crate) fn unload(&mut self) {
        self.provider_plan = None;
        self.session = None;
        self.deferred_first_run_canary = false;
        self.leave_first_run_canary_armed();
    }

    pub(crate) fn load(&mut self) -> MlResult<()> {
        self.ensure_loaded()?;
        self.defer_first_run_canary();
        Ok(())
    }

    #[cfg(test)]
    pub(crate) fn is_loaded(&self) -> bool {
        self.session.is_some()
    }

    #[cfg(test)]
    pub(crate) fn has_load_state(&self) -> bool {
        self.provider_plan.is_some() || self.session.is_some() || self.first_run_canary.is_some()
    }

    #[cfg(test)]
    pub(crate) fn initialize_load_state(&mut self) {
        self.provider_plan = Some(ProviderPlan::new(
            self.mode,
            &self.model_path,
            self.validation,
        ));
    }

    pub(crate) fn run<T>(
        &mut self,
        mut operation: impl FnMut(&mut Session) -> SessionRunResult<T>,
    ) -> MlResult<(T, ProviderUsage)> {
        loop {
            if let Err(error) = self.arm_deferred_first_run_canary() {
                if self.retry_after_provider_failure(&error) {
                    continue;
                }
                return Err(error.into_ml_error());
            }
            let (session, execution_provider) = self.ensure_loaded()?;
            match operation(session) {
                Ok(value) => {
                    self.disarm_first_run_canary();
                    return Ok((value, ProviderUsage::from_provider(execution_provider)));
                }
                Err(error) => {
                    if error.is_retryable() {
                        self.leave_first_run_canary_armed();
                    } else {
                        self.disarm_first_run_canary();
                    }
                    if self.retry_after_provider_failure(&error) {
                        continue;
                    }
                    return Err(error.into_ml_error());
                }
            }
        }
    }

    fn ensure_loaded(&mut self) -> MlResult<(&mut Session, ExecutionProvider)> {
        let slot = match &mut self.session {
            Some((session, provider)) => return Ok((session, *provider)),
            slot => slot,
        };

        let model_path = self.model_path.as_str();
        let model_namespace = self.model_namespace.as_str();
        let provider_plan = self
            .provider_plan
            .get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.validation));
        let model_name = model_file_label(model_path);
        log::info!("loading {model_name} with {:?} execution", self.mode);
        let started_at = std::time::Instant::now();
        let (loaded, execution_provider) = build_next_session(
            model_path,
            provider_plan,
            model_namespace,
            self.validation,
            self.gpu_options.as_ref(),
        )?;
        log::info!(
            "loaded {model_name} with {execution_provider:?} in {:?}",
            started_at.elapsed()
        );
        self.first_run_canary = loaded.first_run_canary;
        let (session, provider) = slot.insert((loaded.session, execution_provider));
        Ok((session, *provider))
    }

    fn defer_first_run_canary(&mut self) {
        if let Some(canary) = self.first_run_canary.take() {
            canary.defer();
            self.deferred_first_run_canary = true;
        }
    }

    fn arm_deferred_first_run_canary(&mut self) -> SessionRunResult<()> {
        #[cfg(any(
            target_os = "android",
            target_os = "linux",
            target_os = "windows",
            test
        ))]
        if self.deferred_first_run_canary {
            self.first_run_canary = Some(
                webgpu::arm_canary(&self.model_path, &self.model_namespace).map_err(|error| {
                    SessionRunError::retryable(MlError::Ort(format!(
                        "failed to arm WebGPU crash canary: {error}"
                    )))
                })?,
            );
            self.deferred_first_run_canary = false;
        }
        Ok(())
    }

    fn disarm_first_run_canary(&mut self) {
        if let Some(canary) = self.first_run_canary.take() {
            canary.disarm();
        }
    }

    fn leave_first_run_canary_armed(&mut self) {
        self.first_run_canary = None;
    }

    fn retry_after_provider_failure(&mut self, error: &SessionRunError) -> bool {
        if !error.is_retryable()
            || !self
                .provider_plan
                .as_ref()
                .is_some_and(ProviderPlan::has_fallback)
        {
            return false;
        }

        self.session = None;
        self.deferred_first_run_canary = false;
        log::warn!(
            "execution provider failed, retrying model with the next provider fallback: {error}"
        );
        true
    }
}

struct LoadedSession {
    session: Session,
    first_run_canary: Option<webgpu::ArmedCanary>,
}

impl LoadedSession {
    fn new(session: Session) -> Self {
        Self {
            session,
            first_run_canary: None,
        }
    }
}

#[derive(Debug)]
pub(crate) enum SessionRunError {
    Retryable(MlError),
    Terminal(MlError),
}

pub(crate) type SessionRunResult<T> = Result<T, SessionRunError>;

impl SessionRunError {
    fn retryable(error: MlError) -> Self {
        Self::Retryable(error)
    }

    pub(crate) fn from_inference_error(error: ort::Error) -> Self {
        match error.code() {
            ort::ErrorCode::GenericFailure
            | ort::ErrorCode::RuntimeException
            | ort::ErrorCode::ExecutionProviderFailure => Self::Retryable(error.into()),
            _ => Self::Terminal(error.into()),
        }
    }

    fn is_retryable(&self) -> bool {
        matches!(self, Self::Retryable(_))
    }

    fn into_ml_error(self) -> MlError {
        match self {
            Self::Retryable(error) | Self::Terminal(error) => error,
        }
    }
}

impl fmt::Display for SessionRunError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Retryable(error) | Self::Terminal(error) => error.fmt(formatter),
        }
    }
}

impl From<MlError> for SessionRunError {
    fn from(error: MlError) -> Self {
        Self::Terminal(error)
    }
}

impl From<SessionRunError> for MlError {
    fn from(error: SessionRunError) -> Self {
        error.into_ml_error()
    }
}

impl<R> From<ort::Error<R>> for SessionRunError {
    fn from(error: ort::Error<R>) -> Self {
        Self::Terminal(error.into())
    }
}

fn session_load_error<R>(model_path: &str, error: ort::Error<R>) -> MlError {
    match error.code() {
        ort::ErrorCode::InvalidProtobuf => MlError::CorruptModel(model_path.to_string()),
        _ => error.into(),
    }
}

fn build_next_session(
    model_path: &str,
    plan: &mut ProviderPlan,
    model_namespace: &str,
    validation: AccelerationValidation,
    gpu_options: Option<&GpuOptions>,
) -> MlResult<(LoadedSession, ExecutionProvider)> {
    let result = providers::run_provider_plan(plan, |execution_provider| {
        let attempt = providers::provider_attempt(
            execution_provider,
            model_path,
            model_namespace,
            gpu_options,
        );
        if attempt.execution_provider() == ExecutionProvider::WebGpu {
            #[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
            {
                return build_webgpu_session_with_canary(
                    model_path,
                    model_namespace,
                    attempt,
                    validation,
                );
            }
            #[cfg(not(any(target_os = "android", target_os = "linux", target_os = "windows")))]
            unreachable!("WebGPU provider attempts are not constructed on this platform");
        }

        let coreml_cache_dir = attempt.coreml_cache_dir().map(Path::to_path_buf);
        match build_and_validate_session(model_path, attempt, validation) {
            Ok(session) => {
                if let Some(cache_dir) = coreml_cache_dir {
                    coreml_cache::finalize(&cache_dir, model_path);
                }
                Ok(LoadedSession::new(session))
            }
            Err(error) => {
                if let Some(cache_dir) = coreml_cache_dir
                    && let Err(cleanup_error) = coreml_cache::invalidate(&cache_dir)
                {
                    log::warn!(
                        "failed to invalidate CoreML cache for '{}' after session construction failed: {cleanup_error}",
                        model_file_label(model_path)
                    );
                }
                Err(error)
            }
        }
    });

    let errors = match result {
        Ok(session) => return Ok(session),
        Err(errors) => errors,
    };

    if errors
        .iter()
        .any(|error| matches!(error, MlError::CorruptModel(_)))
    {
        return Err(MlError::CorruptModel(model_path.to_string()));
    }

    Err(MlError::Ort(format!(
        "failed to create ONNX session for model '{model_path}' across EP fallbacks: {}",
        errors
            .iter()
            .map(ToString::to_string)
            .collect::<Vec<_>>()
            .join(" | ")
    )))
}

fn build_cpu_session(model_path: &str) -> MlResult<Session> {
    let mut plan = ProviderPlan::new(
        ExecutionMode::CpuOnly,
        model_path,
        AccelerationValidation::GoldenRequired,
    );
    build_next_session(
        model_path,
        &mut plan,
        "golden-tooling",
        AccelerationValidation::GoldenRequired,
        None,
    )
    .map(|(loaded, _)| loaded.session)
}

// A CoreML self-test failure is treated as construction failure so the caller
// invalidates the possibly corrupt persistent cache. WebGPU validation stays
// inside its crash-canary window instead.
fn build_and_validate_session(
    model_path: &str,
    attempt: providers::ProviderAttempt,
    _validation: AccelerationValidation,
) -> MlResult<Session> {
    #[cfg(any(
        target_os = "android",
        target_os = "ios",
        target_os = "linux",
        target_os = "macos",
        target_os = "windows"
    ))]
    let execution_provider = attempt.execution_provider();

    let session = match providers::build_session(model_path, attempt) {
        Ok(session) => session,
        Err(error) => {
            #[cfg(any(
                target_os = "android",
                target_os = "ios",
                target_os = "linux",
                target_os = "macos",
                target_os = "windows"
            ))]
            record_provider_attempt_failure(
                execution_provider,
                model_path,
                "session construction",
                &error,
            );
            return Err(error);
        }
    };

    #[cfg(any(target_os = "ios", target_os = "macos"))]
    let mut session = session;

    #[cfg(any(target_os = "ios", target_os = "macos"))]
    if execution_provider == ExecutionProvider::CoreMl
        && _validation == AccelerationValidation::GoldenRequired
    {
        run_session_self_test(model_path, &mut session, "CoreML")?;
    }

    Ok(session)
}

// The durable canary remains armed through construction and self-test, so
// crashes and soft failures both count toward quarantine.
#[cfg(any(target_os = "android", target_os = "linux", target_os = "windows"))]
fn build_webgpu_session_with_canary(
    model_path: &str,
    model_namespace: &str,
    attempt: providers::ProviderAttempt,
    validation: AccelerationValidation,
) -> MlResult<LoadedSession> {
    // Fail closed: without a durable failure record, a crash during the
    // attempt would go unnoticed and the crash loop protection would be lost.
    let canary = match webgpu::arm_canary(model_path, model_namespace) {
        Ok(canary) => canary,
        Err(error) => {
            let error = MlError::Ort(format!("failed to arm WebGPU crash canary: {error}"));
            record_provider_attempt_failure(
                ExecutionProvider::WebGpu,
                model_path,
                "crash-canary setup",
                &error,
            );
            return Err(error);
        }
    };
    #[cfg(target_os = "android")]
    {
        match webgpu::check_adapter() {
            webgpu::AdapterCheck::Allowed => {}
            webgpu::AdapterCheck::Denied => {
                canary.disarm();
                return Err(MlError::Ort(
                    "WebGPU skipped: GPU adapter is not on the allowlist".to_string(),
                ));
            }
            webgpu::AdapterCheck::Failed => {
                let error = MlError::Ort("WebGPU skipped: Vulkan adapter probe failed".to_string());
                record_provider_attempt_failure(
                    ExecutionProvider::WebGpu,
                    model_path,
                    "adapter probe",
                    &error,
                );
                return Err(error);
            }
        }
    }
    let mut session = match providers::build_session(model_path, attempt) {
        Ok(session) => session,
        Err(error) => {
            record_provider_attempt_failure(
                ExecutionProvider::WebGpu,
                model_path,
                "session construction",
                &error,
            );
            return Err(error);
        }
    };
    let first_run_canary = if validation == AccelerationValidation::GoldenRequired {
        run_session_self_test(model_path, &mut session, "WebGPU")?;
        canary.disarm();
        None
    } else {
        Some(canary)
    };
    Ok(LoadedSession {
        session,
        first_run_canary,
    })
}

#[cfg(any(
    target_os = "android",
    target_os = "ios",
    target_os = "linux",
    target_os = "macos",
    target_os = "windows"
))]
fn record_provider_attempt_failure(
    provider: ExecutionProvider,
    model_path: &str,
    stage: &str,
    error: &MlError,
) {
    if let Some(message) = provider_attempt_failure_message(provider, model_path, stage, error) {
        log::warn!("{message}");
    }
}

#[cfg(any(
    target_os = "android",
    target_os = "ios",
    target_os = "linux",
    target_os = "macos",
    target_os = "windows",
    test
))]
fn provider_attempt_failure_message(
    provider: ExecutionProvider,
    model_path: &str,
    stage: &str,
    error: &MlError,
) -> Option<String> {
    let provider_label = match provider {
        ExecutionProvider::CoreMl => "CoreML",
        ExecutionProvider::WebGpu => "WebGPU",
        ExecutionProvider::Xnnpack => "XNNPACK",
        ExecutionProvider::Cpu => return None,
    };
    Some(format!(
        "{provider_label} {stage} failed for '{}': {error}; falling back to the next execution provider",
        model_file_label(model_path)
    ))
}

#[cfg(any(
    target_os = "android",
    target_os = "ios",
    target_os = "linux",
    target_os = "macos",
    target_os = "windows"
))]
fn run_session_self_test(
    model_path: &str,
    session: &mut Session,
    provider_label: &str,
) -> MlResult<()> {
    let model_file = model_file_label(model_path);
    let Some(entry) = golden_test::lookup(model_path) else {
        return Err(MlError::Ort(format!(
            "golden self-test entry missing for '{model_file}'"
        )));
    };
    let golden_input = golden_test::prepare_input(entry).map_err(|reason| {
        MlError::Ort(format!(
            "golden self-test input invalid for '{model_file}': {reason}"
        ))
    })?;
    let zero_input = golden_input.zeroed();

    // Warm up pipeline creation and the first dispatch with an input that is
    // deliberately different from the golden. If a later dispatch reuses
    // this result, the golden comparison below rejects the session.
    let zero_output = match run_golden_tensor(session, entry.input_shape, &zero_input) {
        Ok(output) => output,
        Err(error) => {
            log::warn!(
                "{provider_label} zero-input warm-up inference failed for '{model_file}': \
                 {error}; falling back to the next execution provider"
            );
            return Err(error.into());
        }
    };
    if let Err(reason) = golden_test::validate_output(entry, &zero_output) {
        log::error!(
            "{provider_label} zero-input warm-up failed for '{model_file}': {reason}; \
             falling back to the next execution provider"
        );
        return Err(MlError::Ort(format!(
            "zero-input warm-up failed for '{model_file}': {reason}"
        )));
    }
    log::info!("{provider_label} zero-input warm-up for '{model_file}' passed");

    let golden_output = match run_golden_tensor(session, entry.input_shape, &golden_input) {
        Ok(output) => output,
        Err(error) => {
            // Surface inference failures too: without this, a provider that
            // cannot execute the golden input would fall back invisibly (the
            // session-build error is swallowed once the next attempt succeeds).
            log::warn!(
                "{provider_label} golden self-test inference failed for '{model_file}': \
                 {error}; falling back to the next execution provider"
            );
            return Err(error.into());
        }
    };
    match golden_test::compare_output(entry, &golden_output) {
        Ok(distance) => {
            log::info!(
                "{provider_label} golden self-test for '{model_file}' passed \
                 ({} {distance:.2e})",
                entry.metric.label()
            );
            Ok(())
        }
        Err(reason) => {
            log::error!(
                "{provider_label} golden self-test failed for '{model_file}': {reason}; \
                 falling back to the next execution provider"
            );
            Err(MlError::Ort(format!(
                "golden self-test failed for '{model_file}': {reason}"
            )))
        }
    }
}

fn model_file_label(model_path: &str) -> &str {
    Path::new(model_path)
        .file_name()
        .and_then(|name| name.to_str())
        .unwrap_or(model_path)
}

#[cfg(test)]
mod tests {
    use super::{ExecutionMode, ExecutionProvider, OnnxSession, provider_attempt_failure_message};

    #[test]
    fn unvalidated_acceleration_does_not_enable_ocr_gpu_options() {
        let indexing = OnnxSession::new("model.onnx", "indexing", ExecutionMode::PlatformDefault);
        assert!(indexing.gpu_options.is_none());
        assert_eq!(
            indexing.validation,
            super::AccelerationValidation::GoldenRequired
        );

        let scanner = indexing.with_unvalidated_acceleration();
        assert!(scanner.gpu_options.is_none());
        assert_eq!(
            scanner.validation,
            super::AccelerationValidation::Unvalidated
        );
        assert_eq!(scanner.mode, ExecutionMode::PlatformDefault);
    }

    fn first_run_canary(temp: &tempfile::TempDir) -> super::webgpu::ArmedCanary {
        let model = temp.path().join("model.onnx");
        super::webgpu::arm_canary(&model.to_string_lossy(), "scanner").unwrap()
    }

    fn has_canary(temp: &tempfile::TempDir) -> bool {
        std::fs::read_dir(temp.path()).unwrap().next().is_some()
    }

    #[test]
    fn successful_first_run_disarms_the_canary() {
        let temp = tempfile::tempdir().unwrap();
        let mut session = OnnxSession::new("model.onnx", "scanner", ExecutionMode::CpuOnly);
        session.first_run_canary = Some(first_run_canary(&temp));

        session.disarm_first_run_canary();

        assert!(!has_canary(&temp));
    }

    #[test]
    fn retryable_first_run_failure_leaves_the_canary_armed() {
        let temp = tempfile::tempdir().unwrap();
        let mut session = OnnxSession::new("model.onnx", "scanner", ExecutionMode::CpuOnly);
        session.first_run_canary = Some(first_run_canary(&temp));

        session.leave_first_run_canary_armed();

        assert!(has_canary(&temp));
    }

    #[test]
    fn unused_preloaded_session_does_not_leave_a_crash_canary() {
        let temp = tempfile::tempdir().unwrap();
        let model = temp.path().join("model.onnx");
        let mut session = OnnxSession::new(
            model.to_str().unwrap(),
            "scanner",
            ExecutionMode::GpuPreferred,
        );
        session.first_run_canary = Some(first_run_canary(&temp));

        session.defer_first_run_canary();
        session.defer_first_run_canary();

        assert!(session.deferred_first_run_canary);
        assert!(!has_canary(&temp));
        drop(session);
        assert!(!has_canary(&temp));
    }

    #[test]
    fn preloaded_session_rearms_canary_only_for_first_inference() {
        let temp = tempfile::tempdir().unwrap();
        let model = temp.path().join("model.onnx");
        let mut session = OnnxSession::new(
            model.to_str().unwrap(),
            "scanner",
            ExecutionMode::GpuPreferred,
        );
        session.first_run_canary = Some(first_run_canary(&temp));
        session.defer_first_run_canary();

        session.arm_deferred_first_run_canary().unwrap();

        assert!(has_canary(&temp));
        assert!(!session.deferred_first_run_canary);
        session.disarm_first_run_canary();
        session.arm_deferred_first_run_canary().unwrap();
        assert!(!has_canary(&temp));
    }

    #[test]
    fn preload_preserves_prior_failures_until_successful_inference() {
        for previous_failures in 1..=2 {
            let temp = tempfile::tempdir().unwrap();
            let model = temp.path().join("model.onnx");
            let mut session = OnnxSession::new(
                model.to_str().unwrap(),
                "scanner",
                ExecutionMode::GpuPreferred,
            );
            for _ in 0..previous_failures {
                drop(first_run_canary(&temp));
            }
            session.first_run_canary = Some(first_run_canary(&temp));

            session.defer_first_run_canary();
            session.defer_first_run_canary();

            let canary = std::fs::read_dir(temp.path())
                .unwrap()
                .next()
                .unwrap()
                .unwrap()
                .path();
            assert_eq!(
                std::fs::read_to_string(&canary).unwrap().trim(),
                previous_failures.to_string()
            );
            session.arm_deferred_first_run_canary().unwrap();
            assert_eq!(
                std::fs::read_to_string(&canary).unwrap().trim(),
                (previous_failures + 1).to_string()
            );
            session.disarm_first_run_canary();
            assert!(!has_canary(&temp));
        }
    }

    #[test]
    fn failed_canary_rearm_remains_required_on_retry() {
        let temp = tempfile::tempdir().unwrap();
        let parent = temp.path().join("not-a-directory");
        std::fs::write(&parent, []).unwrap();
        let model = parent.join("model.onnx");
        let mut session = OnnxSession::new(
            model.to_str().unwrap(),
            "scanner",
            ExecutionMode::GpuPreferred,
        );
        session.deferred_first_run_canary = true;

        for _ in 0..2 {
            let error = session.arm_deferred_first_run_canary().unwrap_err();
            assert!(error.is_retryable());
            assert!(session.deferred_first_run_canary);
        }
    }

    #[test]
    fn unloading_preloaded_session_clears_deferred_canary() {
        let temp = tempfile::tempdir().unwrap();
        let model = temp.path().join("model.onnx");
        let mut session = OnnxSession::new(
            model.to_str().unwrap(),
            "ocr-detection-fixed-v1",
            ExecutionMode::GpuPreferred,
        );
        session.initialize_load_state();
        session.first_run_canary = Some(first_run_canary(&temp));
        session.defer_first_run_canary();

        session.unload();
        session.arm_deferred_first_run_canary().unwrap();

        assert!(!session.has_load_state());
        assert!(!session.deferred_first_run_canary);
        assert!(!has_canary(&temp));
    }

    #[test]
    fn unload_preserves_model_identity_and_armed_canary() {
        let temp = tempfile::tempdir().unwrap();
        let mut session = OnnxSession::new(
            "/models/document.onnx",
            "document-segmentation",
            ExecutionMode::CpuOnly,
        );
        session.initialize_load_state();
        session.first_run_canary = Some(first_run_canary(&temp));

        session.unload();

        assert_eq!(session.model_path, "/models/document.onnx");
        assert_eq!(session.model_namespace, "document-segmentation");
        assert!(!session.is_loaded());
        assert!(!session.has_load_state());
        assert!(has_canary(&temp));
    }

    #[test]
    fn reports_accelerated_provider_attempt_failures_with_model_context() {
        let error = super::MlError::Ort("provider registration failed".to_string());
        let message = provider_attempt_failure_message(
            ExecutionProvider::CoreMl,
            "/models/face.onnx",
            "session construction",
            &error,
        )
        .unwrap();

        assert!(message.contains("CoreML session construction failed"));
        assert!(message.contains("'face.onnx'"));
        assert!(message.contains("provider registration failed"));
        assert!(message.contains("falling back"));
    }

    #[test]
    fn does_not_report_final_cpu_construction_failure_as_a_fallback() {
        let error = super::MlError::Ort("session construction failed".to_string());

        assert!(
            provider_attempt_failure_message(
                ExecutionProvider::Cpu,
                "/models/face.onnx",
                "session construction",
                &error,
            )
            .is_none()
        );
    }
}