Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

16.2 KB
Raw Blame
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
pub mod embedding;
pub mod identify_gate;

use std::path::Path;

use anyhow::{anyhow, Result};

pub fn create_session<P: AsRef<Path>>(path: P) -> Result<ort::session::Session> {
    create_session_with_provider(path, SessionKind::Default)
}

/// Speaker turns produce a different fbank time dimension on almost every run.
/// On Apple Silicon the CPU execution path retained several gigabytes of native
/// activation memory while a meeting backlog drained. Keep CoreML on compatible
/// macOS versions; the CPU compatibility path disables both its retaining arena
/// and per-shape memory patterns. Neither path changes the input tensor.
pub(crate) fn create_embedding_session<P: AsRef<Path>>(path: P) -> Result<ort::session::Session> {
    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
    let kind = {
        let version = crate::core::process_tap::macos_version_string();
        let kind = embedding_provider_for_macos(version.as_deref());
        if kind == SessionKind::EmbeddingCpu {
            tracing::warn!(
                macos_version = version.as_deref().unwrap_or("unknown"),
                "speaker embedding using CPU compatibility path: avoiding CoreML native compiler crash (#7238); memory arena and patterns disabled"
            );
        }
        kind
    };
    #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
    let kind = SessionKind::EmbeddingCpu;
    create_session_with_provider(path, kind)
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SessionKind {
    Default,
    EmbeddingCpu,
    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
    EmbeddingCoreMl,
}

#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
fn embedding_provider_for_macos(version: Option<&str>) -> SessionKind {
    let version = version.and_then(|version| {
        let mut parts = version.split('.');
        Some((
            parts.next()?.parse::<u32>().ok()?,
            parts.next()?.parse::<u32>().ok()?,
        ))
    });
    // macOS 27.2's NeuralNetwork convolution compiler raises SIGSEGV, which
    // neither catch_unwind nor the ORT watchdog can contain. MLProgram also
    // rejects this model's dynamic shapes (and corrupts short-input embeddings
    // when restricted to static nodes). Bypass CoreML on 27.2+ until verified
    // compatible, and if the OS version cannot be established.
    if version.is_some_and(|version| version < (27, 2)) {
        SessionKind::EmbeddingCoreMl
    } else {
        SessionKind::EmbeddingCpu
    }
}

fn create_session_with_provider<P: AsRef<Path>>(
    path: P,
    kind: SessionKind,
) -> Result<ort::session::Session> {
    let path = path.as_ref().to_path_buf();
    // ort 2.0.0-rc.10 panics from inside its global OnceLock when the ONNX
    // Runtime API can't be initialized (Windows DLL/version mismatch hits
    // `expect("Failed to initialize ORT API")` at lib.rs:188). That panic
    // bubbles up the tokio worker and Sentry — `catch_panic_into_error`
    // converts it to a normal error so callers fall back gracefully.
    //
    // ort rc.12 added a second, worse failure mode on the same global init: it
    // can *hang* instead of panicking, which panic-catching cannot recover.
    // This is the first ort call on the audio boot path, so a hang here freezes
    // the `building_audio` phase forever and the engine never binds its port.
    // Run the (blocking) session build under a watchdog so a hung runtime
    // degrades to an error and the caller falls back (diarization off) instead
    // of bricking startup. See `utils::ort_watchdog`.
    crate::utils::ort_watchdog::run_with_timeout(
        if kind != SessionKind::Default {
            "speaker_embedding"
        } else {
            "speaker_segmentation"
        },
        crate::utils::ort_watchdog::ORT_INIT_TIMEOUT,
        move || {
            catch_panic_into_error("ort session init", || {
                // ort rc.12: builder ops return `Error<SessionBuilder>` (recovery
                // payload, not Send+Sync) — convert via Display for anyhow.
                let oe = |e: &dyn std::fmt::Display| anyhow!("ort: {e}");
                let b = ort::session::Session::builder().map_err(|e| oe(&e))?;
                let b = b
                    .with_optimization_level(ort::session::builder::GraphOptimizationLevel::Level3)
                    .map_err(|e| oe(&e))?;
                let b = b.with_intra_threads(1).map_err(|e| oe(&e))?;
                let mut b = b.with_inter_threads(1).map_err(|e| oe(&e))?;
                if kind != SessionKind::Default {
                    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
                    {
                        let mut providers = Vec::with_capacity(2);
                        if kind == SessionKind::EmbeddingCoreMl {
                            providers.push(
                                ort::ep::CoreML::default()
                                    .with_static_input_shapes(false)
                                    .with_subgraphs(true)
                                    .build(),
                            );
                        } else {
                            b = b.with_memory_pattern(false).map_err(|e| oe(&e))?;
                        }
                        providers.push(ort::ep::CPU::default().with_arena_allocator(false).build());
                        b = b.with_execution_providers(providers).map_err(|e| oe(&e))?;
                    }
                }
                let session = b.commit_from_file(&path).map_err(|e| oe(&e))?;
                Ok(session)
            })
        },
    )
}

/// Run `f`, converting any panic it unwinds into an error tagged with
/// `context`. Both ort (FFI) and knf-rs (fbank) `unwrap()`/`expect()` deep in
/// native bindings on bad input or environment mismatches; without this guard
/// those panics unwind the audio/tokio worker and crash the runtime instead of
/// being handled. The `context` names which guarded call failed so logs and
/// Sentry stay legible (e.g. "ort session init" vs "compute_fbank").
fn catch_panic_into_error<F, T>(context: &str, f: F) -> Result<T>
where
    F: FnOnce() -> Result<T>,
{
    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
        Ok(result) => result,
        Err(payload) => {
            let msg = payload
                .downcast_ref::<&'static str>()
                .map(|s| (*s).to_string())
                .or_else(|| payload.downcast_ref::<String>().cloned())
                .unwrap_or_else(|| "unknown panic".to_string());
            Err(anyhow!("{context} panicked: {msg}"))
        }
    }
}

/// The output node names of the models we ship.
pub(crate) fn session_output_names(session: &ort::session::Session) -> Vec<String> {
    session
        .outputs()
        .iter()
        .map(|o| o.name().to_string())
        .collect()
}

/// Resolve which output to read from an ORT session's run results.
///
/// The models we ship expose a known canonical output name ("output" for the
/// pyannote segmentation model, "embs" for the wespeaker embedding model), and
/// the inference code looks that name up directly. The trap: a cached model can
/// be a *structurally identical* export whose single output node is named
/// differently — an older pyannote segmentation export names its output "y"
/// instead of "output". That model still loads cleanly, so the on-disk cache
/// never self-heals (it only re-downloads on ORT *load* errors), and then it
/// fails every single inference with "Output tensor not found". In batch mode
/// that silently wedges the whole transcription backlog: capture keeps running,
/// chunks pile up, nothing is ever transcribed.
///
/// These models have exactly one output, so we prefer the canonical name but
/// fall back to the sole output when it's absent. We only error when the model
/// exposes no output, or several with none matching (then there's no safe pick).
pub(crate) fn resolve_output_name(output_names: &[String], preferred: &str) -> Result<String> {
    if output_names.iter().any(|name| name == preferred) {
        return Ok(preferred.to_string());
    }
    match output_names {
        [only] => Ok(only.clone()),
        [] => Err(anyhow!("model exposes no outputs")),
        names => Err(anyhow!(
            "model has no '{}' output and {} candidates with no safe default: {:?}",
            preferred,
            names.len(),
            names
        )),
    }
}

pub mod embedding_manager;
pub mod models;
mod prepare_segments;
pub use prepare_segments::prepare_segments;
pub mod segment;

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
    #[test]
    fn embedding_provider_avoids_affected_macos_compiler() {
        for version in ["14.0", "26.4", "26.6.2", "27.0", "27.1.1"] {
            assert_eq!(
                embedding_provider_for_macos(Some(version)),
                SessionKind::EmbeddingCoreMl
            );
        }
        for version in [
            Some("27.2"),
            Some("27.2.1"),
            Some("28.0"),
            Some("invalid"),
            None,
        ] {
            assert_eq!(
                embedding_provider_for_macos(version),
                SessionKind::EmbeddingCpu
            );
        }
    }

    #[test]
    fn catch_panic_into_error_passes_through_ok() {
        let r: Result<i32> = catch_panic_into_error("ort session init", || Ok(7));
        assert_eq!(r.unwrap(), 7);
    }

    #[test]
    fn catch_panic_into_error_passes_through_err() {
        let r: Result<()> =
            catch_panic_into_error("ort session init", || Err(anyhow!("normal failure")));
        let msg = r.unwrap_err().to_string();
        assert!(msg.contains("normal failure"));
        assert!(!msg.contains("panicked"));
    }

    #[test]
    fn catch_panic_into_error_catches_str_panic() {
        let r: Result<()> = catch_panic_into_error("ort session init", || panic!("boom"));
        let msg = r.unwrap_err().to_string();
        assert!(msg.contains("ort session init panicked"));
        assert!(msg.contains("boom"));
    }

    #[test]
    fn catch_panic_into_error_catches_string_panic() {
        let r: Result<()> =
            catch_panic_into_error("ort session init", || panic!("formatted: {}", 42));
        let msg = r.unwrap_err().to_string();
        assert!(msg.contains("ort session init panicked"));
        assert!(msg.contains("42"));
    }

    #[test]
    fn catch_panic_into_error_simulates_ort_api_init_panic() {
        // Mirrors the exact panic ort 2.0.0-rc.10 raises at lib.rs:188 when
        // `NonNull::new(api).expect("Failed to initialize ORT API")` triggers.
        let r: Result<()> = catch_panic_into_error("ort session init", || {
            panic!("Failed to initialize ORT API")
        });
        let msg = r.unwrap_err().to_string();
        assert!(msg.contains("ort session init panicked"));
        assert!(msg.contains("Failed to initialize ORT API"));
    }

    #[test]
    fn catch_panic_into_error_uses_supplied_context() {
        // The context tag distinguishes which guarded call panicked. embedding.rs
        // passes "compute_fbank"; make sure that label reaches the error.
        let r: Result<()> = catch_panic_into_error("compute_fbank", || panic!("boom"));
        let msg = r.unwrap_err().to_string();
        assert!(msg.contains("compute_fbank panicked"), "got: {msg}");
        assert!(msg.contains("boom"));
    }

    #[test]
    fn compute_fbank_tiny_input_is_contained_by_guard() {
        // SCREENPIPE-CLI-3S regression: knf-rs unwraps a None deep in
        // OnlineGenericBaseFeature::InputFinished for buffers too short to yield
        // a frame, unwinding the audio worker. Routing compute_fbank through the
        // guard must contain that. Reaching the end of this test (no unwind) is
        // the assertion; the result may be Ok or a handled Err depending on knf.
        let guarded = catch_panic_into_error("compute_fbank", || {
            knf_rs::compute_fbank(&[0.0f32; 16]).map_err(anyhow::Error::msg)
        });
        let _ = guarded;
    }

    #[test]
    fn create_session_returns_err_for_missing_path() {
        // Sanity-check the normal error path still flows through ?-propagation
        // (commit_from_file fails, we return Err, no panic conversion needed).
        let r = create_session("/nonexistent/screenpipe-audio-test-model.onnx");
        assert!(r.is_err());
    }

    #[test]
    fn resolve_output_name_prefers_canonical_name() {
        let outputs = vec!["output".to_string()];
        assert_eq!(resolve_output_name(&outputs, "output").unwrap(), "output");
    }

    #[test]
    fn resolve_output_name_prefers_canonical_even_with_extra_outputs() {
        let outputs = vec!["aux".to_string(), "output".to_string()];
        assert_eq!(resolve_output_name(&outputs, "output").unwrap(), "output");
    }

    #[test]
    fn stale_segmentation_model_with_renamed_output_still_resolves() {
        // Reproduction of the field bug: a user's cached segmentation-3.0.onnx
        // was an older export whose single output node is named "y", not
        // "output". The model loads fine (so the cache never re-downloads), but
        // the old inference code hard-coded `ort_outs.get("output")`, which
        // returns None on every chunk -> "Output tensor not found" -> the whole
        // transcription backlog wedges while screen/audio capture keeps running.
        let stale_model_outputs = vec!["y".to_string()];

        // Old behavior: the hard-coded canonical name simply isn't present.
        assert!(!stale_model_outputs.iter().any(|n| n == "output"));

        // New behavior: we fall back to the model's sole output and proceed,
        // so a structurally-identical-but-renamed export transcribes normally.
        assert_eq!(
            resolve_output_name(&stale_model_outputs, "output").unwrap(),
            "y"
        );
    }

    #[test]
    fn resolve_output_name_errors_on_no_outputs() {
        let outputs: Vec<String> = vec![];
        assert!(resolve_output_name(&outputs, "output").is_err());
    }

    #[test]
    fn resolve_output_name_errors_when_ambiguous_and_unmatched() {
        // Multiple outputs and none canonical: refuse to guess.
        let outputs = vec!["a".to_string(), "b".to_string()];
        let err = resolve_output_name(&outputs, "output")
            .unwrap_err()
            .to_string();
        assert!(err.contains("no safe default"));
    }

    // Real-ORT reproduction harness against the model we actually ship. Loads
    // the canonical segmentation model, resolves its output via the same path
    // production uses, and runs a forward pass to prove the resolved name is
    // really retrievable from the run results. `#[ignore]` like the other
    // model-dependent tests in this crate (needs the on-disk model + ORT).
    #[test]
    #[ignore]
    fn segmentation_model_output_resolves_against_real_session() {
        use std::path::PathBuf;

        let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
            .join("models")
            .join("pyannote")
            .join("segmentation-3.0.onnx");

        let mut session = create_session(&model_path).expect("failed to load segmentation model");

        // Resolve the output name the way SegmentIterator now does.
        let output_name = resolve_output_name(&session_output_names(&session), "output")
            .expect("failed to resolve segmentation output name");

        // Forward pass on a 10s window of silence (matches the segmentation
        // window size used in segment.rs: sample_rate * 10 at 16 kHz).
        let window = vec![0.0f32; 16000 * 10];
        let array = ndarray::Array1::from_vec(window)
            .view()
            .insert_axis(ndarray::Axis(0))
            .insert_axis(ndarray::Axis(1))
            .to_owned();
        let inputs = ort::inputs![ort::value::TensorRef::from_array_view(array.view()).unwrap()];
        let ort_outs = session.run(inputs).expect("session run failed");

        // The resolved name must actually exist in the run results — this is
        // exactly the lookup that returned None for the stale "y" model.
        assert!(
            ort_outs.get(&output_name).is_some(),
            "resolved output '{output_name}' not present in run results"
        );
    }
}