YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
pub mod embedding;
pub mod identify_gate;
use std::path::Path;
use anyhow::{anyhow, Result};
pub fn create_session<P: AsRef<Path>>(path: P) -> Result<ort::session::Session> {
create_session_with_provider(path, SessionKind::Default)
}
/// Speaker turns produce a different fbank time dimension on almost every run.
/// On Apple Silicon the CPU execution path retained several gigabytes of native
/// activation memory while a meeting backlog drained. Keep CoreML on compatible
/// macOS versions; the CPU compatibility path disables both its retaining arena
/// and per-shape memory patterns. Neither path changes the input tensor.
pub(crate) fn create_embedding_session<P: AsRef<Path>>(path: P) -> Result<ort::session::Session> {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
let kind = {
let version = crate::core::process_tap::macos_version_string();
let kind = embedding_provider_for_macos(version.as_deref());
if kind == SessionKind::EmbeddingCpu {
tracing::warn!(
macos_version = version.as_deref().unwrap_or("unknown"),
"speaker embedding using CPU compatibility path: avoiding CoreML native compiler crash (#7238); memory arena and patterns disabled"
);
}
kind
};
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
let kind = SessionKind::EmbeddingCpu;
create_session_with_provider(path, kind)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SessionKind {
Default,
EmbeddingCpu,
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
EmbeddingCoreMl,
}
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
fn embedding_provider_for_macos(version: Option<&str>) -> SessionKind {
let version = version.and_then(|version| {
let mut parts = version.split('.');
Some((
parts.next()?.parse::<u32>().ok()?,
parts.next()?.parse::<u32>().ok()?,
))
});
// macOS 27.2's NeuralNetwork convolution compiler raises SIGSEGV, which
// neither catch_unwind nor the ORT watchdog can contain. MLProgram also
// rejects this model's dynamic shapes (and corrupts short-input embeddings
// when restricted to static nodes). Bypass CoreML on 27.2+ until verified
// compatible, and if the OS version cannot be established.
if version.is_some_and(|version| version < (27, 2)) {
SessionKind::EmbeddingCoreMl
} else {
SessionKind::EmbeddingCpu
}
}
fn create_session_with_provider<P: AsRef<Path>>(
path: P,
kind: SessionKind,
) -> Result<ort::session::Session> {
let path = path.as_ref().to_path_buf();
// ort 2.0.0-rc.10 panics from inside its global OnceLock when the ONNX
// Runtime API can't be initialized (Windows DLL/version mismatch hits
// `expect("Failed to initialize ORT API")` at lib.rs:188). That panic
// bubbles up the tokio worker and Sentry — `catch_panic_into_error`
// converts it to a normal error so callers fall back gracefully.
//
// ort rc.12 added a second, worse failure mode on the same global init: it
// can *hang* instead of panicking, which panic-catching cannot recover.
// This is the first ort call on the audio boot path, so a hang here freezes
// the `building_audio` phase forever and the engine never binds its port.
// Run the (blocking) session build under a watchdog so a hung runtime
// degrades to an error and the caller falls back (diarization off) instead
// of bricking startup. See `utils::ort_watchdog`.
crate::utils::ort_watchdog::run_with_timeout(
if kind != SessionKind::Default {
"speaker_embedding"
} else {
"speaker_segmentation"
},
crate::utils::ort_watchdog::ORT_INIT_TIMEOUT,
move || {
catch_panic_into_error("ort session init", || {
// ort rc.12: builder ops return `Error<SessionBuilder>` (recovery
// payload, not Send+Sync) — convert via Display for anyhow.
let oe = |e: &dyn std::fmt::Display| anyhow!("ort: {e}");
let b = ort::session::Session::builder().map_err(|e| oe(&e))?;
let b = b
.with_optimization_level(ort::session::builder::GraphOptimizationLevel::Level3)
.map_err(|e| oe(&e))?;
let b = b.with_intra_threads(1).map_err(|e| oe(&e))?;
let mut b = b.with_inter_threads(1).map_err(|e| oe(&e))?;
if kind != SessionKind::Default {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
{
let mut providers = Vec::with_capacity(2);
if kind == SessionKind::EmbeddingCoreMl {
providers.push(
ort::ep::CoreML::default()
.with_static_input_shapes(false)
.with_subgraphs(true)
.build(),
);
} else {
b = b.with_memory_pattern(false).map_err(|e| oe(&e))?;
}
providers.push(ort::ep::CPU::default().with_arena_allocator(false).build());
b = b.with_execution_providers(providers).map_err(|e| oe(&e))?;
}
}
let session = b.commit_from_file(&path).map_err(|e| oe(&e))?;
Ok(session)
})
},
)
}
/// Run `f`, converting any panic it unwinds into an error tagged with
/// `context`. Both ort (FFI) and knf-rs (fbank) `unwrap()`/`expect()` deep in
/// native bindings on bad input or environment mismatches; without this guard
/// those panics unwind the audio/tokio worker and crash the runtime instead of
/// being handled. The `context` names which guarded call failed so logs and
/// Sentry stay legible (e.g. "ort session init" vs "compute_fbank").
fn catch_panic_into_error<F, T>(context: &str, f: F) -> Result<T>
where
F: FnOnce() -> Result<T>,
{
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let msg = payload
.downcast_ref::<&'static str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "unknown panic".to_string());
Err(anyhow!("{context} panicked: {msg}"))
}
}
}
/// The output node names of the models we ship.
pub(crate) fn session_output_names(session: &ort::session::Session) -> Vec<String> {
session
.outputs()
.iter()
.map(|o| o.name().to_string())
.collect()
}
/// Resolve which output to read from an ORT session's run results.
///
/// The models we ship expose a known canonical output name ("output" for the
/// pyannote segmentation model, "embs" for the wespeaker embedding model), and
/// the inference code looks that name up directly. The trap: a cached model can
/// be a *structurally identical* export whose single output node is named
/// differently — an older pyannote segmentation export names its output "y"
/// instead of "output". That model still loads cleanly, so the on-disk cache
/// never self-heals (it only re-downloads on ORT *load* errors), and then it
/// fails every single inference with "Output tensor not found". In batch mode
/// that silently wedges the whole transcription backlog: capture keeps running,
/// chunks pile up, nothing is ever transcribed.
///
/// These models have exactly one output, so we prefer the canonical name but
/// fall back to the sole output when it's absent. We only error when the model
/// exposes no output, or several with none matching (then there's no safe pick).
pub(crate) fn resolve_output_name(output_names: &[String], preferred: &str) -> Result<String> {
if output_names.iter().any(|name| name == preferred) {
return Ok(preferred.to_string());
}
match output_names {
[only] => Ok(only.clone()),
[] => Err(anyhow!("model exposes no outputs")),
names => Err(anyhow!(
"model has no '{}' output and {} candidates with no safe default: {:?}",
preferred,
names.len(),
names
)),
}
}
pub mod embedding_manager;
pub mod models;
mod prepare_segments;
pub use prepare_segments::prepare_segments;
pub mod segment;
#[cfg(test)]
mod tests {
use super::*;
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
#[test]
fn embedding_provider_avoids_affected_macos_compiler() {
for version in ["14.0", "26.4", "26.6.2", "27.0", "27.1.1"] {
assert_eq!(
embedding_provider_for_macos(Some(version)),
SessionKind::EmbeddingCoreMl
);
}
for version in [
Some("27.2"),
Some("27.2.1"),
Some("28.0"),
Some("invalid"),
None,
] {
assert_eq!(
embedding_provider_for_macos(version),
SessionKind::EmbeddingCpu
);
}
}
#[test]
fn catch_panic_into_error_passes_through_ok() {
let r: Result<i32> = catch_panic_into_error("ort session init", || Ok(7));
assert_eq!(r.unwrap(), 7);
}
#[test]
fn catch_panic_into_error_passes_through_err() {
let r: Result<()> =
catch_panic_into_error("ort session init", || Err(anyhow!("normal failure")));
let msg = r.unwrap_err().to_string();
assert!(msg.contains("normal failure"));
assert!(!msg.contains("panicked"));
}
#[test]
fn catch_panic_into_error_catches_str_panic() {
let r: Result<()> = catch_panic_into_error("ort session init", || panic!("boom"));
let msg = r.unwrap_err().to_string();
assert!(msg.contains("ort session init panicked"));
assert!(msg.contains("boom"));
}
#[test]
fn catch_panic_into_error_catches_string_panic() {
let r: Result<()> =
catch_panic_into_error("ort session init", || panic!("formatted: {}", 42));
let msg = r.unwrap_err().to_string();
assert!(msg.contains("ort session init panicked"));
assert!(msg.contains("42"));
}
#[test]
fn catch_panic_into_error_simulates_ort_api_init_panic() {
// Mirrors the exact panic ort 2.0.0-rc.10 raises at lib.rs:188 when
// `NonNull::new(api).expect("Failed to initialize ORT API")` triggers.
let r: Result<()> = catch_panic_into_error("ort session init", || {
panic!("Failed to initialize ORT API")
});
let msg = r.unwrap_err().to_string();
assert!(msg.contains("ort session init panicked"));
assert!(msg.contains("Failed to initialize ORT API"));
}
#[test]
fn catch_panic_into_error_uses_supplied_context() {
// The context tag distinguishes which guarded call panicked. embedding.rs
// passes "compute_fbank"; make sure that label reaches the error.
let r: Result<()> = catch_panic_into_error("compute_fbank", || panic!("boom"));
let msg = r.unwrap_err().to_string();
assert!(msg.contains("compute_fbank panicked"), "got: {msg}");
assert!(msg.contains("boom"));
}
#[test]
fn compute_fbank_tiny_input_is_contained_by_guard() {
// SCREENPIPE-CLI-3S regression: knf-rs unwraps a None deep in
// OnlineGenericBaseFeature::InputFinished for buffers too short to yield
// a frame, unwinding the audio worker. Routing compute_fbank through the
// guard must contain that. Reaching the end of this test (no unwind) is
// the assertion; the result may be Ok or a handled Err depending on knf.
let guarded = catch_panic_into_error("compute_fbank", || {
knf_rs::compute_fbank(&[0.0f32; 16]).map_err(anyhow::Error::msg)
});
let _ = guarded;
}
#[test]
fn create_session_returns_err_for_missing_path() {
// Sanity-check the normal error path still flows through ?-propagation
// (commit_from_file fails, we return Err, no panic conversion needed).
let r = create_session("/nonexistent/screenpipe-audio-test-model.onnx");
assert!(r.is_err());
}
#[test]
fn resolve_output_name_prefers_canonical_name() {
let outputs = vec!["output".to_string()];
assert_eq!(resolve_output_name(&outputs, "output").unwrap(), "output");
}
#[test]
fn resolve_output_name_prefers_canonical_even_with_extra_outputs() {
let outputs = vec!["aux".to_string(), "output".to_string()];
assert_eq!(resolve_output_name(&outputs, "output").unwrap(), "output");
}
#[test]
fn stale_segmentation_model_with_renamed_output_still_resolves() {
// Reproduction of the field bug: a user's cached segmentation-3.0.onnx
// was an older export whose single output node is named "y", not
// "output". The model loads fine (so the cache never re-downloads), but
// the old inference code hard-coded `ort_outs.get("output")`, which
// returns None on every chunk -> "Output tensor not found" -> the whole
// transcription backlog wedges while screen/audio capture keeps running.
let stale_model_outputs = vec!["y".to_string()];
// Old behavior: the hard-coded canonical name simply isn't present.
assert!(!stale_model_outputs.iter().any(|n| n == "output"));
// New behavior: we fall back to the model's sole output and proceed,
// so a structurally-identical-but-renamed export transcribes normally.
assert_eq!(
resolve_output_name(&stale_model_outputs, "output").unwrap(),
"y"
);
}
#[test]
fn resolve_output_name_errors_on_no_outputs() {
let outputs: Vec<String> = vec![];
assert!(resolve_output_name(&outputs, "output").is_err());
}
#[test]
fn resolve_output_name_errors_when_ambiguous_and_unmatched() {
// Multiple outputs and none canonical: refuse to guess.
let outputs = vec!["a".to_string(), "b".to_string()];
let err = resolve_output_name(&outputs, "output")
.unwrap_err()
.to_string();
assert!(err.contains("no safe default"));
}
// Real-ORT reproduction harness against the model we actually ship. Loads
// the canonical segmentation model, resolves its output via the same path
// production uses, and runs a forward pass to prove the resolved name is
// really retrievable from the run results. `#[ignore]` like the other
// model-dependent tests in this crate (needs the on-disk model + ORT).
#[test]
#[ignore]
fn segmentation_model_output_resolves_against_real_session() {
use std::path::PathBuf;
let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("models")
.join("pyannote")
.join("segmentation-3.0.onnx");
let mut session = create_session(&model_path).expect("failed to load segmentation model");
// Resolve the output name the way SegmentIterator now does.
let output_name = resolve_output_name(&session_output_names(&session), "output")
.expect("failed to resolve segmentation output name");
// Forward pass on a 10s window of silence (matches the segmentation
// window size used in segment.rs: sample_rate * 10 at 16 kHz).
let window = vec![0.0f32; 16000 * 10];
let array = ndarray::Array1::from_vec(window)
.view()
.insert_axis(ndarray::Axis(0))
.insert_axis(ndarray::Axis(1))
.to_owned();
let inputs = ort::inputs![ort::value::TensorRef::from_array_view(array.view()).unwrap()];
let ort_outs = session.run(inputs).expect("session run failed");
// The resolved name must actually exist in the run results — this is
// exactly the lookup that returned None for the stale "y" model.
assert!(
ort_outs.get(&output_name).is_some(),
"resolved output '{output_name}' not present in run results"
);
}
}