YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(redact): batch local ONNX reconciliation (#6577)
Louis Beaumont5e6f37ee752b
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex daed6be9d..75e4c0ec7 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -1267,10 +1267,11 @@ impl ServerCore {let pipeline = pipeline.with_pseudonyms(pseudonymizer);let pipeline_arc = Arc::new(pipeline) as Arc<dyn Redactor>;let cfg = WorkerConfig {- // Local inference is CPU-bound. Keep bursts short so- // the adaptive whole-process 30% controller can react- // quickly; cloud/enclave batching remains at 16.- batch_size: 4,+ // Match the local ONNX inference width. One batched+ // call finishes faster than the former four serial+ // calls on the element backlog, while the adaptive+ // whole-process controller still enforces 30% CPU.+ batch_size: 16,tables: ALL_TARGET_TABLES.to_vec(),..Default::default()};diff --git a/crates/screenpipe-redact/src/adapters/onnx.rs b/crates/screenpipe-redact/src/adapters/onnx.rsindex d6220281a..668b43169 100644--- a/crates/screenpipe-redact/src/adapters/onnx.rs+++ b/crates/screenpipe-redact/src/adapters/onnx.rs@@ -332,10 +332,19 @@ mod runtime {use ndarray::{Array, Axis};use ort::session::{builder::GraphOptimizationLevel, Session};use ort::value::TensorRef;- use std::collections::HashMap;+ use std::collections::{BTreeMap, HashMap};use std::sync::Mutex;use tokenizers::Tokenizer;+ const ONNX_INFERENCE_BATCH_SIZE: usize = 16;++ struct EncodedInput {+ output_index: usize,+ ids: Vec<u32>,+ mask: Vec<u32>,+ offsets: Vec<(usize, usize)>,+ }+/// Loaded model + tokenizer + label vocabulary.pub struct OnnxRedactor {cfg: OnnxConfig,@@ -461,28 +470,141 @@ mod runtime {})}+ /// Tokenize a group of short inputs, bucket them by sequence length,+ /// and execute each bucket as one ONNX batch. Long inputs retain the+ /// established overlapping-window path so batching cannot reintroduce+ /// truncation at the model context boundary.+ fn infer_batch(&self, texts: &[String]) -> Result<Vec<RedactionOutput>, RedactError> {+ let mut outputs: Vec<Option<RedactionOutput>> = vec![None; texts.len()];+ let mut buckets: BTreeMap<usize, Vec<EncodedInput>> = BTreeMap::new();+ let max_len = self.cfg.max_seq_len.max(3);++ for (output_index, text) in texts.iter().enumerate() {+ if text.is_empty() {+ outputs[output_index] = Some(RedactionOutput {+ input: String::new(),+ redacted: String::new(),+ spans: Vec::new(),+ });+ continue;+ }++ let encoding = self+ .tokenizer+ .encode(text.as_str(), true)+ .map_err(|error| RedactError::Runtime(format!("tokenize: {error}")))?;+ if encoding.len() > max_len {+ outputs[output_index] = Some(self.infer(text)?);+ continue;+ }++ // Power-of-two buckets bound padding waste while still+ // coalescing the many similarly sized accessibility strings.+ let bucket = encoding.len().max(1).next_power_of_two();+ buckets.entry(bucket).or_default().push(EncodedInput {+ output_index,+ ids: encoding.get_ids().to_vec(),+ mask: encoding.get_attention_mask().to_vec(),+ offsets: encoding.get_offsets().to_vec(),+ });+ }++ for entries in buckets.values() {+ for chunk in entries.chunks(ONNX_INFERENCE_BATCH_SIZE) {+ let windows: Vec<(&[u32], &[u32])> = chunk+ .iter()+ .map(|entry| (entry.ids.as_slice(), entry.mask.as_slice()))+ .collect();+ let label_batches = self.run_windows(&windows)?;+ for (entry, label_ids) in chunk.iter().zip(label_batches) {+ let text = &texts[entry.output_index];+ let mut spans =+ bio_decode(text, &label_ids, &entry.offsets, &self.id2label);+ merge_spans(&mut spans, text);+ outputs[entry.output_index] = Some(RedactionOutput {+ input: text.clone(),+ redacted: render_redacted(text, &spans),+ spans,+ });+ }+ }+ }++ outputs+ .into_iter()+ .enumerate()+ .map(|(index, output)| {+ output.ok_or_else(|| {+ RedactError::Unexpected(format!(+ "ONNX batch did not produce output at index {index}"+ ))+ })+ })+ .collect()+ }+/// Run one window of token ids (already `<= max_seq_len`, framed/// with the model's special tokens) through the session and/// return the per-token argmax label ids.fn run_window(&self, ids: &[u32], mask: &[u32]) -> Result<Vec<usize>, RedactError> {- let len = ids.len();+ self.run_windows(&[(ids, mask)])?+ .pop()+ .ok_or_else(|| RedactError::Unexpected("empty ONNX window result".into()))+ }++ /// Run multiple token windows in one ONNX invocation. Inputs may have+ /// different lengths; shorter rows are padded and masked within their+ /// length bucket.+ fn run_windows(+ &self,+ windows: &[(&[u32], &[u32])],+ ) -> Result<Vec<Vec<usize>>, RedactError> {+ if windows.is_empty() {+ return Ok(Vec::new());+ }+ let width = windows+ .iter()+ .map(|(ids, _)| ids.len())+ .max()+ .ok_or_else(|| RedactError::Unexpected("empty ONNX window batch".into()))?;+ if width == 0 {+ return Err(RedactError::Unexpected(+ "ONNX window batch contains an empty token sequence".into(),+ ));+ }+ let batch_size = windows.len();+ let pad_id = self.tokenizer.token_to_id("<pad>").unwrap_or(0);// Vocab-pruned models: remap full-tokenizer ids → sliced rows// (ids outside the kept set → the pruned UNK row). Byte offsets are// unaffected — they index `text`, not the model vocab — so decoding// stays correct. Full-vocab models (remap=None) pass ids through.- let input_ids: Vec<i64> = match &self.remap {- Some(remap) => ids- .iter()- .map(|x| *remap.get(x).unwrap_or(&self.unk_remapped) as i64)- .collect(),- None => ids.iter().map(|x| *x as i64).collect(),+ let remap_id = |id: u32| -> i64 {+ match &self.remap {+ Some(remap) => *remap.get(&id).unwrap_or(&self.unk_remapped) as i64,+ None => id as i64,+ }};- let attention_mask: Vec<i64> = mask.iter().map(|x| *x as i64).collect();+ let mut input_ids = vec![remap_id(pad_id); batch_size * width];+ let mut attention_mask = vec![0i64; batch_size * width];+ for (batch_index, (ids, mask)) in windows.iter().enumerate() {+ if ids.len() != mask.len() {+ return Err(RedactError::Unexpected(format!(+ "ONNX ids/mask length mismatch: {} != {}",+ ids.len(),+ mask.len()+ )));+ }+ let row_start = batch_index * width;+ for (offset, id) in ids.iter().enumerate() {+ input_ids[row_start + offset] = remap_id(*id);+ attention_mask[row_start + offset] = mask[offset] as i64;+ }+ }- // ndarray shapes: [batch=1, seq_len]- let ids_arr = Array::from_shape_vec((1, len), input_ids)+ // ndarray shapes: [batch, padded_seq_len]+ let ids_arr = Array::from_shape_vec((batch_size, width), input_ids).map_err(|e| RedactError::Runtime(format!("ids shape: {e}")))?;- let mask_arr = Array::from_shape_vec((1, len), attention_mask)+ let mask_arr = Array::from_shape_vec((batch_size, width), attention_mask).map_err(|e| RedactError::Runtime(format!("mask shape: {e}")))?;let mut session = self@@ -497,7 +619,7 @@ mod runtime {]).map_err(|e| RedactError::Runtime(format!("session.run: {e}")))?;- // logits shape: [1, seq_len, num_labels]+ // logits shape: [batch, padded_seq_len, num_labels]let logits = outputs.get("logits").ok_or_else(|| RedactError::Runtime("no logits output".into()))?;@@ -505,23 +627,39 @@ mod runtime {.try_extract_array::<f32>().map_err(|e| RedactError::Runtime(format!("extract logits: {e}")))?;let logits_view = logits_view.view();- let logits = logits_view.index_axis(Axis(0), 0); // drop batch dim → [seq_len, num_labels]-- // Argmax per token- let mut label_ids = Vec::with_capacity(len);- for row in logits.axis_iter(Axis(0)) {- let mut best_i = 0usize;- let mut best_v = f32::NEG_INFINITY;- for (i, v) in row.iter().enumerate() {- if *v > best_v {- best_v = *v;- best_i = i;+ if logits_view.ndim() != 3 || logits_view.len_of(Axis(0)) != batch_size {+ return Err(RedactError::Runtime(format!(+ "unexpected ONNX logits shape {:?} for batch size {batch_size}",+ logits_view.shape()+ )));+ }++ let mut batches = Vec::with_capacity(batch_size);+ for (batch_index, (ids, _)) in windows.iter().enumerate() {+ let batch_logits = logits_view.index_axis(Axis(0), batch_index);+ if batch_logits.len_of(Axis(0)) < ids.len() {+ return Err(RedactError::Runtime(format!(+ "ONNX logits shorter than input: {} < {}",+ batch_logits.len_of(Axis(0)),+ ids.len()+ )));+ }+ let mut label_ids = Vec::with_capacity(ids.len());+ for row in batch_logits.axis_iter(Axis(0)).take(ids.len()) {+ let mut best_i = 0usize;+ let mut best_v = f32::NEG_INFINITY;+ for (i, value) in row.iter().enumerate() {+ if *value > best_v {+ best_v = *value;+ best_i = i;+ }}+ label_ids.push(best_i);}- label_ids.push(best_i);+ batches.push(label_ids);}// outputs (and the session borrow) drop here at end of block.- Ok(label_ids)+ Ok(batches)}/// Inference for inputs longer than the model context. Slides an@@ -585,15 +723,14 @@ mod runtime {fn version(&self) -> u32 {ONNX_REDACTOR_VERSION}+ fn preferred_batch_size(&self) -> usize {+ ONNX_INFERENCE_BATCH_SIZE+ }async fn redact_batch(&self,texts: &[String],) -> Result<Vec<RedactionOutput>, RedactError> {- let mut out = Vec::with_capacity(texts.len());- for t in texts {- out.push(self.infer(t)?);- }- Ok(out)+ self.infer_batch(texts)}}diff --git a/crates/screenpipe-redact/src/lib.rs b/crates/screenpipe-redact/src/lib.rsindex 33cf0d5bb..741e51e6d 100644--- a/crates/screenpipe-redact/src/lib.rs+++ b/crates/screenpipe-redact/src/lib.rs@@ -175,6 +175,17 @@ pub trait Redactor: Send + Sync {/// trait for logs and human triage.fn version(&self) -> u32;+ /// Preferred number of inputs for one backend inference call.+ ///+ /// The default preserves the historical one-input-at-a-time behavior for+ /// adapters whose `redact_batch` implementation is only a convenience+ /// loop (or a sequence of remote requests). Backends with a real batched+ /// execution path can opt in so [`Pipeline`] coalesces cache misses before+ /// invoking them.+ fn preferred_batch_size(&self) -> usize {+ 1+ }+/// Redact one input. Default impl forwards to `redact_batch` so/// implementors can focus on the batch path.async fn redact(&self, text: &str) -> Result<RedactionOutput, RedactError> {diff --git a/crates/screenpipe-redact/src/pipeline.rs b/crates/screenpipe-redact/src/pipeline.rsindex 0124db5e5..b3c2c7f7b 100644--- a/crates/screenpipe-redact/src/pipeline.rs+++ b/crates/screenpipe-redact/src/pipeline.rs@@ -23,7 +23,7 @@//! Substring search is fine here because we already know the AI//! produced a replacement — we just need offsets for the audit trail.-use std::{sync::Arc, time::Duration};+use std::{collections::HashMap, sync::Arc, time::Duration};use async_trait::async_trait;use tokio::{sync::Mutex, time::Instant};@@ -272,6 +272,137 @@ impl Pipeline {self.pseudonyms = if is_enclave { None } else { pseudonyms };self}++ /// Run cache misses through a backend that has a genuine batched+ /// inference path. Remote adapters keep the serial path below so one+ /// failed request does not discard earlier successful responses.+ async fn redact_batched(+ &self,+ texts: &[String],+ batch_size: usize,+ ) -> Result<Vec<RedactionOutput>, RedactError> {+ struct Pending {+ output_indices: Vec<usize>,+ key: [u8; 32],+ current: RedactionOutput,+ }++ let Some(ai) = self.ai.as_ref() else {+ return Err(RedactError::Unexpected(+ "batched pipeline selected without an AI adapter".into(),+ ));+ };+ let mut results: Vec<Option<RedactionOutput>> = vec![None; texts.len()];+ let mut pending: Vec<Pending> = Vec::new();+ let mut pending_by_key: HashMap<[u8; 32], usize> = HashMap::new();++ for (output_index, text) in texts.iter().enumerate() {+ let key = cache_key(text, self.name(), self.version());+ if let Some(hit) = self.cache.get(&key).await {+ results[output_index] = Some((*hit).clone());+ continue;+ }++ let regex_out = regex_adapter::redact_one(text);+ let current = apply_policy(regex_out, &self.cfg.policy, self.pseudonyms.as_deref());+ let want_ai = current.input.chars().count() >= self.cfg.ai_min_chars+ && current.spans.len() < self.cfg.ai_skip_if_regex_spans;++ if !want_ai {+ self.cache.insert(key, current.clone()).await;+ results[output_index] = Some(current);+ continue;+ }++ // A capture batch contains a lot of repeated accessibility chrome.+ // Coalesce equal cache misses before inference, then fan the single+ // result back out to every original position.+ if let Some(&pending_index) = pending_by_key.get(&key) {+ pending[pending_index].output_indices.push(output_index);+ } else {+ pending_by_key.insert(key, pending.len());+ pending.push(Pending {+ output_indices: vec![output_index],+ key,+ current,+ });+ }+ }++ for chunk in pending.chunks_mut(batch_size.max(1)) {+ let ai_inputs: Vec<String> = chunk+ .iter()+ .map(|item| item.current.redacted.clone())+ .collect();+ let ai_result = ai.redact_batch(&ai_inputs).await.and_then(|outputs| {+ if outputs.len() == chunk.len() {+ Ok(outputs)+ } else {+ Err(RedactError::Unexpected(format!(+ "AI redactor returned {} outputs for {} inputs",+ outputs.len(),+ chunk.len()+ )))+ }+ });++ match ai_result {+ Ok(ai_outputs) => {+ for (item, ai_out) in chunk.iter_mut().zip(ai_outputs) {+ let redacted = if ai_out.spans.is_empty() {+ ai_out.redacted+ } else {+ apply_policy(ai_out, &self.cfg.policy, self.pseudonyms.as_deref())+ .redacted+ };+ item.current = RedactionOutput {+ input: std::mem::take(&mut item.current.input),+ redacted,+ spans: std::mem::take(&mut item.current.spans),+ };+ }+ }+ Err(RedactError::Unavailable(_)) => {+ // Keep the regex outputs, matching the serial path.+ }+ Err(error) => {+ let mut detail = error.to_string();+ let mut source: Option<&dyn std::error::Error> =+ std::error::Error::source(&error);+ while let Some(next) = source {+ detail.push_str(" → ");+ detail.push_str(&next.to_string());+ source = next.source();+ }+ tracing::warn!(+ error = %error,+ detail = %detail,+ batch_rows = chunk.len(),+ "batched AI redactor failed; falling back to regex-only output"+ );+ }+ }++ for item in chunk {+ self.cache.insert(item.key, item.current.clone()).await;+ for &output_index in &item.output_indices {+ results[output_index] = Some(item.current.clone());+ }+ }+ }++ results+ .into_iter()+ .enumerate()+ .map(|(index, output)| {+ output.ok_or_else(|| {+ RedactError::Unexpected(format!(+ "batched pipeline did not produce output at index {index}"+ ))+ })+ })+ .collect()+ }}#[async_trait]@@ -298,6 +429,15 @@ impl Redactor for Pipeline {}async fn redact_batch(&self, texts: &[String]) -> Result<Vec<RedactionOutput>, RedactError> {+ let preferred_batch_size = self+ .ai+ .as_ref()+ .map(|ai| ai.preferred_batch_size())+ .unwrap_or(1);+ if preferred_batch_size > 1 && self.ai_failure_circuit.is_none() {+ return self.redact_batched(texts, preferred_batch_size).await;+ }+let mut out = Vec::with_capacity(texts.len());for text in texts {@@ -467,6 +607,12 @@ mod tests {calls: AtomicUsize,}+ /// Local model-shaped test redactor with a real batch preference.+ struct BatchingAi {+ batches: AtomicUsize,+ inputs: AtomicUsize,+ }+impl UppercaseAi {fn new() -> Self {Self {@@ -475,6 +621,15 @@ mod tests {}}+ impl BatchingAi {+ fn new() -> Self {+ Self {+ batches: AtomicUsize::new(0),+ inputs: AtomicUsize::new(0),+ }+ }+ }+/// Cloud-shaped test redactor that fails a configured number of calls,/// then returns a visible successful result.struct FlakyAi {@@ -550,6 +705,37 @@ mod tests {}}+ #[async_trait]+ impl Redactor for BatchingAi {+ fn name(&self) -> &str {+ "test_onnx"+ }++ fn version(&self) -> u32 {+ 1+ }++ fn preferred_batch_size(&self) -> usize {+ 8+ }++ async fn redact_batch(+ &self,+ texts: &[String],+ ) -> Result<Vec<RedactionOutput>, RedactError> {+ self.batches.fetch_add(1, Ordering::SeqCst);+ self.inputs.fetch_add(texts.len(), Ordering::SeqCst);+ Ok(texts+ .iter()+ .map(|text| RedactionOutput {+ input: text.clone(),+ redacted: text.to_uppercase(),+ spans: vec![],+ })+ .collect())+ }+ }+#[tokio::test]async fn regex_only_runs_without_ai_secret_policy() {// Default policy is `allow=[Secret]`, so a bare email is NOT@@ -579,6 +765,37 @@ mod tests {assert_eq!(ai.calls.load(Ordering::SeqCst), 1);}+ #[tokio::test]+ async fn local_ai_batches_unique_cache_misses() {+ let ai = Arc::new(BatchingAi::new());+ let pipeline = Pipeline::regex_then_ai(ai.clone(), PipelineConfig::default());+ let first = "first accessibility string long enough for inference".to_string();+ let second = "second accessibility string long enough for inference".to_string();+ let texts = vec![+ first.clone(),+ second.clone(),+ first.clone(),+ "short".to_string(),+ ];++ let outputs = pipeline.redact_batch(&texts).await.unwrap();++ assert_eq!(outputs[0].redacted, first.to_uppercase());+ assert_eq!(outputs[1].redacted, second.to_uppercase());+ assert_eq!(outputs[2], outputs[0]);+ assert_eq!(outputs[3].redacted, "short");+ assert_eq!(ai.batches.load(Ordering::SeqCst), 1);+ assert_eq!(+ ai.inputs.load(Ordering::SeqCst),+ 2,+ "duplicate cache misses must share one inference"+ );++ let cached = pipeline.redact_batch(&texts).await.unwrap();+ assert_eq!(cached, outputs);+ assert_eq!(ai.batches.load(Ordering::SeqCst), 1);+ }+#[tokio::test]async fn span_less_ai_output_is_trusted() {// A span-less adapter (like the Tinfoil enclave, which returns