Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

perf(redact): batch local ONNX reconciliation (#6577)

Louis Beaumont5e6f37ee752b
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
index daed6be9d..75e4c0ec7 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs
@@ -1267,10 +1267,11 @@ impl ServerCore {
let pipeline = pipeline.with_pseudonyms(pseudonymizer);
let pipeline_arc = Arc::new(pipeline) as Arc<dyn Redactor>;
let cfg = WorkerConfig {
- // Local inference is CPU-bound. Keep bursts short so
- // the adaptive whole-process 30% controller can react
- // quickly; cloud/enclave batching remains at 16.
- batch_size: 4,
+ // Match the local ONNX inference width. One batched
+ // call finishes faster than the former four serial
+ // calls on the element backlog, while the adaptive
+ // whole-process controller still enforces 30% CPU.
+ batch_size: 16,
tables: ALL_TARGET_TABLES.to_vec(),
..Default::default()
};
diff --git a/crates/screenpipe-redact/src/adapters/onnx.rs b/crates/screenpipe-redact/src/adapters/onnx.rs
index d6220281a..668b43169 100644
--- a/crates/screenpipe-redact/src/adapters/onnx.rs
+++ b/crates/screenpipe-redact/src/adapters/onnx.rs
@@ -332,10 +332,19 @@ mod runtime {
use ndarray::{Array, Axis};
use ort::session::{builder::GraphOptimizationLevel, Session};
use ort::value::TensorRef;
- use std::collections::HashMap;
+ use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use tokenizers::Tokenizer;
+ const ONNX_INFERENCE_BATCH_SIZE: usize = 16;
+
+ struct EncodedInput {
+ output_index: usize,
+ ids: Vec<u32>,
+ mask: Vec<u32>,
+ offsets: Vec<(usize, usize)>,
+ }
+
/// Loaded model + tokenizer + label vocabulary.
pub struct OnnxRedactor {
cfg: OnnxConfig,
@@ -461,28 +470,141 @@ mod runtime {
})
}
+ /// Tokenize a group of short inputs, bucket them by sequence length,
+ /// and execute each bucket as one ONNX batch. Long inputs retain the
+ /// established overlapping-window path so batching cannot reintroduce
+ /// truncation at the model context boundary.
+ fn infer_batch(&self, texts: &[String]) -> Result<Vec<RedactionOutput>, RedactError> {
+ let mut outputs: Vec<Option<RedactionOutput>> = vec![None; texts.len()];
+ let mut buckets: BTreeMap<usize, Vec<EncodedInput>> = BTreeMap::new();
+ let max_len = self.cfg.max_seq_len.max(3);
+
+ for (output_index, text) in texts.iter().enumerate() {
+ if text.is_empty() {
+ outputs[output_index] = Some(RedactionOutput {
+ input: String::new(),
+ redacted: String::new(),
+ spans: Vec::new(),
+ });
+ continue;
+ }
+
+ let encoding = self
+ .tokenizer
+ .encode(text.as_str(), true)
+ .map_err(|error| RedactError::Runtime(format!("tokenize: {error}")))?;
+ if encoding.len() > max_len {
+ outputs[output_index] = Some(self.infer(text)?);
+ continue;
+ }
+
+ // Power-of-two buckets bound padding waste while still
+ // coalescing the many similarly sized accessibility strings.
+ let bucket = encoding.len().max(1).next_power_of_two();
+ buckets.entry(bucket).or_default().push(EncodedInput {
+ output_index,
+ ids: encoding.get_ids().to_vec(),
+ mask: encoding.get_attention_mask().to_vec(),
+ offsets: encoding.get_offsets().to_vec(),
+ });
+ }
+
+ for entries in buckets.values() {
+ for chunk in entries.chunks(ONNX_INFERENCE_BATCH_SIZE) {
+ let windows: Vec<(&[u32], &[u32])> = chunk
+ .iter()
+ .map(|entry| (entry.ids.as_slice(), entry.mask.as_slice()))
+ .collect();
+ let label_batches = self.run_windows(&windows)?;
+ for (entry, label_ids) in chunk.iter().zip(label_batches) {
+ let text = &texts[entry.output_index];
+ let mut spans =
+ bio_decode(text, &label_ids, &entry.offsets, &self.id2label);
+ merge_spans(&mut spans, text);
+ outputs[entry.output_index] = Some(RedactionOutput {
+ input: text.clone(),
+ redacted: render_redacted(text, &spans),
+ spans,
+ });
+ }
+ }
+ }
+
+ outputs
+ .into_iter()
+ .enumerate()
+ .map(|(index, output)| {
+ output.ok_or_else(|| {
+ RedactError::Unexpected(format!(
+ "ONNX batch did not produce output at index {index}"
+ ))
+ })
+ })
+ .collect()
+ }
+
/// Run one window of token ids (already `<= max_seq_len`, framed
/// with the model's special tokens) through the session and
/// return the per-token argmax label ids.
fn run_window(&self, ids: &[u32], mask: &[u32]) -> Result<Vec<usize>, RedactError> {
- let len = ids.len();
+ self.run_windows(&[(ids, mask)])?
+ .pop()
+ .ok_or_else(|| RedactError::Unexpected("empty ONNX window result".into()))
+ }
+
+ /// Run multiple token windows in one ONNX invocation. Inputs may have
+ /// different lengths; shorter rows are padded and masked within their
+ /// length bucket.
+ fn run_windows(
+ &self,
+ windows: &[(&[u32], &[u32])],
+ ) -> Result<Vec<Vec<usize>>, RedactError> {
+ if windows.is_empty() {
+ return Ok(Vec::new());
+ }
+ let width = windows
+ .iter()
+ .map(|(ids, _)| ids.len())
+ .max()
+ .ok_or_else(|| RedactError::Unexpected("empty ONNX window batch".into()))?;
+ if width == 0 {
+ return Err(RedactError::Unexpected(
+ "ONNX window batch contains an empty token sequence".into(),
+ ));
+ }
+ let batch_size = windows.len();
+ let pad_id = self.tokenizer.token_to_id("<pad>").unwrap_or(0);
// Vocab-pruned models: remap full-tokenizer ids → sliced rows
// (ids outside the kept set → the pruned UNK row). Byte offsets are
// unaffected — they index `text`, not the model vocab — so decoding
// stays correct. Full-vocab models (remap=None) pass ids through.
- let input_ids: Vec<i64> = match &self.remap {
- Some(remap) => ids
- .iter()
- .map(|x| *remap.get(x).unwrap_or(&self.unk_remapped) as i64)
- .collect(),
- None => ids.iter().map(|x| *x as i64).collect(),
+ let remap_id = |id: u32| -> i64 {
+ match &self.remap {
+ Some(remap) => *remap.get(&id).unwrap_or(&self.unk_remapped) as i64,
+ None => id as i64,
+ }
};
- let attention_mask: Vec<i64> = mask.iter().map(|x| *x as i64).collect();
+ let mut input_ids = vec![remap_id(pad_id); batch_size * width];
+ let mut attention_mask = vec![0i64; batch_size * width];
+ for (batch_index, (ids, mask)) in windows.iter().enumerate() {
+ if ids.len() != mask.len() {
+ return Err(RedactError::Unexpected(format!(
+ "ONNX ids/mask length mismatch: {} != {}",
+ ids.len(),
+ mask.len()
+ )));
+ }
+ let row_start = batch_index * width;
+ for (offset, id) in ids.iter().enumerate() {
+ input_ids[row_start + offset] = remap_id(*id);
+ attention_mask[row_start + offset] = mask[offset] as i64;
+ }
+ }
- // ndarray shapes: [batch=1, seq_len]
- let ids_arr = Array::from_shape_vec((1, len), input_ids)
+ // ndarray shapes: [batch, padded_seq_len]
+ let ids_arr = Array::from_shape_vec((batch_size, width), input_ids)
.map_err(|e| RedactError::Runtime(format!("ids shape: {e}")))?;
- let mask_arr = Array::from_shape_vec((1, len), attention_mask)
+ let mask_arr = Array::from_shape_vec((batch_size, width), attention_mask)
.map_err(|e| RedactError::Runtime(format!("mask shape: {e}")))?;
let mut session = self
@@ -497,7 +619,7 @@ mod runtime {
])
.map_err(|e| RedactError::Runtime(format!("session.run: {e}")))?;
- // logits shape: [1, seq_len, num_labels]
+ // logits shape: [batch, padded_seq_len, num_labels]
let logits = outputs
.get("logits")
.ok_or_else(|| RedactError::Runtime("no logits output".into()))?;
@@ -505,23 +627,39 @@ mod runtime {
.try_extract_array::<f32>()
.map_err(|e| RedactError::Runtime(format!("extract logits: {e}")))?;
let logits_view = logits_view.view();
- let logits = logits_view.index_axis(Axis(0), 0); // drop batch dim → [seq_len, num_labels]
-
- // Argmax per token
- let mut label_ids = Vec::with_capacity(len);
- for row in logits.axis_iter(Axis(0)) {
- let mut best_i = 0usize;
- let mut best_v = f32::NEG_INFINITY;
- for (i, v) in row.iter().enumerate() {
- if *v > best_v {
- best_v = *v;
- best_i = i;
+ if logits_view.ndim() != 3 || logits_view.len_of(Axis(0)) != batch_size {
+ return Err(RedactError::Runtime(format!(
+ "unexpected ONNX logits shape {:?} for batch size {batch_size}",
+ logits_view.shape()
+ )));
+ }
+
+ let mut batches = Vec::with_capacity(batch_size);
+ for (batch_index, (ids, _)) in windows.iter().enumerate() {
+ let batch_logits = logits_view.index_axis(Axis(0), batch_index);
+ if batch_logits.len_of(Axis(0)) < ids.len() {
+ return Err(RedactError::Runtime(format!(
+ "ONNX logits shorter than input: {} < {}",
+ batch_logits.len_of(Axis(0)),
+ ids.len()
+ )));
+ }
+ let mut label_ids = Vec::with_capacity(ids.len());
+ for row in batch_logits.axis_iter(Axis(0)).take(ids.len()) {
+ let mut best_i = 0usize;
+ let mut best_v = f32::NEG_INFINITY;
+ for (i, value) in row.iter().enumerate() {
+ if *value > best_v {
+ best_v = *value;
+ best_i = i;
+ }
}
+ label_ids.push(best_i);
}
- label_ids.push(best_i);
+ batches.push(label_ids);
}
// outputs (and the session borrow) drop here at end of block.
- Ok(label_ids)
+ Ok(batches)
}
/// Inference for inputs longer than the model context. Slides an
@@ -585,15 +723,14 @@ mod runtime {
fn version(&self) -> u32 {
ONNX_REDACTOR_VERSION
}
+ fn preferred_batch_size(&self) -> usize {
+ ONNX_INFERENCE_BATCH_SIZE
+ }
async fn redact_batch(
&self,
texts: &[String],
) -> Result<Vec<RedactionOutput>, RedactError> {
- let mut out = Vec::with_capacity(texts.len());
- for t in texts {
- out.push(self.infer(t)?);
- }
- Ok(out)
+ self.infer_batch(texts)
}
}
diff --git a/crates/screenpipe-redact/src/lib.rs b/crates/screenpipe-redact/src/lib.rs
index 33cf0d5bb..741e51e6d 100644
--- a/crates/screenpipe-redact/src/lib.rs
+++ b/crates/screenpipe-redact/src/lib.rs
@@ -175,6 +175,17 @@ pub trait Redactor: Send + Sync {
/// trait for logs and human triage.
fn version(&self) -> u32;
+ /// Preferred number of inputs for one backend inference call.
+ ///
+ /// The default preserves the historical one-input-at-a-time behavior for
+ /// adapters whose `redact_batch` implementation is only a convenience
+ /// loop (or a sequence of remote requests). Backends with a real batched
+ /// execution path can opt in so [`Pipeline`] coalesces cache misses before
+ /// invoking them.
+ fn preferred_batch_size(&self) -> usize {
+ 1
+ }
+
/// Redact one input. Default impl forwards to `redact_batch` so
/// implementors can focus on the batch path.
async fn redact(&self, text: &str) -> Result<RedactionOutput, RedactError> {
diff --git a/crates/screenpipe-redact/src/pipeline.rs b/crates/screenpipe-redact/src/pipeline.rs
index 0124db5e5..b3c2c7f7b 100644
--- a/crates/screenpipe-redact/src/pipeline.rs
+++ b/crates/screenpipe-redact/src/pipeline.rs
@@ -23,7 +23,7 @@
//! Substring search is fine here because we already know the AI
//! produced a replacement — we just need offsets for the audit trail.
-use std::{sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc, time::Duration};
use async_trait::async_trait;
use tokio::{sync::Mutex, time::Instant};
@@ -272,6 +272,137 @@ impl Pipeline {
self.pseudonyms = if is_enclave { None } else { pseudonyms };
self
}
+
+ /// Run cache misses through a backend that has a genuine batched
+ /// inference path. Remote adapters keep the serial path below so one
+ /// failed request does not discard earlier successful responses.
+ async fn redact_batched(
+ &self,
+ texts: &[String],
+ batch_size: usize,
+ ) -> Result<Vec<RedactionOutput>, RedactError> {
+ struct Pending {
+ output_indices: Vec<usize>,
+ key: [u8; 32],
+ current: RedactionOutput,
+ }
+
+ let Some(ai) = self.ai.as_ref() else {
+ return Err(RedactError::Unexpected(
+ "batched pipeline selected without an AI adapter".into(),
+ ));
+ };
+ let mut results: Vec<Option<RedactionOutput>> = vec![None; texts.len()];
+ let mut pending: Vec<Pending> = Vec::new();
+ let mut pending_by_key: HashMap<[u8; 32], usize> = HashMap::new();
+
+ for (output_index, text) in texts.iter().enumerate() {
+ let key = cache_key(text, self.name(), self.version());
+ if let Some(hit) = self.cache.get(&key).await {
+ results[output_index] = Some((*hit).clone());
+ continue;
+ }
+
+ let regex_out = regex_adapter::redact_one(text);
+ let current = apply_policy(regex_out, &self.cfg.policy, self.pseudonyms.as_deref());
+ let want_ai = current.input.chars().count() >= self.cfg.ai_min_chars
+ && current.spans.len() < self.cfg.ai_skip_if_regex_spans;
+
+ if !want_ai {
+ self.cache.insert(key, current.clone()).await;
+ results[output_index] = Some(current);
+ continue;
+ }
+
+ // A capture batch contains a lot of repeated accessibility chrome.
+ // Coalesce equal cache misses before inference, then fan the single
+ // result back out to every original position.
+ if let Some(&pending_index) = pending_by_key.get(&key) {
+ pending[pending_index].output_indices.push(output_index);
+ } else {
+ pending_by_key.insert(key, pending.len());
+ pending.push(Pending {
+ output_indices: vec![output_index],
+ key,
+ current,
+ });
+ }
+ }
+
+ for chunk in pending.chunks_mut(batch_size.max(1)) {
+ let ai_inputs: Vec<String> = chunk
+ .iter()
+ .map(|item| item.current.redacted.clone())
+ .collect();
+ let ai_result = ai.redact_batch(&ai_inputs).await.and_then(|outputs| {
+ if outputs.len() == chunk.len() {
+ Ok(outputs)
+ } else {
+ Err(RedactError::Unexpected(format!(
+ "AI redactor returned {} outputs for {} inputs",
+ outputs.len(),
+ chunk.len()
+ )))
+ }
+ });
+
+ match ai_result {
+ Ok(ai_outputs) => {
+ for (item, ai_out) in chunk.iter_mut().zip(ai_outputs) {
+ let redacted = if ai_out.spans.is_empty() {
+ ai_out.redacted
+ } else {
+ apply_policy(ai_out, &self.cfg.policy, self.pseudonyms.as_deref())
+ .redacted
+ };
+ item.current = RedactionOutput {
+ input: std::mem::take(&mut item.current.input),
+ redacted,
+ spans: std::mem::take(&mut item.current.spans),
+ };
+ }
+ }
+ Err(RedactError::Unavailable(_)) => {
+ // Keep the regex outputs, matching the serial path.
+ }
+ Err(error) => {
+ let mut detail = error.to_string();
+ let mut source: Option<&dyn std::error::Error> =
+ std::error::Error::source(&error);
+ while let Some(next) = source {
+ detail.push_str(" → ");
+ detail.push_str(&next.to_string());
+ source = next.source();
+ }
+ tracing::warn!(
+ error = %error,
+ detail = %detail,
+ batch_rows = chunk.len(),
+ "batched AI redactor failed; falling back to regex-only output"
+ );
+ }
+ }
+
+ for item in chunk {
+ self.cache.insert(item.key, item.current.clone()).await;
+ for &output_index in &item.output_indices {
+ results[output_index] = Some(item.current.clone());
+ }
+ }
+ }
+
+ results
+ .into_iter()
+ .enumerate()
+ .map(|(index, output)| {
+ output.ok_or_else(|| {
+ RedactError::Unexpected(format!(
+ "batched pipeline did not produce output at index {index}"
+ ))
+ })
+ })
+ .collect()
+ }
}
#[async_trait]
@@ -298,6 +429,15 @@ impl Redactor for Pipeline {
}
async fn redact_batch(&self, texts: &[String]) -> Result<Vec<RedactionOutput>, RedactError> {
+ let preferred_batch_size = self
+ .ai
+ .as_ref()
+ .map(|ai| ai.preferred_batch_size())
+ .unwrap_or(1);
+ if preferred_batch_size > 1 && self.ai_failure_circuit.is_none() {
+ return self.redact_batched(texts, preferred_batch_size).await;
+ }
+
let mut out = Vec::with_capacity(texts.len());
for text in texts {
@@ -467,6 +607,12 @@ mod tests {
calls: AtomicUsize,
}
+ /// Local model-shaped test redactor with a real batch preference.
+ struct BatchingAi {
+ batches: AtomicUsize,
+ inputs: AtomicUsize,
+ }
+
impl UppercaseAi {
fn new() -> Self {
Self {
@@ -475,6 +621,15 @@ mod tests {
}
}
+ impl BatchingAi {
+ fn new() -> Self {
+ Self {
+ batches: AtomicUsize::new(0),
+ inputs: AtomicUsize::new(0),
+ }
+ }
+ }
+
/// Cloud-shaped test redactor that fails a configured number of calls,
/// then returns a visible successful result.
struct FlakyAi {
@@ -550,6 +705,37 @@ mod tests {
}
}
+ #[async_trait]
+ impl Redactor for BatchingAi {
+ fn name(&self) -> &str {
+ "test_onnx"
+ }
+
+ fn version(&self) -> u32 {
+ 1
+ }
+
+ fn preferred_batch_size(&self) -> usize {
+ 8
+ }
+
+ async fn redact_batch(
+ &self,
+ texts: &[String],
+ ) -> Result<Vec<RedactionOutput>, RedactError> {
+ self.batches.fetch_add(1, Ordering::SeqCst);
+ self.inputs.fetch_add(texts.len(), Ordering::SeqCst);
+ Ok(texts
+ .iter()
+ .map(|text| RedactionOutput {
+ input: text.clone(),
+ redacted: text.to_uppercase(),
+ spans: vec![],
+ })
+ .collect())
+ }
+ }
+
#[tokio::test]
async fn regex_only_runs_without_ai_secret_policy() {
// Default policy is `allow=[Secret]`, so a bare email is NOT
@@ -579,6 +765,37 @@ mod tests {
assert_eq!(ai.calls.load(Ordering::SeqCst), 1);
}
+ #[tokio::test]
+ async fn local_ai_batches_unique_cache_misses() {
+ let ai = Arc::new(BatchingAi::new());
+ let pipeline = Pipeline::regex_then_ai(ai.clone(), PipelineConfig::default());
+ let first = "first accessibility string long enough for inference".to_string();
+ let second = "second accessibility string long enough for inference".to_string();
+ let texts = vec![
+ first.clone(),
+ second.clone(),
+ first.clone(),
+ "short".to_string(),
+ ];
+
+ let outputs = pipeline.redact_batch(&texts).await.unwrap();
+
+ assert_eq!(outputs[0].redacted, first.to_uppercase());
+ assert_eq!(outputs[1].redacted, second.to_uppercase());
+ assert_eq!(outputs[2], outputs[0]);
+ assert_eq!(outputs[3].redacted, "short");
+ assert_eq!(ai.batches.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ ai.inputs.load(Ordering::SeqCst),
+ 2,
+ "duplicate cache misses must share one inference"
+ );
+
+ let cached = pipeline.redact_batch(&texts).await.unwrap();
+ assert_eq!(cached, outputs);
+ assert_eq!(ai.batches.load(Ordering::SeqCst), 1);
+ }
+
#[tokio::test]
async fn span_less_ai_output_is_trusted() {
// A span-less adapter (like the Tinfoil enclave, which returns