YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(redact): rfdetr_v17 — real-data recall + zero false fires across all eval suites (#5217)
Louis Beaumontc15d89fe1b9c
diff --git a/crates/screenpipe-redact/src/adapters/rfdetr.rs b/crates/screenpipe-redact/src/adapters/rfdetr.rsindex 151b81d34..955f6ba76 100644--- a/crates/screenpipe-redact/src/adapters/rfdetr.rs+++ b/crates/screenpipe-redact/src/adapters/rfdetr.rs@@ -20,27 +20,25 @@//!//! ## Reference benchmark numbers//!-//! `rfdetr_v12` (512×512 input, FP16 ONNX, ~54 MB):+//! `rfdetr_v19` (512×512 input, FP16 ONNX with fp32 I/O wrapper, ~60 MB,+//! ~118 ms/frame CPU). Held-out eval, production decode (sigmoid,+//! conf 0.50), [email protected]://!-//! p50 latency, macOS Apple Silicon (M1+) CoreML: ~120 ms — measured.-//! The 512² input is ~2.5× the pixels of v8's 320² (v9 was 384²), so-//! v11 runs ~1.5–2× slower than the v9 figures previously listed here;-//! still well within real-time for per-frame redaction. Other-platform-//! p50s (DirectML / CUDA / CPU) scale similarly — re-measure before-//! quoting.+//! v13 (prev shipped): recall 6.5 %, 10.2 boxes/frame+//! v18: recall 13.7 %, 2.9 boxes/frame+//! v19 (this file): recall 14.2 %, 2.8 boxes/frame; real-app planted+//! secrets 19/19 (v18: 0/19)//!-//! Bench accuracy on `screenpipe-pii-bench-image` val (221 images):-//! 98.9 % zero-leak / 0 % oversmash (v11, realworld-augmented). v11-//! replaces v9, which under-detected badly on real screens (≈0-//! detections on real frames despite passing the synthetic bench);-//! v11 restores real-frame detection AND raises the synthetic ceiling-//! from v8's 95.3 %.+//! False-fire hardening: ZERO detections of any class at conf 0.50+//! across two zero-PII eval suites (300 in-distribution pages + 250+//! held-out independent layouts; the previous shipped model produced+//! 592 + 23) and a known-truth screenshot run (0 decoy hits, 0+//! strays). Zero date false positives (the v13 complaint).//!-//! `v12` is the FP16 re-export of the v11 checkpoint (torch fp16 export,-//! fp32 I/O wrapper): ~54 MB vs ~109 MB and ~1.8× faster on CPU, with-//! zero-leak within ~0.6 pp of v11 fp32 on the corpus eval (essentially-//! lossless — fp16 inference of an fp32-trained detector). Re-export-//! recipe: `screenpipe-pii-bench-image/training/export_fp16.py`.+//! FP16 re-export recipe: torch half() at ONNX export from the FINAL+//! training weights (`last.ckpt` — NOT `checkpoint_best_ema.pth`,+//! whose best-val selection can freeze at epoch 0), then wrap I/O back+//! to fp32 Casts — this adapter feeds f32 and extracts f32.use std::path::{Path, PathBuf};@@ -52,7 +50,7 @@ use crate::RedactError;use crate::SpanLabel;const RFDETR_NAME: &str = "rfdetr";-const RFDETR_VERSION: u32 = 13; // matches the rfdetr_v13 ONNX (fp16, 384px)+const RFDETR_VERSION: u32 = 19; // matches the rfdetr_v19 ONNX (fp16, 512px, real-app trained)#[cfg(feature = "onnx-cpu")]const NUM_CLASSES: usize = 12;@@ -77,10 +75,20 @@ const CLASSES: [SpanLabel; NUM_CLASSES] = [SpanLabel::Secret, // 11];+/// Per-class score floors applied on top of `conf_threshold` (the higher+/// wins). All zero since v19: its genuine secret detections score+/// 0.58–0.69 while it produces ZERO observed secret false fires at+/// conf 0.50 across 719 zero-PII eval images — the old 0.92 floor+/// (calibrated for v15-era false fires) would have suppressed every+/// real catch. Mechanism kept for future per-class calibration.+#[cfg(feature = "onnx-cpu")]+const CLASS_MIN_SCORE: [f32; NUM_CLASSES] =+ [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0];+/// Configuration for [`RfdetrRedactor`].#[derive(Debug, Clone)]pub struct RfdetrConfig {- /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v13.onnx`+ /// Path to `rfdetr_vN.onnx`. We default to `~/.screenpipe/models/rfdetr_v19.onnx`/// in [`Self::default_model_path`] but callers may override (e.g./// for an INT8-quantized variant in the future).pub model_path: PathBuf,@@ -111,14 +119,14 @@ impl Default for RfdetrConfig {}impl RfdetrConfig {- /// `~/.screenpipe/models/rfdetr_v13.onnx`. Created lazily by+ /// `~/.screenpipe/models/rfdetr_v19.onnx`. Created lazily by/// [`Self::ensure_model_present`] on first run.pub fn default_model_path() -> PathBuf {dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")).join(".screenpipe").join("models")- .join("rfdetr_v13.onnx")+ .join("rfdetr_v19.onnx")}/// HuggingFace download URL for the canonical ONNX. Pinned to@@ -126,16 +134,16 @@ impl RfdetrConfig {/// (URL + expected SHA-256 + [`RFDETR_VERSION`] all bumped/// together).pub const HF_DOWNLOAD_URL: &'static str =- "https://huggingface.co/screenpipe/pii-image-redactor/resolve/main/rfdetr_v13.onnx";+ "https://huggingface.co/screenpipe/pii-image-redactor/resolve/main/rfdetr_v19.onnx";- /// Expected SHA-256 of the canonical `rfdetr_v13.onnx`. Verified+ /// Expected SHA-256 of the canonical `rfdetr_v19.onnx`. Verified/// after every download. If a future training run produces a new/// best, bump [`RFDETR_VERSION`], re-publish to HF, update this/// constant. Note: the worker is destructive-only and does NOT/// re-redact already-processed frames, so a model-version bump/// only takes effect for newly-captured frames going forward.pub const EXPECTED_SHA256: &'static str =- "16d10b46c078c1b05e0fb5cdbe2be3a08c3e0541a9f53233a58cff64fd515e05";+ "b9177c3ca7531da10366e57a6d74a5d2929da7479db6cc1f89419da194fa227f";/// Make sure the ONNX is present on disk. Idempotent — does/// nothing if [`Self::model_path`] already exists with the@@ -164,7 +172,7 @@ impl RfdetrConfig {tracing::info!(url = Self::HF_DOWNLOAD_URL,target = %self.model_path.display(),- "downloading rfdetr_v13.onnx (~60 MB) — first-run only"+ "downloading rfdetr_v19.onnx (~60 MB) — first-run only");let resp = reqwest::Client::new().get(Self::HF_DOWNLOAD_URL)@@ -201,7 +209,7 @@ impl RfdetrConfig {tracing::info!(target = %self.model_path.display(),bytes = bytes.len(),- "rfdetr_v13.onnx ready"+ "rfdetr_v19.onnx ready");Ok(())}@@ -270,13 +278,21 @@ mod imp {.with_compute_units(ort::ep::coreml::ComputeUnits::All).with_subgraphs(true).build(),- ort::ep::CPU::default().build(),+ ort::ep::CPU::default().with_arena_allocator(false).build(),])?;#[cfg(feature = "onnx-directml")]let mut builder = builder.with_execution_providers([ort::ep::DirectML::default().with_device_id(0).build(),- ort::ep::CPU::default().build(),+ ort::ep::CPU::default().with_arena_allocator(false).build(),])?;+ // No arena for the CPU EP: this is an occasional background+ // batch worker, and the arena never returns freed memory to+ // the OS. Measured on the fp16 512px model: inference RSS+ // 423 MB (arena) -> 193 MB (no arena) for +16 ms/frame.+ #[cfg(not(any(feature = "onnx-coreml", feature = "onnx-directml")))]+ let mut builder = builder.with_execution_providers([ort::ep::CPU::default()+ .with_arena_allocator(false)+ .build()])?;builder.commit_from_file(model_path)},)) {@@ -410,7 +426,8 @@ mod imp {best_class = c;}}- if best_score < self.cfg.conf_threshold {+ let floor = self.cfg.conf_threshold.max(CLASS_MIN_SCORE[best_class]);+ if best_score < floor {continue;}let bo = q * 4;@@ -539,7 +556,7 @@ mod tests {let p = RfdetrConfig::default_model_path();let expected_suffix = Path::new(".screenpipe").join("models")- .join("rfdetr_v13.onnx");+ .join("rfdetr_v19.onnx");assert!(p.ends_with(&expected_suffix),"default path {} should end with {}",@@ -605,7 +622,7 @@ mod tests {// (Real download path is exercised by integration tests off// the unit-test harness.)let d = tempdir().unwrap();- let p = d.path().join("models").join("rfdetr_v13.onnx");+ let p = d.path().join("models").join("rfdetr_v19.onnx");std::fs::create_dir_all(p.parent().unwrap()).unwrap();std::fs::write(&p, b"not the real model").unwrap();let cfg = RfdetrConfig {