YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(semantic): stop re-storing unchanged screens on the capture write lock (#6085)
Louis Beaumont740b2a35ebb6
diff --git a/COVERAGE.md b/COVERAGE.mdindex 2df6e7625..e90d77a15 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 32- Mapped Rust files: 318-- Active test blocks: 2993+- Active test blocks: 3001- Ignored/manual test blocks: 137-- Weighted coverage points: 2461.2+- Weighted coverage points: 2467.7| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 29 | 2862 | 132 | 2401.2 | 21 | 11 | 100% |-| macos | 29 | 2916 | 112 | 2412.3 | 22 | 11 | 100% |-| linux | 25 | 2549 | 105 | 2118.7 | 20 | 11 | 100% |+| windows | 29 | 2870 | 132 | 2407.7 | 21 | 11 | 100% |+| macos | 29 | 2924 | 112 | 2418.8 | 22 | 11 | 100% |+| linux | 25 | 2557 | 105 | 2125.2 | 20 | 11 | 100% |## Refreshdiff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rsindex 5880beebe..4689f8726 100644--- a/crates/screenpipe-db/src/db/mod.rs+++ b/crates/screenpipe-db/src/db/mod.rs@@ -457,8 +457,9 @@ mod text_positions;mod write_ops;pub use self::semantic::{- SemanticActor, SemanticActorAlias, SemanticActorReference, SemanticCleanupResult,- SemanticContextQuery, SemanticFrameContext, SemanticProjectionWriteResult,+ SemanticActor, SemanticActorAlias, SemanticActorReference, SemanticAttachResult,+ SemanticCleanupResult, SemanticContextQuery, SemanticFrameContext,+ SemanticProjectionWriteResult,};pub(crate) use self::text_positions::calculate_confidence;pub use self::text_positions::{diff --git a/crates/screenpipe-db/src/db/semantic.rs b/crates/screenpipe-db/src/db/semantic.rsindex 3adc8523c..9fecfb7e5 100644--- a/crates/screenpipe-db/src/db/semantic.rs+++ b/crates/screenpipe-db/src/db/semantic.rs@@ -16,6 +16,9 @@ use sqlx::{FromRow, QueryBuilder, Sqlite};use std::collections::{BTreeMap, HashMap};use std::time::Duration;+/// Keeps one attach statement well under SQLite's bound-parameter limit.+const SEMANTIC_ATTACH_FRAME_CHUNK: usize = 256;+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub struct SemanticProjectionWriteResult {pub run_id: i64,@@ -24,6 +27,16 @@ pub struct SemanticProjectionWriteResult {pub items_reused: u32,}+/// Outcome of reattaching already-stored parse runs to newly captured frames.+///+/// `missing_run_ids` reports runs that cleanup removed between the caller+/// observing them and this write, so the caller can drop its cached copy.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct SemanticAttachResult {+ pub frames_attached: u64,+ pub missing_run_ids: Vec<i64>,+}+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]pub struct SemanticCleanupResult {pub runs_deleted: u64,@@ -349,6 +362,89 @@ impl DatabaseManager {}}+ /// Attach already-stored parse runs to newly captured frames in one+ /// transaction.+ ///+ /// An unchanged screen re-parses to the exact same run fingerprint, so the+ /// full store path would take the process-wide write lock and replay the+ /// same statements only to point a new frame at an existing run. Callers+ /// that already know the run id batch those attachments here instead, which+ /// turns one serialized transaction per captured frame into one per batch.+ /// Runs deleted by cleanup in the meantime are skipped and reported rather+ /// than leaving a dangling reference.+ pub async fn attach_semantic_runs_to_frames(+ &self,+ attachments: &[(i64, i64)],+ ) -> Result<SemanticAttachResult, sqlx::Error> {+ if attachments.is_empty() {+ return Ok(SemanticAttachResult::default());+ }++ let mut frames_by_run: BTreeMap<i64, Vec<i64>> = BTreeMap::new();+ for (frame_id, run_id) in attachments {+ frames_by_run.entry(*run_id).or_default().push(*frame_id);+ }++ let mut tx = self.begin_immediate_with_retry().await?;+ let result: Result<SemanticAttachResult, sqlx::Error> = async {+ let run_ids: Vec<i64> = frames_by_run.keys().copied().collect();+ let mut existing =+ QueryBuilder::<Sqlite>::new("SELECT id FROM semantic_runs WHERE id IN (");+ let mut separated = existing.separated(", ");+ for run_id in &run_ids {+ separated.push_bind(*run_id);+ }+ existing.push(")");+ let live_runs: Vec<i64> = existing+ .build_query_scalar()+ .fetch_all(&mut **tx.conn())+ .await?;++ let mut frames_attached = 0_u64;+ for (run_id, frame_ids) in &frames_by_run {+ if !live_runs.contains(run_id) {+ continue;+ }+ for chunk in frame_ids.chunks(SEMANTIC_ATTACH_FRAME_CHUNK) {+ let mut update =+ QueryBuilder::<Sqlite>::new("UPDATE frames SET semantic_run_id = ");+ update.push_bind(*run_id);+ update.push(" WHERE id IN (");+ let mut separated = update.separated(", ");+ for frame_id in chunk {+ separated.push_bind(*frame_id);+ }+ update.push(")");+ frames_attached += update+ .build()+ .execute(&mut **tx.conn())+ .await?+ .rows_affected();+ }+ }++ Ok(SemanticAttachResult {+ frames_attached,+ missing_run_ids: run_ids+ .into_iter()+ .filter(|run_id| !live_runs.contains(run_id))+ .collect(),+ })+ }+ .await;++ match result {+ Ok(result) => {+ tx.commit().await?;+ Ok(result)+ }+ Err(error) => {+ tx.rollback().await?;+ Err(error)+ }+ }+ }+pub async fn get_frame_semantic_context(&self,frame_id: i64,diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rsindex c2a62721b..11372a95b 100644--- a/crates/screenpipe-db/src/lib.rs+++ b/crates/screenpipe-db/src/lib.rs@@ -20,9 +20,9 @@ pub use cancellable_query::{pub use db::{find_matching_a11y_positions, parse_all_text_positions, DatabaseManager, DeleteTimeRangeResult,ImmediateTx, NewMeetingTranscriptSegment, SemanticActor, SemanticActorAlias,- SemanticActorReference, SemanticCleanupResult, SemanticContextQuery, SemanticFrameContext,- SemanticProjectionWriteResult, MEETING_END_REASON_AUTO_END, MEETING_END_REASON_EXPLICIT_STOP,- MEETING_END_REASON_SHUTDOWN,+ SemanticActorReference, SemanticAttachResult, SemanticCleanupResult, SemanticContextQuery,+ SemanticFrameContext, SemanticProjectionWriteResult, MEETING_END_REASON_AUTO_END,+ MEETING_END_REASON_EXPLICIT_STOP, MEETING_END_REASON_SHUTDOWN,};pub use recovery::{rebuild_recovered_fts5_indexes, verify_fresh_sqlite_recovery_candidate, RecoveryVerification,diff --git a/crates/screenpipe-db/tests/semantic_storage_test.rs b/crates/screenpipe-db/tests/semantic_storage_test.rsindex 10ad1f59b..2de30a86d 100644--- a/crates/screenpipe-db/tests/semantic_storage_test.rs+++ b/crates/screenpipe-db/tests/semantic_storage_test.rs@@ -666,3 +666,78 @@ async fn normalized_storage_growth_tracks_unique_semantics_not_raw_frame_count()assert!(repeated_per_frame < 4_096.0);assert!(changing_per_frame < 16_384.0);}++#[tokio::test]+async fn batched_attach_points_frames_at_an_existing_run() {+ let db = database().await;+ let now = Utc::now();+ let stored_frame = insert_frame(&db, now - ChronoDuration::minutes(2)).await;+ let repeated_frames = vec![+ insert_frame(&db, now - ChronoDuration::minutes(1)).await,+ insert_frame(&db, now).await,+ ];++ let write = db+ .store_semantic_projection(+ stored_frame,+ &manifest(),+ &app(),+ 7,+ Duration::from_micros(800),+ &projection("notarization is blocking the release"),+ )+ .await+ .expect("store first projection");++ let attachments: Vec<(i64, i64)> = repeated_frames+ .iter()+ .map(|frame_id| (*frame_id, write.run_id))+ .collect();+ let result = db+ .attach_semantic_runs_to_frames(&attachments)+ .await+ .expect("attach cached run");++ assert_eq!(result.frames_attached, 2);+ assert!(result.missing_run_ids.is_empty());+ for frame_id in repeated_frames {+ let context = db+ .get_frame_semantic_context(frame_id)+ .await+ .expect("read frame context")+ .expect("frame resolves to the reused run");+ assert_eq!(context.run_id, write.run_id);+ }+}++#[tokio::test]+async fn batched_attach_skips_runs_deleted_since_they_were_observed() {+ let db = database().await;+ let now = Utc::now();+ let frame = insert_frame(&db, now).await;+ let missing_run_id = 987_654;++ let result = db+ .attach_semantic_runs_to_frames(&[(frame, missing_run_id)])+ .await+ .expect("attach tolerates a deleted run");++ assert_eq!(result.frames_attached, 0);+ assert_eq!(result.missing_run_ids, vec![missing_run_id]);+ assert!(db+ .get_frame_semantic_context(frame)+ .await+ .expect("read frame context")+ .is_none());+}++#[tokio::test]+async fn batched_attach_without_work_takes_no_write_transaction() {+ let db = database().await;+ let result = db+ .attach_semantic_runs_to_frames(&[])+ .await+ .expect("empty attach succeeds");+ assert_eq!(result.frames_attached, 0);+ assert!(result.missing_run_ids.is_empty());+}diff --git a/crates/screenpipe-engine/src/semantic_worker.rs b/crates/screenpipe-engine/src/semantic_worker.rsindex 253cce190..92bc5fdfb 100644--- a/crates/screenpipe-engine/src/semantic_worker.rs+++ b/crates/screenpipe-engine/src/semantic_worker.rs@@ -20,14 +20,27 @@ use screenpipe_semantic::{TreeBudget, ValidatedParseOutcome,};use serde_json::{json, Value};+use std::collections::{HashMap, VecDeque};use std::sync::{Arc, OnceLock};-use std::time::Instant;+use std::time::{Duration, Instant};use tokio::runtime::Handle;use tokio::sync::watch;+use tokio::time::Instant as DeadlineInstant;use tracing::{debug, info, warn};const SEMANTIC_TELEMETRY_SAMPLE_DENOMINATOR: i64 = 100;+/// Remembered parse runs for screens this process already stored. Bounded so a+/// long session cannot grow the worker's memory with stale app states.+const SEMANTIC_RUN_CACHE_CAPACITY: usize = 256;++/// Frame attachments buffered before taking the process-wide write lock.+const SEMANTIC_ATTACH_BATCH: usize = 32;++/// Longest a buffered attachment waits before it is flushed anyway. Retrieval+/// reads `frames.semantic_run_id`, so a short delay is invisible to callers.+const SEMANTIC_ATTACH_MAX_DELAY: Duration = Duration::from_secs(2);+#[derive(Clone)]pub(crate) struct SemanticProjectionSender {tx: watch::Sender<Option<Arc<SemanticProjectionJob>>>,@@ -97,6 +110,126 @@ const fn semantic_mode_label(mode: SemanticContextMode) -> &'static str {}}+/// Identifies one stored parse run without holding any captured content.+///+/// These are exactly the fields the persisted run fingerprint is derived from,+/// so an equal key provably resolves to the same `semantic_runs` row.+#[derive(Clone, PartialEq, Eq, Hash)]+struct SemanticRunCacheKey {+ platform: u8,+ app_id: Option<String>,+ executable: Option<String>,+ app_version: Option<String>,+ browser_url: Option<String>,+ content_hash: u64,+}++impl SemanticRunCacheKey {+ fn new(app: &AppIdentity, content_hash: u64) -> Self {+ Self {+ platform: app.platform as u8,+ app_id: app.app_id.clone(),+ executable: app.executable.clone(),+ app_version: app.version.clone(),+ browser_url: app.browser_url.clone(),+ content_hash,+ }+ }+}++/// Bounded insertion-ordered cache of screens this process already stored.+#[derive(Default)]+struct SemanticRunCache {+ runs: HashMap<SemanticRunCacheKey, i64>,+ order: VecDeque<SemanticRunCacheKey>,+}++impl SemanticRunCache {+ fn get(&self, key: &SemanticRunCacheKey) -> Option<i64> {+ self.runs.get(key).copied()+ }++ fn insert(&mut self, key: SemanticRunCacheKey, run_id: i64) {+ if self.runs.insert(key.clone(), run_id).is_none() {+ self.order.push_back(key);+ }+ while self.order.len() > SEMANTIC_RUN_CACHE_CAPACITY {+ if let Some(evicted) = self.order.pop_front() {+ self.runs.remove(&evicted);+ }+ }+ }++ /// Drops every screen pointing at a run that no longer exists, so a run+ /// removed by cleanup can never be reattached from memory.+ fn forget_runs(&mut self, run_ids: &[i64]) {+ if run_ids.is_empty() {+ return;+ }+ self.runs.retain(|_, run_id| !run_ids.contains(run_id));+ self.order.retain(|key| self.runs.contains_key(key));+ }+}++/// Worker-local state: what this process has already stored, and which frame+/// attachments are waiting for the next batched write.+#[derive(Default)]+struct SemanticProjectionState {+ cache: SemanticRunCache,+ pending_attachments: Vec<(i64, i64)>,+ flush_deadline: Option<DeadlineInstant>,+}++impl SemanticProjectionState {+ fn queue_attachment(&mut self, frame_id: i64, run_id: i64) {+ self.pending_attachments.push((frame_id, run_id));+ self.flush_deadline+ .get_or_insert_with(|| DeadlineInstant::now() + SEMANTIC_ATTACH_MAX_DELAY);+ }++ fn should_flush(&self) -> bool {+ self.pending_attachments.len() >= SEMANTIC_ATTACH_BATCH+ }+}++/// Writes buffered attachments in one transaction. Failures are logged and the+/// buffer is cleared: the affected frames keep generic capture, and the next+/// capture of the same screen re-stores the projection.+async fn flush_pending_attachments(db: &DatabaseManager, state: &mut SemanticProjectionState) {+ state.flush_deadline = None;+ if state.pending_attachments.is_empty() {+ return;+ }+ let attachments = std::mem::take(&mut state.pending_attachments);+ match db.attach_semantic_runs_to_frames(&attachments).await {+ Ok(result) => {+ state.cache.forget_runs(&result.missing_run_ids);+ debug!(+ attachments = attachments.len(),+ frames_attached = result.frames_attached,+ missing_runs = result.missing_run_ids.len(),+ "semantic run attachments flushed"+ );+ }+ Err(error) => {+ // A failed batch must not strand cached runs that may be gone.+ state.cache = SemanticRunCache::default();+ warn!(+ attachments = attachments.len(),+ %error,+ "semantic run attachment flush failed; generic capture remains available"+ );+ }+ }+}++async fn wait_for_flush_deadline(deadline: Option<DeadlineInstant>) {+ match deadline {+ Some(deadline) => tokio::time::sleep_until(deadline).await,+ None => std::future::pending::<()>().await,+ }+}+fn semantic_projection_channel() -> (SemanticProjectionSender,watch::Receiver<Option<Arc<SemanticProjectionJob>>>,@@ -119,7 +252,22 @@ async fn run_semantic_projection_worker(}};- while rx.changed().await.is_ok() {+ let mut state = SemanticProjectionState::default();+ loop {+ tokio::select! {+ // Buffered attachments must not wait for the next capture, which+ // may never come if the user stops working.+ _ = wait_for_flush_deadline(state.flush_deadline) => {+ flush_pending_attachments(&db, &mut state).await;+ continue;+ }+ changed = rx.changed() => {+ if changed.is_err() {+ break;+ }+ }+ }+let Some(job) = rx.borrow_and_update().clone() else {continue;};@@ -137,14 +285,20 @@ async fn run_semantic_projection_worker(false}});- if let Err(error) = process_semantic_job(&db, ®istry, &job, mode_label).await {+ if let Err(error) = process_semantic_job(&db, ®istry, &job, mode_label, &mut state).await+ {warn!(frame_id = job.frame_id,%error,"semantic projection failed; generic capture remains available");}+ if state.should_flush() {+ flush_pending_attachments(&db, &mut state).await;+ }}++ flush_pending_attachments(&db, &mut state).await;}async fn process_semantic_job(@@ -152,6 +306,7 @@ async fn process_semantic_job(registry: &ParserRegistry,job: &SemanticProjectionJob,mode_label: &'static str,+ state: &mut SemanticProjectionState,) -> anyhow::Result<()> {let app = AppIdentity {platform: current_platform(),@@ -199,6 +354,23 @@ async fn process_semantic_job(}let input_content_hash = adapted.tree.structural_fingerprint();+ // An unchanged screen resolves to the run this process already stored.+ // Parsing it again is deterministic and the write would only reattach the+ // same run, so buffer the attachment and skip both.+ let cache_key = SemanticRunCacheKey::new(&app, input_content_hash);+ if let Some(run_id) = state.cache.get(&cache_key) {+ state.queue_attachment(job.frame_id, run_id);+ emit_sampled_semantic_telemetry(+ job.frame_id,+ &candidate_parser_ids,+ None,+ "handled_cached",+ 0,+ mode_label,+ );+ return Ok(());+ }+let context = ParseContext {frame_id: job.frame_id,captured_at_unix_ms: job.captured_at.timestamp_millis(),@@ -241,6 +413,7 @@ async fn process_semantic_job(adapted.stats.suppressed_offscreen_content_nodes;let suppressed_offscreen_content_bytes =adapted.stats.suppressed_offscreen_content_bytes;+ let store_started = Instant::now();let write = db.store_semantic_projection(job.frame_id,@@ -251,13 +424,19 @@ async fn process_semantic_job(&projection,).await?;- emit_sampled_semantic_telemetry(+ let store_duration_us = store_started.elapsed().as_micros() as u64;+ state.cache.insert(cache_key, write.run_id);+ emit_sampled_semantic_telemetry_with_write(job.frame_id,&candidate_parser_ids,Some(parser_id),"handled",parse_duration_us,mode_label,+ Some(SemanticWriteSample {+ store_duration_us,+ reused_run: write.reused_run,+ }),);debug!(frame_id = job.frame_id,@@ -303,6 +482,34 @@ fn emit_sampled_semantic_telemetry(outcome: &'static str,parse_duration_us: u64,mode_label: &'static str,+) {+ emit_sampled_semantic_telemetry_with_write(+ frame_id,+ candidate_parser_ids,+ selected_parser_id,+ outcome,+ parse_duration_us,+ mode_label,+ None,+ );+}++/// Content-free cost of one persisted projection. Without this the write path+/// is invisible in production and a storage regression cannot be measured.+#[derive(Debug, Clone, Copy)]+struct SemanticWriteSample {+ store_duration_us: u64,+ reused_run: bool,+}++fn emit_sampled_semantic_telemetry_with_write(+ frame_id: i64,+ candidate_parser_ids: &[String],+ selected_parser_id: Option<&str>,+ outcome: &'static str,+ parse_duration_us: u64,+ mode_label: &'static str,+ write: Option<SemanticWriteSample>,) {if !should_sample_semantic_telemetry(frame_id) {return;@@ -316,6 +523,7 @@ fn emit_sampled_semantic_telemetry(outcome,parse_duration_us,mode_label,+ write,),);}@@ -330,6 +538,7 @@ fn semantic_telemetry_properties(outcome: &'static str,parse_duration_us: u64,mode_label: &'static str,+ write: Option<SemanticWriteSample>,) -> Value {json!({"semantic_candidate_parser_ids": candidate_parser_ids,@@ -338,6 +547,8 @@ fn semantic_telemetry_properties("semantic_parse_duration_us": parse_duration_us,"semantic_platform": platform_label(current_platform()),"semantic_mode": mode_label,+ "semantic_store_duration_us": write.map(|write| write.store_duration_us),+ "semantic_reused_run": write.map(|write| write.reused_run),})}@@ -528,6 +739,86 @@ mod tests {)}+ fn cache_key(app_id: &str, content_hash: u64) -> SemanticRunCacheKey {+ SemanticRunCacheKey::new(+ &AppIdentity {+ platform: current_platform(),+ app_id: Some(app_id.into()),+ executable: None,+ display_name: "Slack".into(),+ version: Some("4.39.95".into()),+ browser_url: None,+ },+ content_hash,+ )+ }++ #[test]+ fn unchanged_screen_resolves_to_the_stored_run() {+ let mut cache = SemanticRunCache::default();+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 11), 7);++ assert_eq!(+ cache.get(&cache_key("com.tinyspeck.slackmacgap", 11)),+ Some(7)+ );+ // A changed screen and a different app must both miss, or a frame+ // would be attached to a run that never described it.+ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 12)), None);+ assert_eq!(cache.get(&cache_key("com.other.app", 11)), None);+ }++ #[test]+ fn cache_stays_bounded_for_a_long_session() {+ let mut cache = SemanticRunCache::default();+ for index in 0..(SEMANTIC_RUN_CACHE_CAPACITY as u64 + 10) {+ cache.insert(cache_key("com.tinyspeck.slackmacgap", index), index as i64);+ }++ assert_eq!(cache.runs.len(), SEMANTIC_RUN_CACHE_CAPACITY);+ assert_eq!(cache.order.len(), SEMANTIC_RUN_CACHE_CAPACITY);+ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 0)), None);+ let newest = SEMANTIC_RUN_CACHE_CAPACITY as u64 + 9;+ assert_eq!(+ cache.get(&cache_key("com.tinyspeck.slackmacgap", newest)),+ Some(newest as i64)+ );+ }++ #[test]+ fn deleted_runs_are_never_reattached_from_memory() {+ let mut cache = SemanticRunCache::default();+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 1), 41);+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 2), 42);++ cache.forget_runs(&[41]);++ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 1)), None);+ assert_eq!(+ cache.get(&cache_key("com.tinyspeck.slackmacgap", 2)),+ Some(42)+ );+ assert_eq!(cache.order.len(), 1);+ }++ #[test]+ fn attachments_batch_before_taking_the_write_lock() {+ let mut state = SemanticProjectionState::default();+ assert!(state.flush_deadline.is_none());++ for frame_id in 0..(SEMANTIC_ATTACH_BATCH as i64 - 1) {+ state.queue_attachment(frame_id, 7);+ assert!(!state.should_flush());+ }+ // A partial batch still has a deadline so it cannot wait forever for a+ // capture that may never arrive.+ assert!(state.flush_deadline.is_some());++ state.queue_attachment(SEMANTIC_ATTACH_BATCH as i64, 7);+ assert!(state.should_flush());+ assert_eq!(state.pending_attachments.len(), SEMANTIC_ATTACH_BATCH);+ }+#[tokio::test]async fn pending_slot_keeps_only_latest_frame() {let (sender, mut receiver) = semantic_projection_channel();@@ -589,6 +880,10 @@ mod tests {"handled",73,"memory",+ Some(SemanticWriteSample {+ store_duration_us: 1_200,+ reused_run: true,+ }),);assert_eq!(properties["semantic_outcome"], "handled");@@ -598,7 +893,9 @@ mod tests {platform_label(current_platform()));assert_eq!(properties["semantic_mode"], "memory");- assert_eq!(properties.as_object().map(|object| object.len()), Some(6));+ assert_eq!(properties["semantic_store_duration_us"], 1_200);+ assert_eq!(properties["semantic_reused_run"], true);+ assert_eq!(properties.as_object().map(|object| object.len()), Some(8));let serialized = properties.to_string();assert!(!serialized.contains("app_identifier"));assert!(!serialized.contains("display_name"));@@ -609,6 +906,80 @@ mod tests {assert!(!serialized.contains("text_content"));}+ #[tokio::test]+ async fn unchanged_screen_stores_once_and_batches_later_frames() {+ let db = DatabaseManager::new("sqlite::memory:", Default::default())+ .await+ .expect("in-memory database");+ let registry = builtin_parser_registry().expect("built-in registry");+ let mut state = SemanticProjectionState::default();++ let mut frame_ids = Vec::new();+ for index in 0..3 {+ let captured_at = Utc::now();+ let frame_id = db+ .insert_snapshot_frame(+ "test-device",+ captured_at,+ &format!("/tmp/semantic-worker-{index}.jpg"),+ Some("Slack"),+ Some("#release - Slack"),+ None,+ true,+ Some("test"),+ Some("release"),+ Some("accessibility"),+ None,+ None,+ None,+ )+ .await+ .expect("insert frame");+ frame_ids.push(frame_id);++ let job = SemanticProjectionJob::from_capture(+ frame_id,+ captured_at,+ Some("Slack".into()),+ None,+ // The identical screen, captured repeatedly.+ slack_snapshot("notarization is blocking the release"),+ false,+ );+ process_semantic_job(&db, ®istry, &job, "memory", &mut state)+ .await+ .expect("process semantic projection");+ }++ // Only the first capture paid for a store transaction. The other two+ // resolved from memory and are waiting for one batched write.+ assert_eq!(state.cache.runs.len(), 1);+ assert_eq!(state.pending_attachments.len(), 2);+ assert_eq!(+ db.get_frame_semantic_context(frame_ids[1])+ .await+ .expect("read frame context")+ .map(|context| context.run_id),+ None+ );++ flush_pending_attachments(&db, &mut state).await;+ assert!(state.pending_attachments.is_empty());+ assert!(state.flush_deadline.is_none());++ let mut run_ids = Vec::new();+ for frame_id in frame_ids {+ let context = db+ .get_frame_semantic_context(frame_id)+ .await+ .expect("read frame context")+ .expect("every repeated frame resolves to the stored run");+ run_ids.push(context.run_id);+ }+ assert_eq!(run_ids[0], run_ids[1]);+ assert_eq!(run_ids[1], run_ids[2]);+ }+#[tokio::test]async fn supported_capture_is_parsed_redacted_and_persisted() {let db = DatabaseManager::new("sqlite::memory:", Default::default())@@ -643,7 +1014,8 @@ mod tests {true,);- process_semantic_job(&db, ®istry, &projection_job, "memory")+ let mut state = SemanticProjectionState::default();+ process_semantic_job(&db, ®istry, &projection_job, "memory", &mut state).await.expect("process semantic projection");diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex f33817205..586f0f863 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 32- Mapped Rust files: 318-- Active test blocks: 2993+- Active test blocks: 3001- Ignored/manual test blocks: 137-- Declared test blocks: 3130-- Weighted coverage points: 2461.2+- Declared test blocks: 3138+- Weighted coverage points: 2467.7Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 29 | 2862 | 132 | 2401.2 | 21 | 11 | 100% |-| macos | 29 | 2916 | 112 | 2412.3 | 22 | 11 | 100% |-| linux | 25 | 2549 | 105 | 2118.7 | 20 | 11 | 100% |+| windows | 29 | 2870 | 132 | 2407.7 | 21 | 11 | 100% |+| macos | 29 | 2924 | 112 | 2418.8 | 22 | 11 | 100% |+| linux | 25 | 2557 | 105 | 2125.2 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 10 | 18 | 102 | 1432 | 42 | 1098.3 | 10 |-| screenpipe-db | 5 | 51 | 14 | 420 | 16 | 399.7 | 9 |+| screenpipe-engine | 10 | 18 | 102 | 1437 | 42 | 1101.8 | 10 |+| screenpipe-db | 5 | 51 | 14 | 423 | 16 | 402.7 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 2 | 11 | 0 | 11.0 | 2 || screenpipe-audio | 6 | 25 | 49 | 563 | 43 | 491.2 | 5 || screenpipe-screen | 6 | 9 | 18 | 234 | 9 | 213.5 | 4 |@@ -65,24 +65,24 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| audio | 7 suites / 659 active / 44 ignored / 587.2 pts | 7 suites / 659 active / 44 ignored / 587.2 pts | 6 suites / 584 active / 43 ignored / 534.7 pts || audio-device | 2 suites / 206 active / 7 ignored / 183.5 pts | 2 suites / 206 active / 7 ignored / 183.5 pts | 1 suites / 131 active / 6 ignored / 131.0 pts || configuration | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts |-| database | 6 suites / 338 active / 12 ignored / 317.7 pts | 6 suites / 338 active / 12 ignored / 317.7 pts | 6 suites / 338 active / 12 ignored / 317.7 pts |+| database | 6 suites / 341 active / 12 ignored / 320.7 pts | 6 suites / 341 active / 12 ignored / 320.7 pts | 6 suites / 341 active / 12 ignored / 320.7 pts || db-search | 2 suites / 107 active / 9 ignored / 107.0 pts | 2 suites / 107 active / 9 ignored / 107.0 pts | 2 suites / 107 active / 9 ignored / 107.0 pts || engine-lifecycle | 6 suites / 205 active / 1 ignored / 180.0 pts | 6 suites / 205 active / 1 ignored / 180.0 pts | 5 suites / 199 active / 1 ignored / 178.3 pts || local-api | 2 suites / 322 active / 9 ignored / 227.2 pts | 2 suites / 322 active / 9 ignored / 227.2 pts | 2 suites / 322 active / 9 ignored / 227.2 pts || meeting | 6 suites / 1373 active / 19 ignored / 1117.4 pts | 6 suites / 1373 active / 19 ignored / 1117.4 pts | 4 suites / 1095 active / 15 ignored / 861.9 pts || ocr | 4 suites / 120 active / 7 ignored / 114.3 pts | 4 suites / 124 active / 7 ignored / 119.8 pts | 3 suites / 115 active / 6 ignored / 110.8 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |-| performance | 13 suites / 1342 active / 67 ignored / 1186.2 pts | 14 suites / 1440 active / 70 ignored / 1225.4 pts | 13 suites / 1342 active / 67 ignored / 1186.2 pts |+| performance | 13 suites / 1350 active / 67 ignored / 1192.7 pts | 14 suites / 1448 active / 70 ignored / 1231.9 pts | 13 suites / 1350 active / 67 ignored / 1192.7 pts || pipes | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts || privacy | 5 suites / 869 active / 36 ignored / 706.1 pts | 5 suites / 919 active / 16 ignored / 711.7 pts | 5 suites / 845 active / 14 ignored / 684.3 pts || real-app | - | 1 suites / 98 active / 3 ignored / 39.2 pts | - || speaker | 2 suites / 318 active / 8 ignored / 318.0 pts | 2 suites / 318 active / 8 ignored / 318.0 pts | 2 suites / 318 active / 8 ignored / 318.0 pts |-| storage | 3 suites / 464 active / 29 ignored / 376.4 pts | 3 suites / 464 active / 29 ignored / 376.4 pts | 3 suites / 464 active / 29 ignored / 376.4 pts |+| storage | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts || sync | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts |-| timeline | 4 suites / 917 active / 33 ignored / 746.6 pts | 4 suites / 917 active / 33 ignored / 746.6 pts | 4 suites / 917 active / 33 ignored / 746.6 pts |+| timeline | 4 suites / 925 active / 33 ignored / 753.1 pts | 4 suites / 925 active / 33 ignored / 753.1 pts | 4 suites / 925 active / 33 ignored / 753.1 pts || transcription | 5 suites / 673 active / 40 ignored / 528.9 pts | 5 suites / 673 active / 40 ignored / 528.9 pts | 5 suites / 673 active / 40 ignored / 528.9 pts || ui-events | 4 suites / 696 active / 28 ignored / 531.1 pts | 3 suites / 648 active / 5 ignored / 497.5 pts | 3 suites / 648 active / 5 ignored / 497.5 pts |-| vision-capture | 5 suites / 477 active / 32 ignored / 380.9 pts | 5 suites / 481 active / 32 ignored / 386.4 pts | 4 suites / 472 active / 31 ignored / 377.4 pts |+| vision-capture | 5 suites / 482 active / 32 ignored / 384.4 pts | 5 suites / 486 active / 32 ignored / 389.9 pts | 4 suites / 477 active / 31 ignored / 380.9 pts |## Critical Flow Matrix@@ -91,14 +91,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Settings to engine recording config | configuration | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) || Engine health, sleep, and lifecycle | engine-lifecycle | covered (strong; engine-config-lifecycle, engine-retention-storage) | covered (strong; engine-config-lifecycle, engine-retention-storage) | covered (strong; engine-config-lifecycle, engine-retention-storage) || Capture, OCR, and frame persistence | vision-capture, ocr | covered (partial; screen-capture-ocr-contract, screen-windows-ocr) | covered (strong; screen-capture-ocr-contract, screen-macos-ocr) | covered (partial; screen-capture-ocr-contract) |-| Timeline frame and stream delivery | timeline | covered (strong; engine-api-routes, screen-capture-windowing) | covered (strong; engine-api-routes, screen-capture-windowing) | covered (strong; engine-api-routes, screen-capture-windowing) |+| Timeline frame and stream delivery | timeline | covered (strong; engine-api-routes, engine-capture-timeline) | covered (strong; engine-api-routes, engine-capture-timeline) | covered (strong; engine-api-routes, engine-capture-timeline) || Local API search and indexing | local-api, db-search | covered (strong; engine-local-api-search-integration) | covered (strong; engine-local-api-search-integration) | covered (strong; engine-local-api-search-integration) || Audio record, transcribe, and reconcile | audio, transcription | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) || Audio device and stream health | audio-device | covered (strong; audio-device-stream-health, audio-platform-output-capture) | covered (strong; audio-device-stream-health, audio-platform-output-capture) | covered (strong; audio-device-stream-health) || Meeting detection and live transcript merge | meeting | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) || Privacy filters, DRM guards, and redaction | privacy | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) || Accessibility tree and UI event capture | accessibility, ui-events | covered (strong; a11y-core-tree-cross-platform, a11y-windows-tree) | covered (strong; a11y-core-tree-cross-platform, db-accessibility-ui-events) | covered (strong; a11y-core-tree-cross-platform, db-accessibility-ui-events) |-| Performance, backpressure, and liveness | performance | covered (strong; screen-capture-windowing, engine-capture-timeline) | covered (strong; screen-capture-windowing, engine-capture-timeline) | covered (strong; screen-capture-windowing, engine-capture-timeline) |+| Performance, backpressure, and liveness | performance | covered (strong; engine-capture-timeline, screen-capture-windowing) | covered (strong; engine-capture-timeline, screen-capture-windowing) | covered (strong; engine-capture-timeline, screen-capture-windowing) |## Critical Gaps@@ -132,9 +132,9 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 96 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 12 | 27 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 101 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 172 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 175 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 31 | 316 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 252 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 257 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. || engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. || engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 8 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. || engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |@@ -307,7 +307,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-timeline-frames | screenpipe-db | tests/range_export_frame_sources_test.rs | integration | 3 | 0 | 3 || db-search-indexing | screenpipe-db | tests/search_issue_4474_bench.rs | integration | 0 | 1 | 1 || db-search-indexing | screenpipe-db | tests/search_ocr_snapshot_test.rs | integration | 4 | 0 | 4 |-| db-timeline-frames | screenpipe-db | tests/semantic_storage_test.rs | integration | 6 | 0 | 6 |+| db-timeline-frames | screenpipe-db | tests/semantic_storage_test.rs | integration | 9 | 0 | 9 || db-audio-meetings-speakers | screenpipe-db | tests/single_open_meeting_invariant_test.rs | integration | 3 | 0 | 3 || db-audio-meetings-speakers | screenpipe-db | tests/speaker_benchmark.rs | integration | 0 | 1 | 1 || db-audio-meetings-speakers | screenpipe-db | tests/speaker_delete_test.rs | integration | 3 | 0 | 3 |@@ -412,7 +412,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-api-routes | screenpipe-engine | src/routes/timezone.rs | source | 8 | 0 | 8 || engine-api-routes | screenpipe-engine | src/routes/websocket.rs | source | 4 | 0 | 4 || engine-config-lifecycle | screenpipe-engine | src/schedule_monitor.rs | source | 8 | 0 | 8 |-| engine-capture-timeline | screenpipe-engine | src/semantic_worker.rs | source | 6 | 0 | 6 |+| engine-capture-timeline | screenpipe-engine | src/semantic_worker.rs | source | 11 | 0 | 11 || engine-api-routes | screenpipe-engine | src/server.rs | source | 13 | 0 | 13 || engine-config-lifecycle | screenpipe-engine | src/sleep_monitor.rs | source | 11 | 1 | 12 || engine-capture-timeline | screenpipe-engine | src/snapshot_compaction.rs | source | 18 | 0 | 18 |