Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

perf(semantic): stop re-storing unchanged screens on the capture write lock (#6085)

Louis Beaumont740b2a35ebb6
diff --git a/COVERAGE.md b/COVERAGE.md
index 2df6e7625..e90d77a15 100644
--- a/COVERAGE.md
+++ b/COVERAGE.md
@@ -32,15 +32,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 32
- Mapped Rust files: 318
-- Active test blocks: 2993
+- Active test blocks: 3001
- Ignored/manual test blocks: 137
-- Weighted coverage points: 2461.2
+- Weighted coverage points: 2467.7
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2862 | 132 | 2401.2 | 21 | 11 | 100% |
-| macos | 29 | 2916 | 112 | 2412.3 | 22 | 11 | 100% |
-| linux | 25 | 2549 | 105 | 2118.7 | 20 | 11 | 100% |
+| windows | 29 | 2870 | 132 | 2407.7 | 21 | 11 | 100% |
+| macos | 29 | 2924 | 112 | 2418.8 | 22 | 11 | 100% |
+| linux | 25 | 2557 | 105 | 2125.2 | 20 | 11 | 100% |
## Refresh
diff --git a/crates/screenpipe-db/src/db/mod.rs b/crates/screenpipe-db/src/db/mod.rs
index 5880beebe..4689f8726 100644
--- a/crates/screenpipe-db/src/db/mod.rs
+++ b/crates/screenpipe-db/src/db/mod.rs
@@ -457,8 +457,9 @@ mod text_positions;
mod write_ops;
pub use self::semantic::{
- SemanticActor, SemanticActorAlias, SemanticActorReference, SemanticCleanupResult,
- SemanticContextQuery, SemanticFrameContext, SemanticProjectionWriteResult,
+ SemanticActor, SemanticActorAlias, SemanticActorReference, SemanticAttachResult,
+ SemanticCleanupResult, SemanticContextQuery, SemanticFrameContext,
+ SemanticProjectionWriteResult,
};
pub(crate) use self::text_positions::calculate_confidence;
pub use self::text_positions::{
diff --git a/crates/screenpipe-db/src/db/semantic.rs b/crates/screenpipe-db/src/db/semantic.rs
index 3adc8523c..9fecfb7e5 100644
--- a/crates/screenpipe-db/src/db/semantic.rs
+++ b/crates/screenpipe-db/src/db/semantic.rs
@@ -16,6 +16,9 @@ use sqlx::{FromRow, QueryBuilder, Sqlite};
use std::collections::{BTreeMap, HashMap};
use std::time::Duration;
+/// Keeps one attach statement well under SQLite's bound-parameter limit.
+const SEMANTIC_ATTACH_FRAME_CHUNK: usize = 256;
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SemanticProjectionWriteResult {
pub run_id: i64,
@@ -24,6 +27,16 @@ pub struct SemanticProjectionWriteResult {
pub items_reused: u32,
}
+/// Outcome of reattaching already-stored parse runs to newly captured frames.
+///
+/// `missing_run_ids` reports runs that cleanup removed between the caller
+/// observing them and this write, so the caller can drop its cached copy.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct SemanticAttachResult {
+ pub frames_attached: u64,
+ pub missing_run_ids: Vec<i64>,
+}
+
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SemanticCleanupResult {
pub runs_deleted: u64,
@@ -349,6 +362,89 @@ impl DatabaseManager {
}
}
+ /// Attach already-stored parse runs to newly captured frames in one
+ /// transaction.
+ ///
+ /// An unchanged screen re-parses to the exact same run fingerprint, so the
+ /// full store path would take the process-wide write lock and replay the
+ /// same statements only to point a new frame at an existing run. Callers
+ /// that already know the run id batch those attachments here instead, which
+ /// turns one serialized transaction per captured frame into one per batch.
+ /// Runs deleted by cleanup in the meantime are skipped and reported rather
+ /// than leaving a dangling reference.
+ pub async fn attach_semantic_runs_to_frames(
+ &self,
+ attachments: &[(i64, i64)],
+ ) -> Result<SemanticAttachResult, sqlx::Error> {
+ if attachments.is_empty() {
+ return Ok(SemanticAttachResult::default());
+ }
+
+ let mut frames_by_run: BTreeMap<i64, Vec<i64>> = BTreeMap::new();
+ for (frame_id, run_id) in attachments {
+ frames_by_run.entry(*run_id).or_default().push(*frame_id);
+ }
+
+ let mut tx = self.begin_immediate_with_retry().await?;
+ let result: Result<SemanticAttachResult, sqlx::Error> = async {
+ let run_ids: Vec<i64> = frames_by_run.keys().copied().collect();
+ let mut existing =
+ QueryBuilder::<Sqlite>::new("SELECT id FROM semantic_runs WHERE id IN (");
+ let mut separated = existing.separated(", ");
+ for run_id in &run_ids {
+ separated.push_bind(*run_id);
+ }
+ existing.push(")");
+ let live_runs: Vec<i64> = existing
+ .build_query_scalar()
+ .fetch_all(&mut **tx.conn())
+ .await?;
+
+ let mut frames_attached = 0_u64;
+ for (run_id, frame_ids) in &frames_by_run {
+ if !live_runs.contains(run_id) {
+ continue;
+ }
+ for chunk in frame_ids.chunks(SEMANTIC_ATTACH_FRAME_CHUNK) {
+ let mut update =
+ QueryBuilder::<Sqlite>::new("UPDATE frames SET semantic_run_id = ");
+ update.push_bind(*run_id);
+ update.push(" WHERE id IN (");
+ let mut separated = update.separated(", ");
+ for frame_id in chunk {
+ separated.push_bind(*frame_id);
+ }
+ update.push(")");
+ frames_attached += update
+ .build()
+ .execute(&mut **tx.conn())
+ .await?
+ .rows_affected();
+ }
+ }
+
+ Ok(SemanticAttachResult {
+ frames_attached,
+ missing_run_ids: run_ids
+ .into_iter()
+ .filter(|run_id| !live_runs.contains(run_id))
+ .collect(),
+ })
+ }
+ .await;
+
+ match result {
+ Ok(result) => {
+ tx.commit().await?;
+ Ok(result)
+ }
+ Err(error) => {
+ tx.rollback().await?;
+ Err(error)
+ }
+ }
+ }
+
pub async fn get_frame_semantic_context(
&self,
frame_id: i64,
diff --git a/crates/screenpipe-db/src/lib.rs b/crates/screenpipe-db/src/lib.rs
index c2a62721b..11372a95b 100644
--- a/crates/screenpipe-db/src/lib.rs
+++ b/crates/screenpipe-db/src/lib.rs
@@ -20,9 +20,9 @@ pub use cancellable_query::{
pub use db::{
find_matching_a11y_positions, parse_all_text_positions, DatabaseManager, DeleteTimeRangeResult,
ImmediateTx, NewMeetingTranscriptSegment, SemanticActor, SemanticActorAlias,
- SemanticActorReference, SemanticCleanupResult, SemanticContextQuery, SemanticFrameContext,
- SemanticProjectionWriteResult, MEETING_END_REASON_AUTO_END, MEETING_END_REASON_EXPLICIT_STOP,
- MEETING_END_REASON_SHUTDOWN,
+ SemanticActorReference, SemanticAttachResult, SemanticCleanupResult, SemanticContextQuery,
+ SemanticFrameContext, SemanticProjectionWriteResult, MEETING_END_REASON_AUTO_END,
+ MEETING_END_REASON_EXPLICIT_STOP, MEETING_END_REASON_SHUTDOWN,
};
pub use recovery::{
rebuild_recovered_fts5_indexes, verify_fresh_sqlite_recovery_candidate, RecoveryVerification,
diff --git a/crates/screenpipe-db/tests/semantic_storage_test.rs b/crates/screenpipe-db/tests/semantic_storage_test.rs
index 10ad1f59b..2de30a86d 100644
--- a/crates/screenpipe-db/tests/semantic_storage_test.rs
+++ b/crates/screenpipe-db/tests/semantic_storage_test.rs
@@ -666,3 +666,78 @@ async fn normalized_storage_growth_tracks_unique_semantics_not_raw_frame_count()
assert!(repeated_per_frame < 4_096.0);
assert!(changing_per_frame < 16_384.0);
}
+
+#[tokio::test]
+async fn batched_attach_points_frames_at_an_existing_run() {
+ let db = database().await;
+ let now = Utc::now();
+ let stored_frame = insert_frame(&db, now - ChronoDuration::minutes(2)).await;
+ let repeated_frames = vec![
+ insert_frame(&db, now - ChronoDuration::minutes(1)).await,
+ insert_frame(&db, now).await,
+ ];
+
+ let write = db
+ .store_semantic_projection(
+ stored_frame,
+ &manifest(),
+ &app(),
+ 7,
+ Duration::from_micros(800),
+ &projection("notarization is blocking the release"),
+ )
+ .await
+ .expect("store first projection");
+
+ let attachments: Vec<(i64, i64)> = repeated_frames
+ .iter()
+ .map(|frame_id| (*frame_id, write.run_id))
+ .collect();
+ let result = db
+ .attach_semantic_runs_to_frames(&attachments)
+ .await
+ .expect("attach cached run");
+
+ assert_eq!(result.frames_attached, 2);
+ assert!(result.missing_run_ids.is_empty());
+ for frame_id in repeated_frames {
+ let context = db
+ .get_frame_semantic_context(frame_id)
+ .await
+ .expect("read frame context")
+ .expect("frame resolves to the reused run");
+ assert_eq!(context.run_id, write.run_id);
+ }
+}
+
+#[tokio::test]
+async fn batched_attach_skips_runs_deleted_since_they_were_observed() {
+ let db = database().await;
+ let now = Utc::now();
+ let frame = insert_frame(&db, now).await;
+ let missing_run_id = 987_654;
+
+ let result = db
+ .attach_semantic_runs_to_frames(&[(frame, missing_run_id)])
+ .await
+ .expect("attach tolerates a deleted run");
+
+ assert_eq!(result.frames_attached, 0);
+ assert_eq!(result.missing_run_ids, vec![missing_run_id]);
+ assert!(db
+ .get_frame_semantic_context(frame)
+ .await
+ .expect("read frame context")
+ .is_none());
+}
+
+#[tokio::test]
+async fn batched_attach_without_work_takes_no_write_transaction() {
+ let db = database().await;
+ let result = db
+ .attach_semantic_runs_to_frames(&[])
+ .await
+ .expect("empty attach succeeds");
+ assert_eq!(result.frames_attached, 0);
+ assert!(result.missing_run_ids.is_empty());
+}
diff --git a/crates/screenpipe-engine/src/semantic_worker.rs b/crates/screenpipe-engine/src/semantic_worker.rs
index 253cce190..92bc5fdfb 100644
--- a/crates/screenpipe-engine/src/semantic_worker.rs
+++ b/crates/screenpipe-engine/src/semantic_worker.rs
@@ -20,14 +20,27 @@ use screenpipe_semantic::{
TreeBudget, ValidatedParseOutcome,
};
use serde_json::{json, Value};
+use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, OnceLock};
-use std::time::Instant;
+use std::time::{Duration, Instant};
use tokio::runtime::Handle;
use tokio::sync::watch;
+use tokio::time::Instant as DeadlineInstant;
use tracing::{debug, info, warn};
const SEMANTIC_TELEMETRY_SAMPLE_DENOMINATOR: i64 = 100;
+/// Remembered parse runs for screens this process already stored. Bounded so a
+/// long session cannot grow the worker's memory with stale app states.
+const SEMANTIC_RUN_CACHE_CAPACITY: usize = 256;
+
+/// Frame attachments buffered before taking the process-wide write lock.
+const SEMANTIC_ATTACH_BATCH: usize = 32;
+
+/// Longest a buffered attachment waits before it is flushed anyway. Retrieval
+/// reads `frames.semantic_run_id`, so a short delay is invisible to callers.
+const SEMANTIC_ATTACH_MAX_DELAY: Duration = Duration::from_secs(2);
+
#[derive(Clone)]
pub(crate) struct SemanticProjectionSender {
tx: watch::Sender<Option<Arc<SemanticProjectionJob>>>,
@@ -97,6 +110,126 @@ const fn semantic_mode_label(mode: SemanticContextMode) -> &'static str {
}
}
+/// Identifies one stored parse run without holding any captured content.
+///
+/// These are exactly the fields the persisted run fingerprint is derived from,
+/// so an equal key provably resolves to the same `semantic_runs` row.
+#[derive(Clone, PartialEq, Eq, Hash)]
+struct SemanticRunCacheKey {
+ platform: u8,
+ app_id: Option<String>,
+ executable: Option<String>,
+ app_version: Option<String>,
+ browser_url: Option<String>,
+ content_hash: u64,
+}
+
+impl SemanticRunCacheKey {
+ fn new(app: &AppIdentity, content_hash: u64) -> Self {
+ Self {
+ platform: app.platform as u8,
+ app_id: app.app_id.clone(),
+ executable: app.executable.clone(),
+ app_version: app.version.clone(),
+ browser_url: app.browser_url.clone(),
+ content_hash,
+ }
+ }
+}
+
+/// Bounded insertion-ordered cache of screens this process already stored.
+#[derive(Default)]
+struct SemanticRunCache {
+ runs: HashMap<SemanticRunCacheKey, i64>,
+ order: VecDeque<SemanticRunCacheKey>,
+}
+
+impl SemanticRunCache {
+ fn get(&self, key: &SemanticRunCacheKey) -> Option<i64> {
+ self.runs.get(key).copied()
+ }
+
+ fn insert(&mut self, key: SemanticRunCacheKey, run_id: i64) {
+ if self.runs.insert(key.clone(), run_id).is_none() {
+ self.order.push_back(key);
+ }
+ while self.order.len() > SEMANTIC_RUN_CACHE_CAPACITY {
+ if let Some(evicted) = self.order.pop_front() {
+ self.runs.remove(&evicted);
+ }
+ }
+ }
+
+ /// Drops every screen pointing at a run that no longer exists, so a run
+ /// removed by cleanup can never be reattached from memory.
+ fn forget_runs(&mut self, run_ids: &[i64]) {
+ if run_ids.is_empty() {
+ return;
+ }
+ self.runs.retain(|_, run_id| !run_ids.contains(run_id));
+ self.order.retain(|key| self.runs.contains_key(key));
+ }
+}
+
+/// Worker-local state: what this process has already stored, and which frame
+/// attachments are waiting for the next batched write.
+#[derive(Default)]
+struct SemanticProjectionState {
+ cache: SemanticRunCache,
+ pending_attachments: Vec<(i64, i64)>,
+ flush_deadline: Option<DeadlineInstant>,
+}
+
+impl SemanticProjectionState {
+ fn queue_attachment(&mut self, frame_id: i64, run_id: i64) {
+ self.pending_attachments.push((frame_id, run_id));
+ self.flush_deadline
+ .get_or_insert_with(|| DeadlineInstant::now() + SEMANTIC_ATTACH_MAX_DELAY);
+ }
+
+ fn should_flush(&self) -> bool {
+ self.pending_attachments.len() >= SEMANTIC_ATTACH_BATCH
+ }
+}
+
+/// Writes buffered attachments in one transaction. Failures are logged and the
+/// buffer is cleared: the affected frames keep generic capture, and the next
+/// capture of the same screen re-stores the projection.
+async fn flush_pending_attachments(db: &DatabaseManager, state: &mut SemanticProjectionState) {
+ state.flush_deadline = None;
+ if state.pending_attachments.is_empty() {
+ return;
+ }
+ let attachments = std::mem::take(&mut state.pending_attachments);
+ match db.attach_semantic_runs_to_frames(&attachments).await {
+ Ok(result) => {
+ state.cache.forget_runs(&result.missing_run_ids);
+ debug!(
+ attachments = attachments.len(),
+ frames_attached = result.frames_attached,
+ missing_runs = result.missing_run_ids.len(),
+ "semantic run attachments flushed"
+ );
+ }
+ Err(error) => {
+ // A failed batch must not strand cached runs that may be gone.
+ state.cache = SemanticRunCache::default();
+ warn!(
+ attachments = attachments.len(),
+ %error,
+ "semantic run attachment flush failed; generic capture remains available"
+ );
+ }
+ }
+}
+
+async fn wait_for_flush_deadline(deadline: Option<DeadlineInstant>) {
+ match deadline {
+ Some(deadline) => tokio::time::sleep_until(deadline).await,
+ None => std::future::pending::<()>().await,
+ }
+}
+
fn semantic_projection_channel() -> (
SemanticProjectionSender,
watch::Receiver<Option<Arc<SemanticProjectionJob>>>,
@@ -119,7 +252,22 @@ async fn run_semantic_projection_worker(
}
};
- while rx.changed().await.is_ok() {
+ let mut state = SemanticProjectionState::default();
+ loop {
+ tokio::select! {
+ // Buffered attachments must not wait for the next capture, which
+ // may never come if the user stops working.
+ _ = wait_for_flush_deadline(state.flush_deadline) => {
+ flush_pending_attachments(&db, &mut state).await;
+ continue;
+ }
+ changed = rx.changed() => {
+ if changed.is_err() {
+ break;
+ }
+ }
+ }
+
let Some(job) = rx.borrow_and_update().clone() else {
continue;
};
@@ -137,14 +285,20 @@ async fn run_semantic_projection_worker(
false
}
});
- if let Err(error) = process_semantic_job(&db, &registry, &job, mode_label).await {
+ if let Err(error) = process_semantic_job(&db, &registry, &job, mode_label, &mut state).await
+ {
warn!(
frame_id = job.frame_id,
%error,
"semantic projection failed; generic capture remains available"
);
}
+ if state.should_flush() {
+ flush_pending_attachments(&db, &mut state).await;
+ }
}
+
+ flush_pending_attachments(&db, &mut state).await;
}
async fn process_semantic_job(
@@ -152,6 +306,7 @@ async fn process_semantic_job(
registry: &ParserRegistry,
job: &SemanticProjectionJob,
mode_label: &'static str,
+ state: &mut SemanticProjectionState,
) -> anyhow::Result<()> {
let app = AppIdentity {
platform: current_platform(),
@@ -199,6 +354,23 @@ async fn process_semantic_job(
}
let input_content_hash = adapted.tree.structural_fingerprint();
+ // An unchanged screen resolves to the run this process already stored.
+ // Parsing it again is deterministic and the write would only reattach the
+ // same run, so buffer the attachment and skip both.
+ let cache_key = SemanticRunCacheKey::new(&app, input_content_hash);
+ if let Some(run_id) = state.cache.get(&cache_key) {
+ state.queue_attachment(job.frame_id, run_id);
+ emit_sampled_semantic_telemetry(
+ job.frame_id,
+ &candidate_parser_ids,
+ None,
+ "handled_cached",
+ 0,
+ mode_label,
+ );
+ return Ok(());
+ }
+
let context = ParseContext {
frame_id: job.frame_id,
captured_at_unix_ms: job.captured_at.timestamp_millis(),
@@ -241,6 +413,7 @@ async fn process_semantic_job(
adapted.stats.suppressed_offscreen_content_nodes;
let suppressed_offscreen_content_bytes =
adapted.stats.suppressed_offscreen_content_bytes;
+ let store_started = Instant::now();
let write = db
.store_semantic_projection(
job.frame_id,
@@ -251,13 +424,19 @@ async fn process_semantic_job(
&projection,
)
.await?;
- emit_sampled_semantic_telemetry(
+ let store_duration_us = store_started.elapsed().as_micros() as u64;
+ state.cache.insert(cache_key, write.run_id);
+ emit_sampled_semantic_telemetry_with_write(
job.frame_id,
&candidate_parser_ids,
Some(parser_id),
"handled",
parse_duration_us,
mode_label,
+ Some(SemanticWriteSample {
+ store_duration_us,
+ reused_run: write.reused_run,
+ }),
);
debug!(
frame_id = job.frame_id,
@@ -303,6 +482,34 @@ fn emit_sampled_semantic_telemetry(
outcome: &'static str,
parse_duration_us: u64,
mode_label: &'static str,
+) {
+ emit_sampled_semantic_telemetry_with_write(
+ frame_id,
+ candidate_parser_ids,
+ selected_parser_id,
+ outcome,
+ parse_duration_us,
+ mode_label,
+ None,
+ );
+}
+
+/// Content-free cost of one persisted projection. Without this the write path
+/// is invisible in production and a storage regression cannot be measured.
+#[derive(Debug, Clone, Copy)]
+struct SemanticWriteSample {
+ store_duration_us: u64,
+ reused_run: bool,
+}
+
+fn emit_sampled_semantic_telemetry_with_write(
+ frame_id: i64,
+ candidate_parser_ids: &[String],
+ selected_parser_id: Option<&str>,
+ outcome: &'static str,
+ parse_duration_us: u64,
+ mode_label: &'static str,
+ write: Option<SemanticWriteSample>,
) {
if !should_sample_semantic_telemetry(frame_id) {
return;
@@ -316,6 +523,7 @@ fn emit_sampled_semantic_telemetry(
outcome,
parse_duration_us,
mode_label,
+ write,
),
);
}
@@ -330,6 +538,7 @@ fn semantic_telemetry_properties(
outcome: &'static str,
parse_duration_us: u64,
mode_label: &'static str,
+ write: Option<SemanticWriteSample>,
) -> Value {
json!({
"semantic_candidate_parser_ids": candidate_parser_ids,
@@ -338,6 +547,8 @@ fn semantic_telemetry_properties(
"semantic_parse_duration_us": parse_duration_us,
"semantic_platform": platform_label(current_platform()),
"semantic_mode": mode_label,
+ "semantic_store_duration_us": write.map(|write| write.store_duration_us),
+ "semantic_reused_run": write.map(|write| write.reused_run),
})
}
@@ -528,6 +739,86 @@ mod tests {
)
}
+ fn cache_key(app_id: &str, content_hash: u64) -> SemanticRunCacheKey {
+ SemanticRunCacheKey::new(
+ &AppIdentity {
+ platform: current_platform(),
+ app_id: Some(app_id.into()),
+ executable: None,
+ display_name: "Slack".into(),
+ version: Some("4.39.95".into()),
+ browser_url: None,
+ },
+ content_hash,
+ )
+ }
+
+ #[test]
+ fn unchanged_screen_resolves_to_the_stored_run() {
+ let mut cache = SemanticRunCache::default();
+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 11), 7);
+
+ assert_eq!(
+ cache.get(&cache_key("com.tinyspeck.slackmacgap", 11)),
+ Some(7)
+ );
+ // A changed screen and a different app must both miss, or a frame
+ // would be attached to a run that never described it.
+ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 12)), None);
+ assert_eq!(cache.get(&cache_key("com.other.app", 11)), None);
+ }
+
+ #[test]
+ fn cache_stays_bounded_for_a_long_session() {
+ let mut cache = SemanticRunCache::default();
+ for index in 0..(SEMANTIC_RUN_CACHE_CAPACITY as u64 + 10) {
+ cache.insert(cache_key("com.tinyspeck.slackmacgap", index), index as i64);
+ }
+
+ assert_eq!(cache.runs.len(), SEMANTIC_RUN_CACHE_CAPACITY);
+ assert_eq!(cache.order.len(), SEMANTIC_RUN_CACHE_CAPACITY);
+ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 0)), None);
+ let newest = SEMANTIC_RUN_CACHE_CAPACITY as u64 + 9;
+ assert_eq!(
+ cache.get(&cache_key("com.tinyspeck.slackmacgap", newest)),
+ Some(newest as i64)
+ );
+ }
+
+ #[test]
+ fn deleted_runs_are_never_reattached_from_memory() {
+ let mut cache = SemanticRunCache::default();
+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 1), 41);
+ cache.insert(cache_key("com.tinyspeck.slackmacgap", 2), 42);
+
+ cache.forget_runs(&[41]);
+
+ assert_eq!(cache.get(&cache_key("com.tinyspeck.slackmacgap", 1)), None);
+ assert_eq!(
+ cache.get(&cache_key("com.tinyspeck.slackmacgap", 2)),
+ Some(42)
+ );
+ assert_eq!(cache.order.len(), 1);
+ }
+
+ #[test]
+ fn attachments_batch_before_taking_the_write_lock() {
+ let mut state = SemanticProjectionState::default();
+ assert!(state.flush_deadline.is_none());
+
+ for frame_id in 0..(SEMANTIC_ATTACH_BATCH as i64 - 1) {
+ state.queue_attachment(frame_id, 7);
+ assert!(!state.should_flush());
+ }
+ // A partial batch still has a deadline so it cannot wait forever for a
+ // capture that may never arrive.
+ assert!(state.flush_deadline.is_some());
+
+ state.queue_attachment(SEMANTIC_ATTACH_BATCH as i64, 7);
+ assert!(state.should_flush());
+ assert_eq!(state.pending_attachments.len(), SEMANTIC_ATTACH_BATCH);
+ }
+
#[tokio::test]
async fn pending_slot_keeps_only_latest_frame() {
let (sender, mut receiver) = semantic_projection_channel();
@@ -589,6 +880,10 @@ mod tests {
"handled",
73,
"memory",
+ Some(SemanticWriteSample {
+ store_duration_us: 1_200,
+ reused_run: true,
+ }),
);
assert_eq!(properties["semantic_outcome"], "handled");
@@ -598,7 +893,9 @@ mod tests {
platform_label(current_platform())
);
assert_eq!(properties["semantic_mode"], "memory");
- assert_eq!(properties.as_object().map(|object| object.len()), Some(6));
+ assert_eq!(properties["semantic_store_duration_us"], 1_200);
+ assert_eq!(properties["semantic_reused_run"], true);
+ assert_eq!(properties.as_object().map(|object| object.len()), Some(8));
let serialized = properties.to_string();
assert!(!serialized.contains("app_identifier"));
assert!(!serialized.contains("display_name"));
@@ -609,6 +906,80 @@ mod tests {
assert!(!serialized.contains("text_content"));
}
+ #[tokio::test]
+ async fn unchanged_screen_stores_once_and_batches_later_frames() {
+ let db = DatabaseManager::new("sqlite::memory:", Default::default())
+ .await
+ .expect("in-memory database");
+ let registry = builtin_parser_registry().expect("built-in registry");
+ let mut state = SemanticProjectionState::default();
+
+ let mut frame_ids = Vec::new();
+ for index in 0..3 {
+ let captured_at = Utc::now();
+ let frame_id = db
+ .insert_snapshot_frame(
+ "test-device",
+ captured_at,
+ &format!("/tmp/semantic-worker-{index}.jpg"),
+ Some("Slack"),
+ Some("#release - Slack"),
+ None,
+ true,
+ Some("test"),
+ Some("release"),
+ Some("accessibility"),
+ None,
+ None,
+ None,
+ )
+ .await
+ .expect("insert frame");
+ frame_ids.push(frame_id);
+
+ let job = SemanticProjectionJob::from_capture(
+ frame_id,
+ captured_at,
+ Some("Slack".into()),
+ None,
+ // The identical screen, captured repeatedly.
+ slack_snapshot("notarization is blocking the release"),
+ false,
+ );
+ process_semantic_job(&db, &registry, &job, "memory", &mut state)
+ .await
+ .expect("process semantic projection");
+ }
+
+ // Only the first capture paid for a store transaction. The other two
+ // resolved from memory and are waiting for one batched write.
+ assert_eq!(state.cache.runs.len(), 1);
+ assert_eq!(state.pending_attachments.len(), 2);
+ assert_eq!(
+ db.get_frame_semantic_context(frame_ids[1])
+ .await
+ .expect("read frame context")
+ .map(|context| context.run_id),
+ None
+ );
+
+ flush_pending_attachments(&db, &mut state).await;
+ assert!(state.pending_attachments.is_empty());
+ assert!(state.flush_deadline.is_none());
+
+ let mut run_ids = Vec::new();
+ for frame_id in frame_ids {
+ let context = db
+ .get_frame_semantic_context(frame_id)
+ .await
+ .expect("read frame context")
+ .expect("every repeated frame resolves to the stored run");
+ run_ids.push(context.run_id);
+ }
+ assert_eq!(run_ids[0], run_ids[1]);
+ assert_eq!(run_ids[1], run_ids[2]);
+ }
+
#[tokio::test]
async fn supported_capture_is_parsed_redacted_and_persisted() {
let db = DatabaseManager::new("sqlite::memory:", Default::default())
@@ -643,7 +1014,8 @@ mod tests {
true,
);
- process_semantic_job(&db, &registry, &projection_job, "memory")
+ let mut state = SemanticProjectionState::default();
+ process_semantic_job(&db, &registry, &projection_job, "memory", &mut state)
.await
.expect("process semantic projection");
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index f33817205..586f0f863 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y
- Mapped suites: 32
- Mapped Rust files: 318
-- Active test blocks: 2993
+- Active test blocks: 3001
- Ignored/manual test blocks: 137
-- Declared test blocks: 3130
-- Weighted coverage points: 2461.2
+- Declared test blocks: 3138
+- Weighted coverage points: 2467.7
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,16 +23,16 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 29 | 2862 | 132 | 2401.2 | 21 | 11 | 100% |
-| macos | 29 | 2916 | 112 | 2412.3 | 22 | 11 | 100% |
-| linux | 25 | 2549 | 105 | 2118.7 | 20 | 11 | 100% |
+| windows | 29 | 2870 | 132 | 2407.7 | 21 | 11 | 100% |
+| macos | 29 | 2924 | 112 | 2418.8 | 22 | 11 | 100% |
+| linux | 25 | 2557 | 105 | 2125.2 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 10 | 18 | 102 | 1432 | 42 | 1098.3 | 10 |
-| screenpipe-db | 5 | 51 | 14 | 420 | 16 | 399.7 | 9 |
+| screenpipe-engine | 10 | 18 | 102 | 1437 | 42 | 1101.8 | 10 |
+| screenpipe-db | 5 | 51 | 14 | 423 | 16 | 402.7 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 2 | 11 | 0 | 11.0 | 2 |
| screenpipe-audio | 6 | 25 | 49 | 563 | 43 | 491.2 | 5 |
| screenpipe-screen | 6 | 9 | 18 | 234 | 9 | 213.5 | 4 |
@@ -65,24 +65,24 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| audio | 7 suites / 659 active / 44 ignored / 587.2 pts | 7 suites / 659 active / 44 ignored / 587.2 pts | 6 suites / 584 active / 43 ignored / 534.7 pts |
| audio-device | 2 suites / 206 active / 7 ignored / 183.5 pts | 2 suites / 206 active / 7 ignored / 183.5 pts | 1 suites / 131 active / 6 ignored / 131.0 pts |
| configuration | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts | 2 suites / 135 active / 3 ignored / 122.8 pts |
-| database | 6 suites / 338 active / 12 ignored / 317.7 pts | 6 suites / 338 active / 12 ignored / 317.7 pts | 6 suites / 338 active / 12 ignored / 317.7 pts |
+| database | 6 suites / 341 active / 12 ignored / 320.7 pts | 6 suites / 341 active / 12 ignored / 320.7 pts | 6 suites / 341 active / 12 ignored / 320.7 pts |
| db-search | 2 suites / 107 active / 9 ignored / 107.0 pts | 2 suites / 107 active / 9 ignored / 107.0 pts | 2 suites / 107 active / 9 ignored / 107.0 pts |
| engine-lifecycle | 6 suites / 205 active / 1 ignored / 180.0 pts | 6 suites / 205 active / 1 ignored / 180.0 pts | 5 suites / 199 active / 1 ignored / 178.3 pts |
| local-api | 2 suites / 322 active / 9 ignored / 227.2 pts | 2 suites / 322 active / 9 ignored / 227.2 pts | 2 suites / 322 active / 9 ignored / 227.2 pts |
| meeting | 6 suites / 1373 active / 19 ignored / 1117.4 pts | 6 suites / 1373 active / 19 ignored / 1117.4 pts | 4 suites / 1095 active / 15 ignored / 861.9 pts |
| ocr | 4 suites / 120 active / 7 ignored / 114.3 pts | 4 suites / 124 active / 7 ignored / 119.8 pts | 3 suites / 115 active / 6 ignored / 110.8 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
-| performance | 13 suites / 1342 active / 67 ignored / 1186.2 pts | 14 suites / 1440 active / 70 ignored / 1225.4 pts | 13 suites / 1342 active / 67 ignored / 1186.2 pts |
+| performance | 13 suites / 1350 active / 67 ignored / 1192.7 pts | 14 suites / 1448 active / 70 ignored / 1231.9 pts | 13 suites / 1350 active / 67 ignored / 1192.7 pts |
| pipes | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts |
| privacy | 5 suites / 869 active / 36 ignored / 706.1 pts | 5 suites / 919 active / 16 ignored / 711.7 pts | 5 suites / 845 active / 14 ignored / 684.3 pts |
| real-app | - | 1 suites / 98 active / 3 ignored / 39.2 pts | - |
| speaker | 2 suites / 318 active / 8 ignored / 318.0 pts | 2 suites / 318 active / 8 ignored / 318.0 pts | 2 suites / 318 active / 8 ignored / 318.0 pts |
-| storage | 3 suites / 464 active / 29 ignored / 376.4 pts | 3 suites / 464 active / 29 ignored / 376.4 pts | 3 suites / 464 active / 29 ignored / 376.4 pts |
+| storage | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts | 3 suites / 472 active / 29 ignored / 382.9 pts |
| sync | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts | 1 suites / 461 active / 3 ignored / 322.7 pts |
-| timeline | 4 suites / 917 active / 33 ignored / 746.6 pts | 4 suites / 917 active / 33 ignored / 746.6 pts | 4 suites / 917 active / 33 ignored / 746.6 pts |
+| timeline | 4 suites / 925 active / 33 ignored / 753.1 pts | 4 suites / 925 active / 33 ignored / 753.1 pts | 4 suites / 925 active / 33 ignored / 753.1 pts |
| transcription | 5 suites / 673 active / 40 ignored / 528.9 pts | 5 suites / 673 active / 40 ignored / 528.9 pts | 5 suites / 673 active / 40 ignored / 528.9 pts |
| ui-events | 4 suites / 696 active / 28 ignored / 531.1 pts | 3 suites / 648 active / 5 ignored / 497.5 pts | 3 suites / 648 active / 5 ignored / 497.5 pts |
-| vision-capture | 5 suites / 477 active / 32 ignored / 380.9 pts | 5 suites / 481 active / 32 ignored / 386.4 pts | 4 suites / 472 active / 31 ignored / 377.4 pts |
+| vision-capture | 5 suites / 482 active / 32 ignored / 384.4 pts | 5 suites / 486 active / 32 ignored / 389.9 pts | 4 suites / 477 active / 31 ignored / 380.9 pts |
## Critical Flow Matrix
@@ -91,14 +91,14 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| Settings to engine recording config | configuration | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) | covered (strong; engine-config-lifecycle, db-accessibility-ui-events) |
| Engine health, sleep, and lifecycle | engine-lifecycle | covered (strong; engine-config-lifecycle, engine-retention-storage) | covered (strong; engine-config-lifecycle, engine-retention-storage) | covered (strong; engine-config-lifecycle, engine-retention-storage) |
| Capture, OCR, and frame persistence | vision-capture, ocr | covered (partial; screen-capture-ocr-contract, screen-windows-ocr) | covered (strong; screen-capture-ocr-contract, screen-macos-ocr) | covered (partial; screen-capture-ocr-contract) |
-| Timeline frame and stream delivery | timeline | covered (strong; engine-api-routes, screen-capture-windowing) | covered (strong; engine-api-routes, screen-capture-windowing) | covered (strong; engine-api-routes, screen-capture-windowing) |
+| Timeline frame and stream delivery | timeline | covered (strong; engine-api-routes, engine-capture-timeline) | covered (strong; engine-api-routes, engine-capture-timeline) | covered (strong; engine-api-routes, engine-capture-timeline) |
| Local API search and indexing | local-api, db-search | covered (strong; engine-local-api-search-integration) | covered (strong; engine-local-api-search-integration) | covered (strong; engine-local-api-search-integration) |
| Audio record, transcribe, and reconcile | audio, transcription | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) | covered (strong; audio-meetings-speakers-dedup, audio-transcription-pipeline) |
| Audio device and stream health | audio-device | covered (strong; audio-device-stream-health, audio-platform-output-capture) | covered (strong; audio-device-stream-health, audio-platform-output-capture) | covered (strong; audio-device-stream-health) |
| Meeting detection and live transcript merge | meeting | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) | covered (strong; engine-meeting-privacy-sync, audio-meetings-speakers-dedup) |
| Privacy filters, DRM guards, and redaction | privacy | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) | covered (strong; engine-meeting-privacy-sync, screen-capture-windowing) |
| Accessibility tree and UI event capture | accessibility, ui-events | covered (strong; a11y-core-tree-cross-platform, a11y-windows-tree) | covered (strong; a11y-core-tree-cross-platform, db-accessibility-ui-events) | covered (strong; a11y-core-tree-cross-platform, db-accessibility-ui-events) |
-| Performance, backpressure, and liveness | performance | covered (strong; screen-capture-windowing, engine-capture-timeline) | covered (strong; screen-capture-windowing, engine-capture-timeline) | covered (strong; screen-capture-windowing, engine-capture-timeline) |
+| Performance, backpressure, and liveness | performance | covered (strong; engine-capture-timeline, screen-capture-windowing) | covered (strong; engine-capture-timeline, screen-capture-windowing) | covered (strong; engine-capture-timeline, screen-capture-windowing) |
## Critical Gaps
@@ -132,9 +132,9 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 96 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. |
| db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 12 | 27 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes. |
| db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 101 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |
-| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 172 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 18 | 175 | 3 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. |
| engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 31 | 316 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. |
-| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 252 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
+| engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 23 | 257 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |
| engine-config-lifecycle | screenpipe-engine | windows, macos, linux | configuration, engine-lifecycle, performance | settings-to-engine-config, engine-health-lifecycle, performance-liveness | high | strong | mixed | 11 | 111 | 1 | Fast logic coverage for the config bridge, tray health debounce, sleep/power policies, and queue backpressure. |
| engine-db-recovery-cli | screenpipe-engine | windows, macos, linux | database, engine-lifecycle | engine-health-lifecycle, performance-liveness | high | strong | unit | 1 | 8 | 0 | Exact DB/WAL/SHM working-copy preservation, rollback on archive failure, and restart repair for crashes during the multi-file generation swap. |
| engine-focus-os | screenpipe-engine | windows, macos | engine-lifecycle, os-integration | engine-health-lifecycle, performance-liveness | medium | conditional | unit | 3 | 6 | 0 | Platform focus-tracker parsing/helpers. These files are cfg-gated and only execute on their target OS. |
@@ -307,7 +307,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| db-timeline-frames | screenpipe-db | tests/range_export_frame_sources_test.rs | integration | 3 | 0 | 3 |
| db-search-indexing | screenpipe-db | tests/search_issue_4474_bench.rs | integration | 0 | 1 | 1 |
| db-search-indexing | screenpipe-db | tests/search_ocr_snapshot_test.rs | integration | 4 | 0 | 4 |
-| db-timeline-frames | screenpipe-db | tests/semantic_storage_test.rs | integration | 6 | 0 | 6 |
+| db-timeline-frames | screenpipe-db | tests/semantic_storage_test.rs | integration | 9 | 0 | 9 |
| db-audio-meetings-speakers | screenpipe-db | tests/single_open_meeting_invariant_test.rs | integration | 3 | 0 | 3 |
| db-audio-meetings-speakers | screenpipe-db | tests/speaker_benchmark.rs | integration | 0 | 1 | 1 |
| db-audio-meetings-speakers | screenpipe-db | tests/speaker_delete_test.rs | integration | 3 | 0 | 3 |
@@ -412,7 +412,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/routes/timezone.rs | source | 8 | 0 | 8 |
| engine-api-routes | screenpipe-engine | src/routes/websocket.rs | source | 4 | 0 | 4 |
| engine-config-lifecycle | screenpipe-engine | src/schedule_monitor.rs | source | 8 | 0 | 8 |
-| engine-capture-timeline | screenpipe-engine | src/semantic_worker.rs | source | 6 | 0 | 6 |
+| engine-capture-timeline | screenpipe-engine | src/semantic_worker.rs | source | 11 | 0 | 11 |
| engine-api-routes | screenpipe-engine | src/server.rs | source | 13 | 0 | 13 |
| engine-config-lifecycle | screenpipe-engine | src/sleep_monitor.rs | source | 11 | 1 | 12 |
| engine-capture-timeline | screenpipe-engine | src/snapshot_compaction.rs | source | 18 | 0 | 18 |