Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ui-recorder): merge enriched click context events (#4909)

Louis Beaumontdc2a61377d3b
diff --git a/crates/screenpipe-engine/src/ui_recorder.rs b/crates/screenpipe-engine/src/ui_recorder.rs
index 75923035f..9fd68be78 100644
--- a/crates/screenpipe-engine/src/ui_recorder.rs
+++ b/crates/screenpipe-engine/src/ui_recorder.rs
@@ -47,6 +47,43 @@ impl EventBatch {
self.events.push(event);
self.correlation_ids.push(correlation_id);
}
+
+ fn try_merge_click_context(&mut self, event: &InsertUiEvent) -> bool {
+ if !is_click_context_enrichment(event) {
+ return false;
+ }
+
+ let Some(candidate) = self
+ .events
+ .iter_mut()
+ .rev()
+ .find(|candidate| click_context_merge_candidate(candidate, event))
+ else {
+ return false;
+ };
+
+ merge_optional(&mut candidate.app_name, event.app_name.clone());
+ merge_optional(&mut candidate.window_title, event.window_title.clone());
+ merge_optional(&mut candidate.browser_url, event.browser_url.clone());
+ merge_optional(&mut candidate.element_role, event.element_role.clone());
+ merge_optional(&mut candidate.element_name, event.element_name.clone());
+ merge_optional(&mut candidate.element_value, event.element_value.clone());
+ merge_optional(
+ &mut candidate.element_description,
+ event.element_description.clone(),
+ );
+ merge_optional(
+ &mut candidate.element_automation_id,
+ event.element_automation_id.clone(),
+ );
+ merge_optional(&mut candidate.element_bounds, event.element_bounds.clone());
+ merge_optional(
+ &mut candidate.element_ancestors,
+ event.element_ancestors.clone(),
+ );
+ true
+ }
+
fn len(&self) -> usize {
debug_assert_eq!(self.events.len(), self.correlation_ids.len());
self.events.len()
@@ -66,6 +103,50 @@ impl EventBatch {
}
}
+fn merge_optional<T>(dst: &mut Option<T>, src: Option<T>) {
+ if src.is_some() {
+ *dst = src;
+ }
+}
+
+fn is_click_context_enrichment(event: &InsertUiEvent) -> bool {
+ event.event_type == screenpipe_db::UiEventType::Click
+ && (event.element_role.is_some()
+ || event.element_name.is_some()
+ || event.element_value.is_some()
+ || event.element_description.is_some()
+ || event.element_automation_id.is_some()
+ || event.element_bounds.is_some()
+ || event.element_ancestors.is_some())
+}
+
+fn click_context_merge_candidate(candidate: &InsertUiEvent, enrichment: &InsertUiEvent) -> bool {
+ candidate.event_type == screenpipe_db::UiEventType::Click
+ && candidate.x == enrichment.x
+ && candidate.y == enrichment.y
+ && compatible_text_context(&candidate.app_name, &enrichment.app_name)
+ && compatible_text_context(&candidate.window_title, &enrichment.window_title)
+ && click_enrichment_follows_candidate(candidate, enrichment)
+}
+
+fn compatible_text_context(a: &Option<String>, b: &Option<String>) -> bool {
+ match (a.as_deref(), b.as_deref()) {
+ (Some(a), Some(b)) => a == b,
+ _ => true,
+ }
+}
+
+fn click_enrichment_follows_candidate(
+ candidate: &InsertUiEvent,
+ enrichment: &InsertUiEvent,
+) -> bool {
+ let delta = enrichment
+ .timestamp
+ .signed_duration_since(candidate.timestamp)
+ .num_milliseconds();
+ (0..=100).contains(&delta)
+}
+
/// Configuration for UI event capture
#[derive(Debug, Clone)]
pub struct UiRecorderConfig {
@@ -769,6 +850,9 @@ pub async fn start_ui_recording(
record_clipboard_events,
record_click_events,
);
+ if should_record_event && batch.try_merge_click_context(&db_event) {
+ continue;
+ }
// Decide whether this event warrants a capture and, if so,
// mint a correlation id that travels with the trigger AND
@@ -1259,7 +1343,7 @@ fn next_ui_event_recv_timeout_at(
#[cfg(test)]
mod event_batch_tests {
use super::*;
- use chrono::Utc;
+ use chrono::{Duration as ChronoDuration, Utc};
use screenpipe_db::UiEventType;
fn evt() -> InsertUiEvent {
@@ -1292,6 +1376,17 @@ mod event_batch_tests {
}
}
+ fn click_at(x: i32, y: i32) -> InsertUiEvent {
+ let mut e = evt();
+ e.x = Some(x);
+ e.y = Some(y);
+ e.button = Some(0);
+ e.click_count = Some(1);
+ e.app_name = Some("Arc".to_string());
+ e.window_title = Some("screenpipe".to_string());
+ e
+ }
+
#[test]
fn push_keeps_parallel_vecs_aligned() {
let mut b = EventBatch::with_capacity(4);
@@ -1326,6 +1421,114 @@ mod event_batch_tests {
assert_eq!(b.correlation_ids.len(), 0);
}
+ #[test]
+ fn enriched_click_merges_into_pending_raw_click_and_keeps_corr_id() {
+ let mut b = EventBatch::with_capacity(4);
+ let raw = click_at(10, 20);
+ let raw_ts = raw.timestamp;
+ b.push(raw, Some(42));
+
+ let mut enriched = click_at(10, 20);
+ enriched.timestamp = raw_ts + ChronoDuration::milliseconds(5);
+ enriched.element_role = Some("AXButton".to_string());
+ enriched.element_name = Some("Continue".to_string());
+ enriched.element_bounds = Some(r#"{"x":1.0,"y":2.0,"width":3.0,"height":4.0}"#.to_string());
+ enriched.element_ancestors =
+ Some(r#"[{"role":"AXWindow","name":"screenpipe"}]"#.to_string());
+
+ assert!(b.try_merge_click_context(&enriched));
+ assert_eq!(b.len(), 1, "supplemental click should not add a row");
+ assert_eq!(
+ b.correlation_ids,
+ vec![Some(42)],
+ "raw click linkage survives"
+ );
+
+ let merged = &b.events[0];
+ assert_eq!(merged.x, Some(10));
+ assert_eq!(merged.y, Some(20));
+ assert_eq!(merged.button, Some(0));
+ assert_eq!(merged.click_count, Some(1));
+ assert_eq!(merged.app_name.as_deref(), Some("Arc"));
+ assert_eq!(merged.window_title.as_deref(), Some("screenpipe"));
+ assert_eq!(merged.element_role.as_deref(), Some("AXButton"));
+ assert_eq!(merged.element_name.as_deref(), Some("Continue"));
+ assert_eq!(
+ merged.element_ancestors.as_deref(),
+ Some(r#"[{"role":"AXWindow","name":"screenpipe"}]"#)
+ );
+ }
+
+ #[test]
+ fn enriched_click_with_missing_context_keeps_raw_app_window() {
+ let mut b = EventBatch::with_capacity(4);
+ let raw = click_at(33, 44);
+ let raw_ts = raw.timestamp;
+ b.push(raw, Some(7));
+
+ let mut enriched = click_at(33, 44);
+ enriched.timestamp = raw_ts + ChronoDuration::milliseconds(3);
+ enriched.app_name = None;
+ enriched.window_title = None;
+ enriched.click_count = Some(0);
+ enriched.element_role = Some("Button".to_string());
+
+ assert!(b.try_merge_click_context(&enriched));
+ assert_eq!(b.len(), 1);
+ assert_eq!(b.events[0].app_name.as_deref(), Some("Arc"));
+ assert_eq!(b.events[0].window_title.as_deref(), Some("screenpipe"));
+ assert_eq!(b.events[0].click_count, Some(1));
+ assert_eq!(b.events[0].element_role.as_deref(), Some("Button"));
+ }
+
+ #[test]
+ fn enriched_click_replaces_approximate_context_without_losing_linkage() {
+ let mut b = EventBatch::with_capacity(4);
+ let mut raw = click_at(50, 60);
+ raw.element_role = Some("FocusedElement".to_string());
+ raw.element_name = Some("Nearby".to_string());
+ let raw_ts = raw.timestamp;
+ b.push(raw, Some(99));
+
+ let mut enriched = click_at(50, 60);
+ enriched.timestamp = raw_ts + ChronoDuration::milliseconds(8);
+ enriched.app_name = None;
+ enriched.window_title = None;
+ enriched.click_count = Some(0);
+ enriched.element_role = Some("Button".to_string());
+ enriched.element_name = Some("Save".to_string());
+ enriched.element_ancestors = Some(r#"[{"role":"Window","name":"screenpipe"}]"#.to_string());
+
+ assert!(b.try_merge_click_context(&enriched));
+ assert_eq!(b.len(), 1);
+ assert_eq!(b.correlation_ids, vec![Some(99)]);
+ assert_eq!(b.events[0].app_name.as_deref(), Some("Arc"));
+ assert_eq!(b.events[0].window_title.as_deref(), Some("screenpipe"));
+ assert_eq!(b.events[0].click_count, Some(1));
+ assert_eq!(b.events[0].element_role.as_deref(), Some("Button"));
+ assert_eq!(b.events[0].element_name.as_deref(), Some("Save"));
+ assert_eq!(
+ b.events[0].element_ancestors.as_deref(),
+ Some(r#"[{"role":"Window","name":"screenpipe"}]"#)
+ );
+ }
+
+ #[test]
+ fn enriched_click_does_not_merge_unrelated_raw_click() {
+ let mut b = EventBatch::with_capacity(4);
+ let raw = click_at(10, 20);
+ let raw_ts = raw.timestamp;
+ b.push(raw, Some(42));
+
+ let mut enriched = click_at(10, 20);
+ enriched.timestamp = raw_ts + ChronoDuration::milliseconds(300);
+ enriched.element_role = Some("AXButton".to_string());
+
+ assert!(!b.try_merge_click_context(&enriched));
+ assert_eq!(b.len(), 1);
+ assert!(b.events[0].element_role.is_none());
+ }
+
#[test]
fn idle_recv_timeout_uses_long_backoff_without_pending_work() {
let batch = EventBatch::with_capacity(0);