YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat: expose frame event source and keep metadata coherent (#6230)
Louis Beaumontaaa57cae1e11
diff --git a/crates/screenpipe-db/src/db/accessibility.rs b/crates/screenpipe-db/src/db/accessibility.rsindex 2d7a45b05..36450e898 100644--- a/crates/screenpipe-db/src/db/accessibility.rs+++ b/crates/screenpipe-db/src/db/accessibility.rs@@ -91,7 +91,8 @@ impl DatabaseManager {COALESCE(vc.file_path, '') as file_path,COALESCE(f.offset_index, 0) as offset_index,f.name as frame_name,- f.browser_url+ f.browser_url,+ f.capture_triggerFROM candidates cJOIN frames f ON f.id = c.idLEFT JOIN video_chunks vc ON f.video_chunk_id = vc.id@@ -217,7 +218,8 @@ impl DatabaseManager {COALESCE(vc.file_path, '') as file_path,COALESCE(f.offset_index, 0) as offset_index,f.name as frame_name,- f.browser_url+ f.browser_url,+ f.capture_triggerFROM candidates cJOIN frames f ON f.id = c.idLEFT JOIN video_chunks vc ON f.video_chunk_id = vc.iddiff --git a/crates/screenpipe-db/src/db/search.rs b/crates/screenpipe-db/src/db/search.rsindex ef1ef8b5d..2aba4e079 100644--- a/crates/screenpipe-db/src/db/search.rs+++ b/crates/screenpipe-db/src/db/search.rs@@ -652,7 +652,8 @@ impl DatabaseManager {GROUP_CONCAT(tags.name, ',') as tags,frames.browser_url,frames.focused,- frames.text_source+ frames.text_source,+ frames.capture_triggerFROM candidatesJOIN frames ON frames.id = candidates.idLEFT JOIN video_chunks ON frames.video_chunk_id = video_chunks.id@@ -702,6 +703,7 @@ impl DatabaseManager {browser_url: raw.browser_url,focused: raw.focused,text_source: raw.text_source,+ capture_trigger: raw.capture_trigger,}).collect()}@@ -859,7 +861,8 @@ impl DatabaseManager {GROUP_CONCAT(tags.name, ',') as tags,frames.browser_url,frames.focused,- frames.text_source+ frames.text_source,+ frames.capture_triggerFROM candidatesJOIN frames ON frames.id = candidates.idLEFT JOIN video_chunks ON frames.video_chunk_id = video_chunks.iddiff --git a/crates/screenpipe-db/src/types.rs b/crates/screenpipe-db/src/types.rsindex 12a72cc5b..4d154957d 100644--- a/crates/screenpipe-db/src/types.rs+++ b/crates/screenpipe-db/src/types.rs@@ -169,6 +169,8 @@ pub struct OCRResultRaw {/// or `"ocr"` (fallback for terminals, canvas-rendered apps, weak a11y)./// `None` for legacy rows captured before text_source was tracked.pub text_source: Option<String>,+ /// Why the underlying frame was captured. `None` for legacy rows.+ pub capture_trigger: Option<String>,}#[derive(OaSchema, Debug, Serialize, Deserialize)]@@ -191,6 +193,8 @@ pub struct OCRResult {/// `"ocr"` (fallback). `None` for legacy rows. Despite the field name/// `ocr_text`, the content is accessibility-derived for most captures.pub text_source: Option<String>,+ /// Why the underlying frame was captured. `None` for legacy rows.+ pub capture_trigger: Option<String>,}/// Content type for search queries.@@ -521,6 +525,8 @@ pub struct UiContent {pub offset_index: i64,pub frame_name: Option<String>,pub browser_url: Option<String>,+ /// Why the underlying frame was captured. `None` for legacy rows.+ pub capture_trigger: Option<String>,}#[derive(OaSchema, Debug, Clone)]diff --git a/crates/screenpipe-db/tests/on_screen_filter_test.rs b/crates/screenpipe-db/tests/on_screen_filter_test.rsindex 73cb87f8b..4b836f116 100644--- a/crates/screenpipe-db/tests/on_screen_filter_test.rs+++ b/crates/screenpipe-db/tests/on_screen_filter_test.rs@@ -258,7 +258,7 @@ mod tests {None, // browser_urlNone, // document_pathtrue,- Some("event"),+ Some("visual_change"),Some("alpha-marker first\nalpha-marker second"),Some("accessibility"),Some(&tree),@@ -280,5 +280,6 @@ mod tests {1,"GROUP BY f.id must collapse multiple matching elements to one frame row");+ assert_eq!(rows[0].capture_trigger.as_deref(), Some("visual_change"));}}diff --git a/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs b/crates/screenpipe-db/tests/search_ocr_snapshot_test.rsindex d24d0b28d..a27b9fa26 100644--- a/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs+++ b/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs@@ -40,7 +40,7 @@ mod tests {Some("GitHub"),None,true,- None,+ Some("typing_pause"),None,None,None,@@ -99,6 +99,38 @@ mod tests {ocr.ocr_text);assert_eq!(ocr.app_name, "chrome.exe");+ assert_eq!(ocr.capture_trigger.as_deref(), Some("typing_pause"));+ }+ other => panic!("Expected OCR result, got: {:?}", other),+ }++ // Empty-query browsing uses a separate optimized SQL projection.+ let browse_results = db+ .search(+ "",+ ContentType::OCR,+ 100,+ 0,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ )+ .await+ .unwrap();+ match &browse_results[0] {+ SearchResult::OCR(ocr) => {+ assert_eq!(ocr.capture_trigger.as_deref(), Some("typing_pause"));}other => panic!("Expected OCR result, got: {:?}", other),}diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rsindex e4a08f838..404e49cc1 100644--- a/crates/screenpipe-engine/src/event_driven_capture.rs+++ b/crates/screenpipe-engine/src/event_driven_capture.rs@@ -2283,7 +2283,7 @@ fn resolve_capture_metadata_with_policy(Option<String>,Option<String>,) {- let (mut app_name, mut window_name, browser_url, document_path) = match tree_snapshot {+ let (mut app_name, mut window_name, mut browser_url, mut document_path) = match tree_snapshot {Some(snap) => (Some(snap.app_name.clone()),Some(snap.window_name.clone()),@@ -2320,9 +2320,15 @@ fn resolve_capture_metadata_with_policy(} if !trigger_app_name.is_empty() => {if app_name.as_deref() != Some(trigger_app_name.as_str()) {debug!(- "focused app mismatch on app_switch: trigger='{}', tree={:?}; using trigger value",+ "focused app mismatch on app_switch: trigger='{}', tree={:?}; using trigger value and dropping stale window context",trigger_app_name, app_name);+ // App-switch events typically do not carry the new window+ // title. Keeping the previous tree's title, URL, or document+ // path would create a metadata pair that never existed.+ window_name = None;+ browser_url = None;+ document_path = None;}app_name = Some(trigger_app_name.clone());}@@ -3918,6 +3924,45 @@ mod tests {));}+ #[test]+ fn resolve_capture_metadata_drops_stale_window_context_on_app_switch() {+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {+ app_name: "TextEdit".into(),+ app_id: Some("com.apple.TextEdit".into()),+ executable: None,+ app_version: None,+ window_name: "M1_B.txt".into(),+ text_content: "visible text".into(),+ nodes: Vec::new(),+ semantic_nodes: Vec::new(),+ browser_url: Some("https://stale.example".into()),+ document_path: Some("/tmp/M1_B.txt".into()),+ timestamp: Utc::now(),+ node_count: 0,+ walk_duration: Duration::from_millis(1),+ content_hash: 0,+ simhash: 0,+ truncated: false,+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,+ max_depth_reached: 0,+ window_bounds: None,+ };++ let (app_name, window_name, browser_url, document_path) = resolve_capture_metadata(+ Some(&snapshot),+ &CaptureTrigger::AppSwitch {+ app_name: "Finder".into(),+ target: None,+ },+ None,+ );++ assert_eq!(app_name.as_deref(), Some("Finder"));+ assert_eq!(window_name, None);+ assert_eq!(browser_url, None);+ assert_eq!(document_path, None);+ }+#[test]fn workflow_checkpoint_trigger_classification() {// The union still covers everything the old single class did — onlydiff --git a/crates/screenpipe-engine/src/routes/content.rs b/crates/screenpipe-engine/src/routes/content.rsindex aa6cd4ded..90ecf886f 100644--- a/crates/screenpipe-engine/src/routes/content.rs+++ b/crates/screenpipe-engine/src/routes/content.rs@@ -140,6 +140,9 @@ pub struct OCRContent {/// historically called OCR but most captures are accessibility-derived/// — read this field to know which path produced the text.pub text_source: Option<String>,+ /// Why this frame was captured, such as `click`, `typing_pause`, `idle`,+ /// or `visual_change`. `None` for legacy rows.+ pub event_source: Option<String>,}#[derive(OaSchema, Serialize, Deserialize, Debug, Clone)]@@ -184,6 +187,8 @@ pub struct UiContent {pub offset_index: i64,pub frame_name: Option<String>,pub browser_url: Option<String>,+ /// Why this frame was captured. `None` for legacy rows.+ pub event_source: Option<String>,}/// User input event content (clicks, keystrokes, clipboard, etc.)diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rsindex e3d01cbf7..2e8ff6f69 100644--- a/crates/screenpipe-engine/src/routes/search.rs+++ b/crates/screenpipe-engine/src/routes/search.rs@@ -623,6 +623,7 @@ pub fn search_result_to_content_item(focused: ocr.focused,device_name: ocr.device_name.clone(),text_source: ocr.text_source.clone(),+ event_source: ocr.capture_trigger.clone(),}),SearchResult::Audio(audio) => {let transcription = truncate(audio.transcription.clone());@@ -660,6 +661,7 @@ pub fn search_result_to_content_item(offset_index: ui.offset_index,frame_name: ui.frame_name.clone(),browser_url: ui.browser_url.clone(),+ event_source: ui.capture_trigger.clone(),}),SearchResult::Input(input) => ContentItem::Input(InputContent {id: input.id,@@ -1693,6 +1695,7 @@ mod tests {focused: None,device_name: "test-device".to_string(),text_source: Some("ocr".to_string()),+ event_source: None,}}@@ -1708,6 +1711,7 @@ mod tests {offset_index: 0,frame_name: None,browser_url: None,+ event_source: None,}}@@ -1822,6 +1826,7 @@ mod tests {focused: Some(true),device_name: "test-device".to_string(),text_source: Some("ocr".to_string()),+ capture_trigger: Some("typing_pause".to_string()),})};@@ -1830,12 +1835,35 @@ mod tests {None,);let lightweight_projection = search_result_to_content_item(&make_result(""), None);+ let full_json = serde_json::to_value(full_projection).unwrap();+ let lightweight_json = serde_json::to_value(lightweight_projection).unwrap();assert_eq!(- serde_json::to_value(full_projection).unwrap(),- serde_json::to_value(lightweight_projection).unwrap(),+ full_json, lightweight_json,"the HTTP search shape must not expose OCR bounding-box JSON");+ assert_eq!(full_json["content"]["event_source"], "typing_pause");+ }++ #[test]+ fn accessibility_event_source_is_observable_in_search_response() {+ let result = SearchResult::UI(screenpipe_db::UiContent {+ id: 42,+ text: "visible text".to_string(),+ timestamp: Utc::now(),+ app_name: "test".to_string(),+ window_name: "test window".to_string(),+ initial_traversal_at: None,+ file_path: "frame.jpg".to_string(),+ offset_index: 0,+ frame_name: Some("display-1".to_string()),+ browser_url: None,+ capture_trigger: Some("visual_change".to_string()),+ });++ let json = serde_json::to_value(search_result_to_content_item(&result, None)).unwrap();++ assert_eq!(json["content"]["event_source"], "visual_change");}#[test]