Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat: expose frame event source and keep metadata coherent (#6230)

Louis Beaumontaaa57cae1e11
diff --git a/crates/screenpipe-db/src/db/accessibility.rs b/crates/screenpipe-db/src/db/accessibility.rs
index 2d7a45b05..36450e898 100644
--- a/crates/screenpipe-db/src/db/accessibility.rs
+++ b/crates/screenpipe-db/src/db/accessibility.rs
@@ -91,7 +91,8 @@ impl DatabaseManager {
COALESCE(vc.file_path, '') as file_path,
COALESCE(f.offset_index, 0) as offset_index,
f.name as frame_name,
- f.browser_url
+ f.browser_url,
+ f.capture_trigger
FROM candidates c
JOIN frames f ON f.id = c.id
LEFT JOIN video_chunks vc ON f.video_chunk_id = vc.id
@@ -217,7 +218,8 @@ impl DatabaseManager {
COALESCE(vc.file_path, '') as file_path,
COALESCE(f.offset_index, 0) as offset_index,
f.name as frame_name,
- f.browser_url
+ f.browser_url,
+ f.capture_trigger
FROM candidates c
JOIN frames f ON f.id = c.id
LEFT JOIN video_chunks vc ON f.video_chunk_id = vc.id
diff --git a/crates/screenpipe-db/src/db/search.rs b/crates/screenpipe-db/src/db/search.rs
index ef1ef8b5d..2aba4e079 100644
--- a/crates/screenpipe-db/src/db/search.rs
+++ b/crates/screenpipe-db/src/db/search.rs
@@ -652,7 +652,8 @@ impl DatabaseManager {
GROUP_CONCAT(tags.name, ',') as tags,
frames.browser_url,
frames.focused,
- frames.text_source
+ frames.text_source,
+ frames.capture_trigger
FROM candidates
JOIN frames ON frames.id = candidates.id
LEFT JOIN video_chunks ON frames.video_chunk_id = video_chunks.id
@@ -702,6 +703,7 @@ impl DatabaseManager {
browser_url: raw.browser_url,
focused: raw.focused,
text_source: raw.text_source,
+ capture_trigger: raw.capture_trigger,
})
.collect()
}
@@ -859,7 +861,8 @@ impl DatabaseManager {
GROUP_CONCAT(tags.name, ',') as tags,
frames.browser_url,
frames.focused,
- frames.text_source
+ frames.text_source,
+ frames.capture_trigger
FROM candidates
JOIN frames ON frames.id = candidates.id
LEFT JOIN video_chunks ON frames.video_chunk_id = video_chunks.id
diff --git a/crates/screenpipe-db/src/types.rs b/crates/screenpipe-db/src/types.rs
index 12a72cc5b..4d154957d 100644
--- a/crates/screenpipe-db/src/types.rs
+++ b/crates/screenpipe-db/src/types.rs
@@ -169,6 +169,8 @@ pub struct OCRResultRaw {
/// or `"ocr"` (fallback for terminals, canvas-rendered apps, weak a11y).
/// `None` for legacy rows captured before text_source was tracked.
pub text_source: Option<String>,
+ /// Why the underlying frame was captured. `None` for legacy rows.
+ pub capture_trigger: Option<String>,
}
#[derive(OaSchema, Debug, Serialize, Deserialize)]
@@ -191,6 +193,8 @@ pub struct OCRResult {
/// `"ocr"` (fallback). `None` for legacy rows. Despite the field name
/// `ocr_text`, the content is accessibility-derived for most captures.
pub text_source: Option<String>,
+ /// Why the underlying frame was captured. `None` for legacy rows.
+ pub capture_trigger: Option<String>,
}
/// Content type for search queries.
@@ -521,6 +525,8 @@ pub struct UiContent {
pub offset_index: i64,
pub frame_name: Option<String>,
pub browser_url: Option<String>,
+ /// Why the underlying frame was captured. `None` for legacy rows.
+ pub capture_trigger: Option<String>,
}
#[derive(OaSchema, Debug, Clone)]
diff --git a/crates/screenpipe-db/tests/on_screen_filter_test.rs b/crates/screenpipe-db/tests/on_screen_filter_test.rs
index 73cb87f8b..4b836f116 100644
--- a/crates/screenpipe-db/tests/on_screen_filter_test.rs
+++ b/crates/screenpipe-db/tests/on_screen_filter_test.rs
@@ -258,7 +258,7 @@ mod tests {
None, // browser_url
None, // document_path
true,
- Some("event"),
+ Some("visual_change"),
Some("alpha-marker first\nalpha-marker second"),
Some("accessibility"),
Some(&tree),
@@ -280,5 +280,6 @@ mod tests {
1,
"GROUP BY f.id must collapse multiple matching elements to one frame row"
);
+ assert_eq!(rows[0].capture_trigger.as_deref(), Some("visual_change"));
}
}
diff --git a/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs b/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs
index d24d0b28d..a27b9fa26 100644
--- a/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs
+++ b/crates/screenpipe-db/tests/search_ocr_snapshot_test.rs
@@ -40,7 +40,7 @@ mod tests {
Some("GitHub"),
None,
true,
- None,
+ Some("typing_pause"),
None,
None,
None,
@@ -99,6 +99,38 @@ mod tests {
ocr.ocr_text
);
assert_eq!(ocr.app_name, "chrome.exe");
+ assert_eq!(ocr.capture_trigger.as_deref(), Some("typing_pause"));
+ }
+ other => panic!("Expected OCR result, got: {:?}", other),
+ }
+
+ // Empty-query browsing uses a separate optimized SQL projection.
+ let browse_results = db
+ .search(
+ "",
+ ContentType::OCR,
+ 100,
+ 0,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ match &browse_results[0] {
+ SearchResult::OCR(ocr) => {
+ assert_eq!(ocr.capture_trigger.as_deref(), Some("typing_pause"));
}
other => panic!("Expected OCR result, got: {:?}", other),
}
diff --git a/crates/screenpipe-engine/src/event_driven_capture.rs b/crates/screenpipe-engine/src/event_driven_capture.rs
index e4a08f838..404e49cc1 100644
--- a/crates/screenpipe-engine/src/event_driven_capture.rs
+++ b/crates/screenpipe-engine/src/event_driven_capture.rs
@@ -2283,7 +2283,7 @@ fn resolve_capture_metadata_with_policy(
Option<String>,
Option<String>,
) {
- let (mut app_name, mut window_name, browser_url, document_path) = match tree_snapshot {
+ let (mut app_name, mut window_name, mut browser_url, mut document_path) = match tree_snapshot {
Some(snap) => (
Some(snap.app_name.clone()),
Some(snap.window_name.clone()),
@@ -2320,9 +2320,15 @@ fn resolve_capture_metadata_with_policy(
} if !trigger_app_name.is_empty() => {
if app_name.as_deref() != Some(trigger_app_name.as_str()) {
debug!(
- "focused app mismatch on app_switch: trigger='{}', tree={:?}; using trigger value",
+ "focused app mismatch on app_switch: trigger='{}', tree={:?}; using trigger value and dropping stale window context",
trigger_app_name, app_name
);
+ // App-switch events typically do not carry the new window
+ // title. Keeping the previous tree's title, URL, or document
+ // path would create a metadata pair that never existed.
+ window_name = None;
+ browser_url = None;
+ document_path = None;
}
app_name = Some(trigger_app_name.clone());
}
@@ -3918,6 +3924,45 @@ mod tests {
));
}
+ #[test]
+ fn resolve_capture_metadata_drops_stale_window_context_on_app_switch() {
+ let snapshot = screenpipe_a11y::tree::TreeSnapshot {
+ app_name: "TextEdit".into(),
+ app_id: Some("com.apple.TextEdit".into()),
+ executable: None,
+ app_version: None,
+ window_name: "M1_B.txt".into(),
+ text_content: "visible text".into(),
+ nodes: Vec::new(),
+ semantic_nodes: Vec::new(),
+ browser_url: Some("https://stale.example".into()),
+ document_path: Some("/tmp/M1_B.txt".into()),
+ timestamp: Utc::now(),
+ node_count: 0,
+ walk_duration: Duration::from_millis(1),
+ content_hash: 0,
+ simhash: 0,
+ truncated: false,
+ truncation_reason: screenpipe_a11y::tree::TruncationReason::None,
+ max_depth_reached: 0,
+ window_bounds: None,
+ };
+
+ let (app_name, window_name, browser_url, document_path) = resolve_capture_metadata(
+ Some(&snapshot),
+ &CaptureTrigger::AppSwitch {
+ app_name: "Finder".into(),
+ target: None,
+ },
+ None,
+ );
+
+ assert_eq!(app_name.as_deref(), Some("Finder"));
+ assert_eq!(window_name, None);
+ assert_eq!(browser_url, None);
+ assert_eq!(document_path, None);
+ }
+
#[test]
fn workflow_checkpoint_trigger_classification() {
// The union still covers everything the old single class did — only
diff --git a/crates/screenpipe-engine/src/routes/content.rs b/crates/screenpipe-engine/src/routes/content.rs
index aa6cd4ded..90ecf886f 100644
--- a/crates/screenpipe-engine/src/routes/content.rs
+++ b/crates/screenpipe-engine/src/routes/content.rs
@@ -140,6 +140,9 @@ pub struct OCRContent {
/// historically called OCR but most captures are accessibility-derived
/// — read this field to know which path produced the text.
pub text_source: Option<String>,
+ /// Why this frame was captured, such as `click`, `typing_pause`, `idle`,
+ /// or `visual_change`. `None` for legacy rows.
+ pub event_source: Option<String>,
}
#[derive(OaSchema, Serialize, Deserialize, Debug, Clone)]
@@ -184,6 +187,8 @@ pub struct UiContent {
pub offset_index: i64,
pub frame_name: Option<String>,
pub browser_url: Option<String>,
+ /// Why this frame was captured. `None` for legacy rows.
+ pub event_source: Option<String>,
}
/// User input event content (clicks, keystrokes, clipboard, etc.)
diff --git a/crates/screenpipe-engine/src/routes/search.rs b/crates/screenpipe-engine/src/routes/search.rs
index e3d01cbf7..2e8ff6f69 100644
--- a/crates/screenpipe-engine/src/routes/search.rs
+++ b/crates/screenpipe-engine/src/routes/search.rs
@@ -623,6 +623,7 @@ pub fn search_result_to_content_item(
focused: ocr.focused,
device_name: ocr.device_name.clone(),
text_source: ocr.text_source.clone(),
+ event_source: ocr.capture_trigger.clone(),
}),
SearchResult::Audio(audio) => {
let transcription = truncate(audio.transcription.clone());
@@ -660,6 +661,7 @@ pub fn search_result_to_content_item(
offset_index: ui.offset_index,
frame_name: ui.frame_name.clone(),
browser_url: ui.browser_url.clone(),
+ event_source: ui.capture_trigger.clone(),
}),
SearchResult::Input(input) => ContentItem::Input(InputContent {
id: input.id,
@@ -1693,6 +1695,7 @@ mod tests {
focused: None,
device_name: "test-device".to_string(),
text_source: Some("ocr".to_string()),
+ event_source: None,
}
}
@@ -1708,6 +1711,7 @@ mod tests {
offset_index: 0,
frame_name: None,
browser_url: None,
+ event_source: None,
}
}
@@ -1822,6 +1826,7 @@ mod tests {
focused: Some(true),
device_name: "test-device".to_string(),
text_source: Some("ocr".to_string()),
+ capture_trigger: Some("typing_pause".to_string()),
})
};
@@ -1830,12 +1835,35 @@ mod tests {
None,
);
let lightweight_projection = search_result_to_content_item(&make_result(""), None);
+ let full_json = serde_json::to_value(full_projection).unwrap();
+ let lightweight_json = serde_json::to_value(lightweight_projection).unwrap();
assert_eq!(
- serde_json::to_value(full_projection).unwrap(),
- serde_json::to_value(lightweight_projection).unwrap(),
+ full_json, lightweight_json,
"the HTTP search shape must not expose OCR bounding-box JSON"
);
+ assert_eq!(full_json["content"]["event_source"], "typing_pause");
+ }
+
+ #[test]
+ fn accessibility_event_source_is_observable_in_search_response() {
+ let result = SearchResult::UI(screenpipe_db::UiContent {
+ id: 42,
+ text: "visible text".to_string(),
+ timestamp: Utc::now(),
+ app_name: "test".to_string(),
+ window_name: "test window".to_string(),
+ initial_traversal_at: None,
+ file_path: "frame.jpg".to_string(),
+ offset_index: 0,
+ frame_name: Some("display-1".to_string()),
+ browser_url: None,
+ capture_trigger: Some("visual_change".to_string()),
+ });
+
+ let json = serde_json::to_value(search_result_to_content_item(&result, None)).unwrap();
+
+ assert_eq!(json["content"]["event_source"], "visual_change");
}
#[test]