YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(activity): bound representative text lookup by frame range (#7111)
Ezra Ellettee7d80df7cee5
diff --git a/crates/screenpipe-engine/src/routes/activity_summary.rs b/crates/screenpipe-engine/src/routes/activity_summary.rsindex 1944f9072..e55759a1d 100644--- a/crates/screenpipe-engine/src/routes/activity_summary.rs+++ b/crates/screenpipe-engine/src/routes/activity_summary.rs@@ -550,6 +550,50 @@ fn resolved_frames_cte(start: &str, end: &str) -> String {)}+fn key_texts_query(resolved_frames_cte: &str, app_filter_f: &str) -> String {+ // One representative text per app+window context. Prefer user input+ // (AXTextArea/AXTextField) over static text, cap at 300 chars to skip+ // marketing copy walls.+ // CROSS JOIN keeps the time-bounded frames outermost even without planner+ // statistics; an ordinary JOIN can scan all historical accessibility text.+ format!(+ "{resolved_frames_cte}, ranked_contexts AS ( \+ SELECT e.text, f.app_name, \+ COALESCE(f.window_name, '') as window_name, \+ f.timestamp, \+ DATE(f.timestamp) AS bucket, \+ ROW_NUMBER() OVER ( \+ PARTITION BY DATE(f.timestamp), f.app_name, f.window_name \+ ORDER BY \+ CASE WHEN e.role IN ('AXTextArea', 'AXTextField') THEN 0 ELSE 1 END, \+ LENGTH(e.text) DESC, \+ f.timestamp DESC \+ ) as rn \+ FROM resolved_frames f \+ CROSS JOIN elements e ON e.frame_id = f.id \+ WHERE 1 = 1{app_filter_f} \+ AND e.text IS NOT NULL \+ AND e.source = 'accessibility' \+ AND LENGTH(e.text) BETWEEN 30 AND 300 \+ AND e.text NOT LIKE 'http%' \+ AND e.text NOT LIKE 'cdn.%' \+ ), balanced AS ( \+ SELECT text, app_name, window_name, timestamp, bucket, \+ ROW_NUMBER() OVER (PARTITION BY bucket ORDER BY timestamp DESC) AS bucket_rank \+ FROM ranked_contexts \+ WHERE rn = 1 \+ ) \+ SELECT text, app_name, window_name, timestamp \+ FROM balanced \+ ORDER BY bucket_rank ASC, timestamp ASC \+ LIMIT 20"+ )+}++#[cfg(test)]+#[path = "activity_summary_key_text_tests.rs"]+mod key_text_tests;+async fn collect_summary_core(db: &DatabaseManager,query: &ActivitySummaryQuery,@@ -715,41 +759,7 @@ async fn collect_summary_core(ORDER BY section_order, row_order");- // One representative text per app+window context. Prefer user input- // (AXTextArea/AXTextField) over static text, cap at 300 chars to skip- // marketing copy walls.- let texts_query = format!(- "{resolved_frames_cte}, ranked_contexts AS ( \- SELECT e.text, f.app_name, \- COALESCE(f.window_name, '') as window_name, \- f.timestamp, \- DATE(f.timestamp) AS bucket, \- ROW_NUMBER() OVER ( \- PARTITION BY DATE(f.timestamp), f.app_name, f.window_name \- ORDER BY \- CASE WHEN e.role IN ('AXTextArea', 'AXTextField') THEN 0 ELSE 1 END, \- LENGTH(e.text) DESC, \- f.timestamp DESC \- ) as rn \- FROM elements e \- JOIN resolved_frames f ON f.id = e.frame_id \- WHERE 1 = 1{app_filter_f} \- AND e.text IS NOT NULL \- AND e.source = 'accessibility' \- AND LENGTH(e.text) BETWEEN 30 AND 300 \- AND e.text NOT LIKE 'http%' \- AND e.text NOT LIKE 'cdn.%' \- ), balanced AS ( \- SELECT text, app_name, window_name, timestamp, bucket, \- ROW_NUMBER() OVER (PARTITION BY bucket ORDER BY timestamp DESC) AS bucket_rank \- FROM ranked_contexts \- WHERE rn = 1 \- ) \- SELECT text, app_name, window_name, timestamp \- FROM balanced \- ORDER BY bucket_rank ASC, timestamp ASC \- LIMIT 20"- );+ let texts_query = key_texts_query(&resolved_frames_cte, &app_filter_f);let audio_speakers_query = format!("SELECT COALESCE(s.name, 'Unknown') as speaker_name, COUNT(*) as segment_count \@@ -1910,7 +1920,7 @@ mod db_tests {/// A throwaway migrated DB. We use a temp FILE rather than `sqlite::memory:`/// because the manager opens a multi-connection pool and each connection to/// `:memory:` is a separate database; a file is shared across the pool.- async fn fresh_db() -> (DatabaseManager, tempfile::TempDir) {+ pub(super) async fn fresh_db() -> (DatabaseManager, tempfile::TempDir) {let dir = tempfile::tempdir().expect("tempdir");let path = dir.path().join("test.db");let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())diff --git a/crates/screenpipe-engine/src/routes/activity_summary_key_text_tests.rs b/crates/screenpipe-engine/src/routes/activity_summary_key_text_tests.rsnew file mode 100644index 000000000..0688cb466--- /dev/null+++ b/crates/screenpipe-engine/src/routes/activity_summary_key_text_tests.rs@@ -0,0 +1,221 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++use super::{key_texts_query, resolved_frames_cte};+use rusqlite::{Connection, StatementStatus};+use std::time::{Duration, Instant};++// Migrate through the real manager, then close its pools/background maintenance.+// A single connection makes planner-statistics state and VM work reproducible.+async fn fixture(history_frames: usize, recent_frames: usize) -> (Connection, tempfile::TempDir) {+ let (db, dir) = super::db_tests::fresh_db().await;+ db.close().await;+ drop(db);+ let conn = Connection::open(dir.path().join("test.db")).unwrap();+ conn.execute_batch(&format!(+ "BEGIN;+ WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x < {history_frames})+ INSERT INTO frames (timestamp, app_name, window_name)+ SELECT datetime('2026-01-01', '+' || (x * 30) || ' seconds'),+ 'OldApp', 'OldWindow' FROM n;+ WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x < {recent_frames})+ INSERT INTO frames (timestamp, app_name, window_name)+ SELECT datetime('2026-06-02', '+' || ((x-1) * 30) || ' seconds'),+ CASE WHEN x % 17 = 0 THEN NULL ELSE 'App' || ((x / 12) % 4) END,+ 'Window' || (x % 12) FROM n;+ INSERT INTO ui_events (timestamp, relative_ms, event_type, app_name, window_title, frame_id)+ SELECT timestamp, 0, 'click', 'Recovered', 'RecoveredWindow', id+ FROM frames WHERE app_name IS NULL;+ WITH slots(n) AS (VALUES(0),(1),(2),(3),(4),(5),(6),(7),(8),(9))+ INSERT INTO elements (frame_id, source, role, text)+ SELECT f.id,+ CASE WHEN n = 9 THEN 'ocr' ELSE 'accessibility' END,+ CASE WHEN n < 2 THEN 'AXTextField' ELSE 'AXStaticText' END,+ printf('Planning context for frame %06d, candidate %d: ', f.id, n)+ || substr('abcdefghij', 1, n)+ FROM frames f CROSS JOIN slots;+ COMMIT;+ ANALYZE;+ DELETE FROM sqlite_stat1;+ DELETE FROM sqlite_stat4;+ ANALYZE sqlite_schema;"+ ))+ .unwrap();+ (conn, dir)+}++fn queries(start: &str, end: &str, app: Option<&str>) -> [String; 2] {+ let filter = app+ .map(|app| format!(" AND f.app_name = '{}'", super::sql_escape(app)))+ .unwrap_or_default();+ let after = key_texts_query(&resolved_frames_cte(start, end), &filter);+ // The complete production statement, changing only the two original join+ // lines from pre-fix main. Ranking, attribution and filters stay identical.+ let before = after.replace(+ "FROM resolved_frames f CROSS JOIN elements e ON e.frame_id = f.id",+ "FROM elements e JOIN resolved_frames f ON f.id = e.frame_id",+ );+ [before, after]+}++type TextRow = (String, Option<String>, String, String);++fn run(conn: &Connection, sql: &str) -> (Vec<TextRow>, i32, Duration) {+ let start = Instant::now();+ let mut statement = conn.prepare(sql).unwrap();+ let rows = statement+ .query_map([], |row| {+ Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))+ })+ .unwrap()+ .collect::<Result<Vec<_>, _>>()+ .unwrap();+ (+ rows,+ statement.get_status(StatementStatus::VmStep),+ start.elapsed(),+ )+}++fn plan(conn: &Connection, sql: &str) -> Vec<String> {+ conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}"))+ .unwrap()+ .query_map([], |row| row.get(3))+ .unwrap()+ .collect::<Result<Vec<_>, _>>()+ .unwrap()+}++#[tokio::test]+async fn key_text_work_is_bounded_by_frames_in_the_requested_range() {+ let (conn, _dir) = fixture(2_000, 120).await;+ for (start, end) in [+ ("2026-06-02 00:00:00", "2026-06-02 00:05:00"),+ ("2026-07-01 00:00:00", "2026-07-02 00:00:00"),+ ] {+ let [before, after] = queries(start, end, None);+ let (expected, before_steps, _) = run(&conn, &before);+ let (actual, after_steps, _) = run(&conn, &after);+ assert_eq!(actual, expected);+ // A work ceiling, not a wall-clock timeout: at most 11 frames and 99+ // accessibility elements here, regardless of the older history.+ assert!(+ after_steps < 20_000,+ "narrow/empty ranges must not scan history: before={before_steps}, after={after_steps}"+ );+ let details = plan(&conn, &after);+ assert!(+ details+ .iter()+ .any(|line| line.contains("SEARCH e USING INDEX")+ && line.contains("frame_id=? AND source=?")),+ "expected per-frame indexed element lookup: {details:?}"+ );+ }+}++#[tokio::test]+async fn key_text_join_preserves_selection_filters_and_recovered_attribution() {+ let (conn, _dir) = fixture(200, 120).await;+ // Invalid candidates must never win even with the preferred input role.+ conn.execute_batch(+ "INSERT INTO elements (frame_id, source, role, text)+ SELECT id, 'accessibility', 'AXTextArea', invalid.text+ FROM frames CROSS JOIN (+ SELECT NULL AS text UNION ALL SELECT 'short'+ UNION ALL SELECT 'https://example.com/this-is-not-representative-text'+ UNION ALL SELECT 'cdn.example.com/this-is-not-representative-text'+ UNION ALL SELECT printf('%0301d', 0)+ ) invalid;",+ )+ .unwrap();+ for analyzed in [false, true] {+ if analyzed {+ conn.execute_batch("ANALYZE").unwrap();+ }+ for app in [None, Some("App1"), Some("Recovered"), Some("Absent'app")] {+ let [before, after] = queries("2026-06-02 00:00:00", "2026-06-03 00:00:00", app);+ let (expected, _, _) = run(&conn, &before);+ let (actual, _, _) = run(&conn, &after);+ assert_eq!(actual, expected, "app={app:?}, analyzed={analyzed}");+ if app == Some("Absent'app") {+ assert!(actual.is_empty());+ } else {+ assert!(!actual.is_empty());+ assert!(actual.len() <= 20);+ for (text, name, window, _) in actual {+ assert!(+ text.contains("candidate 1:"),+ "input/length preference: {text}"+ );+ if let Some(app) = app {+ assert_eq!(name.as_deref(), Some(app));+ }+ if app == Some("Recovered") {+ assert_eq!(window, "RecoveredWindow");+ }+ }+ }+ }+ }+}++/// cargo test -p screenpipe-engine --lib benchmark_activity_summary_key_texts -- --ignored --nocapture+/// SQL-only, synthetic migrated file DB; does not measure HTTP or Windows.+#[tokio::test]+#[ignore = "manual paired benchmark with one million historical elements"]+async fn benchmark_activity_summary_key_texts() {+ let (conn, _dir) = fixture(100_000, 2_880).await;+ eprintln!("SQLite {}, {} {}, 102880 frames, 1028800 elements, 7 alternating samples after one warmup per query",+ rusqlite::version(), std::env::consts::OS, std::env::consts::ARCH);+ for analyzed in [false, true] {+ if analyzed {+ conn.execute_batch("ANALYZE").unwrap();+ }+ for (label, start, end, app) in [+ ("24h", "2026-06-02 00:00:00", "2026-06-03 00:00:00", None),+ ("5m", "2026-06-02 12:00:00", "2026-06-02 12:05:00", None),+ ("empty", "2026-07-01 00:00:00", "2026-07-02 00:00:00", None),+ (+ "24h app filter",+ "2026-06-02 00:00:00",+ "2026-06-03 00:00:00",+ Some("App1"),+ ),+ (+ "all history",+ "2026-01-01 00:00:00",+ "2026-07-01 00:00:00",+ None,+ ),+ ] {+ let sql = queries(start, end, app);+ assert_ne!(sql[0], sql[1], "baseline join replacement must match");+ let expected = run(&conn, &sql[0]).0;+ assert_eq!(run(&conn, &sql[1]).0, expected);+ let mut samples = [Vec::new(), Vec::new()];+ let mut steps = [0, 0];+ for round in 0..7 {+ for index in [round % 2, 1 - round % 2] {+ let (rows, vm_steps, elapsed) = run(&conn, &sql[index]);+ assert_eq!(rows, expected, "{label}, analyzed={analyzed}");+ samples[index].push(elapsed.as_secs_f64() * 1000.0);+ steps[index] = vm_steps;+ }+ }+ for sample in &mut samples {+ sample.sort_by(f64::total_cmp);+ }+ eprintln!("analyzed={analyzed} {label}: rows={} before_ms={:.3} after_ms={:.3} speedup={:.2}x vm_steps={steps:?} samples_ms={samples:?}",+ expected.len(), samples[0][3], samples[1][3], samples[0][3] / samples[1][3]);+ if label == "24h" {+ for (name, sql) in ["before", "after"].into_iter().zip(&sql) {+ eprintln!(+ "analyzed={analyzed} {name} plan:\n{}",+ plan(&conn, sql).join("\n")+ );+ }+ }+ }+ }+}