YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix native timeline search and stream reliability
Louis Beaumonta4c1161d2c0c
diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.tsindex a5d0f385c..4fe889056 100644--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts@@ -636,6 +636,64 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {);});+ it("clicks a standalone Search thumbnail and opens the overlay native Timeline at that exact frame", async function () {+ const nativeAvailable = await invokeOrThrow<boolean>("native_timeline_is_available");+ if (!nativeAvailable) this.skip();++ await openHomeWindow();+ await invokeOrThrow("open_search_window", {+ query: null,+ timelineOrigin: null,+ });+ await browser.waitUntil(+ async () => (await browser.getWindowHandles()).includes("search"),+ { timeout: t(20_000), timeoutMsg: "standalone Search window did not open" },+ );+ await browser.switchToWindow("search");++ // This is the production path used by the global Search button: unlike a+ // Search launched from inside Timeline, it has no timelineOrigin. The old+ // fallback opened the overlay at its live edge and sent only a React event,+ // which the native AppKit Timeline never receives.+ const searchQuery = "vector search result number 5";+ const searchResult = await readySearchThumbnail(searchQuery);+ await $(searchResult.selector).click();+ const expectedFrameId = searchResult.frameId;++ const overlayLabel = await browser.waitUntil(+ async () => {+ const handles = await browser.getWindowHandles();+ return handles.find((handle) => handle === "main" || handle === "main-window") ?? false;+ },+ {+ timeout: t(20_000),+ interval: 200,+ timeoutMsg: "standalone Search did not open the overlay Timeline",+ },+ );+ await browser.switchToWindow(overlayLabel as string);+ await browser.waitUntil(+ async () => {+ const state = await nativeTimelineSearchState(overlayLabel as string);+ return state?.attached === true &&+ state.currentFrameId === expectedFrameId &&+ state.displayedFrameId === expectedFrameId &&+ state.loadedImageFrameId === expectedFrameId &&+ Math.abs(+ new Date(state.currentTimestamp ?? 0).getTime() -+ new Date(searchResult.timestamp).getTime(),+ ) < 2 &&+ state.searchQuery === searchQuery &&+ state.searchFrameIds.includes(expectedFrameId);+ },+ {+ timeout: t(30_000),+ interval: 200,+ timeoutMsg: `standalone Search did not move the native Timeline to frame ${expectedFrameId}`,+ },+ );+ });+it("clicks a real Search thumbnail and restores the overlay native Timeline at that exact frame", async function () {const nativeAvailable = await invokeOrThrow<boolean>("native_timeline_is_available");if (!nativeAvailable) this.skip();diff --git a/apps/screenpipe-app-tauri/src-tauri/src/commands.rs b/apps/screenpipe-app-tauri/src-tauri/src/commands.rsindex 7c6570d7e..a7376ce35 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/commands.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/commands.rs@@ -2352,9 +2352,9 @@ mod search_navigation_origin_tests {/// Navigate from Search to the timeline that opened it.///-/// Native Swift timelines are addressed by their host-window label. Searches-/// opened elsewhere retain the legacy React event so non-macOS and older-/// surfaces continue to work.+/// Native Swift timelines are addressed by their host-window label. Global+/// Search defaults to the overlay's `main` label on macOS; non-macOS surfaces+/// retain the legacy React event.#[tauri::command]#[specta::specta]pub async fn search_navigate_to_timeline(@@ -2369,52 +2369,56 @@ pub async fn search_navigate_to_timeline(let timeline_origin = validated_timeline_origin(timeline_origin.as_deref())?;#[cfg(target_os = "macos")]- if let Some(origin) = timeline_origin {- if crate::native_timeline::is_available() {- match origin {- "home" => ShowRewindWindow::Home { page: None }- .show(&app_handle)- .map_err(|e| e.to_string())?,- "main" | "main-window" => ShowRewindWindow::Main- .show(&app_handle)- .map_err(|e| e.to_string())?,- _ => unreachable!("timeline origin was validated"),- };+ if crate::native_timeline::is_available() {+ // Global Search has no timeline origin. On macOS its destination is+ // still the native Timeline attached to the overlay, not the hidden+ // React fallback inside that overlay. The old origin-less branch only+ // emitted `search-navigate-to-timestamp`, so the visible native child+ // opened at the live edge and never received the clicked frame.+ let origin = timeline_origin.unwrap_or("main");+ match origin {+ "home" => ShowRewindWindow::Home { page: None }+ .show(&app_handle)+ .map_err(|e| e.to_string())?,+ "main" | "main-window" => ShowRewindWindow::Main+ .show(&app_handle)+ .map_err(|e| e.to_string())?,+ _ => unreachable!("timeline origin was validated"),+ };- // The source timeline normally remains attached while Search is- // visible. Retry as well so a restored overlay can finish mounting- // its native child before the hand-off arrives.- let _ = crate::native_timeline::navigate_to_search_result(- ×tamp,- frame_id,- origin,- search_terms.as_deref(),- search_results_json.as_deref(),- search_query.as_deref(),- );+ // The source timeline normally remains attached while Search is+ // visible. Retry as well so a restored or newly-created overlay can+ // finish mounting its native child before the hand-off arrives.+ let _ = crate::native_timeline::navigate_to_search_result(+ ×tamp,+ frame_id,+ origin,+ search_terms.as_deref(),+ search_results_json.as_deref(),+ search_query.as_deref(),+ );- let app = app_handle.clone();- let timestamp_retry = timestamp.clone();- let origin_retry = origin.to_string();- let search_terms_retry = search_terms.clone();- let search_results_retry = search_results_json.clone();- let search_query_retry = search_query.clone();- tokio::spawn(async move {- for _ in 0..4 {- tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;- let _ = crate::native_timeline::navigate_to_search_result(- ×tamp_retry,- frame_id,- &origin_retry,- search_terms_retry.as_deref(),- search_results_retry.as_deref(),- search_query_retry.as_deref(),- );- }- let _ = ShowRewindWindow::Search { query: None }.close(&app);- });- return Ok(());- }+ let app = app_handle.clone();+ let timestamp_retry = timestamp.clone();+ let origin_retry = origin.to_string();+ let search_terms_retry = search_terms.clone();+ let search_results_retry = search_results_json.clone();+ let search_query_retry = search_query.clone();+ tokio::spawn(async move {+ for _ in 0..4 {+ tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;+ let _ = crate::native_timeline::navigate_to_search_result(+ ×tamp_retry,+ frame_id,+ &origin_retry,+ search_terms_retry.as_deref(),+ search_results_retry.as_deref(),+ search_query_retry.as_deref(),+ );+ }+ let _ = ShowRewindWindow::Search { query: None }.close(&app);+ });+ return Ok(());}// Show the Main timelinediff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swiftindex 3ee9ab619..f57a49ce8 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift@@ -308,6 +308,14 @@ final class FrameStreamClient: NSObject {switch result {case .failure(let error):guard !self.stopped else { return }+ let nsError = error as NSError+ NSLog(+ "[native-timeline] frame stream receive failed domain=%@ code=%ld description=%@ reason=%@",+ nsError.domain,+ nsError.code,+ nsError.localizedDescription,+ nsError.localizedFailureReason ?? "none"+ )self.delegate?.frameStream(didFail: error.localizedDescription)self.scheduleReconnect()case .success(let message):@@ -336,8 +344,13 @@ final class FrameStreamClient: NSObject {delegate?.frameStream(didReceive: update)case .some(.serverError(let message)):delegate?.frameStream(didFail: message)- case .some(.keepAlive), .none:+ case .some(.keepAlive):break+ case .none:+ NSLog(+ "[native-timeline] ignored undecodable frame stream message bytes=%ld",+ text.utf8.count+ )}}diff --git a/crates/screenpipe-engine/src/routes/streaming.rs b/crates/screenpipe-engine/src/routes/streaming.rsindex c0ad952a1..bdddc01c1 100644--- a/crates/screenpipe-engine/src/routes/streaming.rs+++ b/crates/screenpipe-engine/src/routes/streaming.rs@@ -47,6 +47,11 @@ pub struct StreamFramesRequest {const MAX_STREAM_FRAME_LIMIT: usize = 10_000;const DEFAULT_STREAM_FRAME_LIMIT: usize = MAX_STREAM_FRAME_LIMIT;const STREAM_BATCH_CAPACITY: usize = 100;+/// Safari/Chromium accept very large WebSocket messages, but native clients+/// commonly enforce a much smaller receive ceiling. Production transcript and+/// screen-text payloads can make 100 otherwise-small timeline rows exceed that+/// ceiling, so the wire batch must be bounded by encoded bytes as well as rows.+const MAX_STREAM_BATCH_BYTES: usize = 8 * 1024 * 1024;const STREAM_BATCH_FLUSH_DELAY: Duration = Duration::from_millis(100);fn stream_frame_limit(requested: Option<usize>) -> usize {@@ -828,14 +833,52 @@ async fn send_batch(sender: &mut futures::stream::SplitSink<WebSocket, Message>,buffer: &mut Vec<StreamTimeSeriesResponse>,) -> Result<(), Box<dyn std::error::Error>> {+ for json in encode_stream_batches(buffer, MAX_STREAM_BATCH_BYTES)? {+ sender.send(Message::Text(json)).await?;+ }+ Ok(())+}++/// Serialize buffered rows into independently decodable JSON arrays while+/// keeping each WebSocket message below `max_bytes` whenever one row itself+/// fits. The caller owns retrying the request if the socket closes mid-send.+fn encode_stream_batches(+ buffer: &mut Vec<StreamTimeSeriesResponse>,+ max_bytes: usize,+) -> Result<Vec<String>, serde_json::Error> {if buffer.is_empty() {- return Ok(());+ return Ok(Vec::new());}- let json = serde_json::to_string(&buffer)?;- sender.send(Message::Text(json)).await?;- buffer.clear();- Ok(())+ let rows = std::mem::take(buffer);+ let mut messages = Vec::new();+ let mut message = String::from("[");+ let mut row_count = 0usize;++ for row in rows {+ let encoded = serde_json::to_string(&row)?;+ let separator_bytes = usize::from(row_count > 0);+ let next_size = message.len() + separator_bytes + encoded.len() + 1;+ if row_count > 0 && next_size > max_bytes {+ message.push(']');+ messages.push(message);+ message = String::from("[");+ row_count = 0;+ }++ if row_count > 0 {+ message.push(',');+ }+ message.push_str(&encoded);+ row_count += 1;+ }++ if row_count > 0 {+ message.push(']');+ messages.push(message);+ }++ Ok(messages)}#[cfg(test)]@@ -888,6 +931,54 @@ mod tests {}}+ #[test]+ fn websocket_batches_are_split_by_encoded_size() {+ let mut responses = Vec::new();+ for frame_id in 0..24 {+ let mut response = StreamTimeSeriesResponse::from(create_time_series_frame(+ create_test_frame_data(1, 0),+ ));+ response.devices[0].frame_id = frame_id;+ // `text` and its legacy `ocr_text` alias are both sent today. This+ // deliberately makes the original 100-row batch larger than a+ // native WebSocket receive ceiling without needing private data.+ let payload = "x".repeat(512 * 1024);+ response.devices[0].metadata.text = payload.clone();+ response.devices[0].metadata.ocr_text = payload;+ responses.push(response);+ }++ let messages = encode_stream_batches(&mut responses, MAX_STREAM_BATCH_BYTES).unwrap();+ assert!(+ responses.is_empty(),+ "encoded rows must drain the pending buffer"+ );+ assert!(+ messages.len() > 1,+ "oversized production batch must be split"+ );+ assert!(+ messages+ .iter()+ .all(|message| message.len() <= MAX_STREAM_BATCH_BYTES),+ "every multi-row WebSocket message must fit the native receive ceiling"+ );+ let decoded_rows: usize = messages+ .iter()+ .map(|message| {+ serde_json::from_str::<serde_json::Value>(message)+ .unwrap()+ .as_array()+ .unwrap()+ .len()+ })+ .sum();+ assert_eq!(+ decoded_rows, 24,+ "splitting must not drop or duplicate rows"+ );+ }+/// TEST: Demonstrate and verify the audio duplication bug is fixed/// With the fix, audio should appear only once, not duplicated per OCR entry#[test]diff --git a/crates/screenpipe-engine/tests/stream_frames_test.rs b/crates/screenpipe-engine/tests/stream_frames_test.rsindex 33f4dc19d..4b2f612e6 100644--- a/crates/screenpipe-engine/tests/stream_frames_test.rs+++ b/crates/screenpipe-engine/tests/stream_frames_test.rs@@ -22,7 +22,7 @@ struct StreamTimeSeriesResponse {mod tests {use super::*;use screenpipe_audio::audio_manager::AudioManagerBuilder;- use screenpipe_db::{DatabaseManager, OcrEngine};+ use screenpipe_db::{AudioDevice, DatabaseManager, DeviceType, FrameWindowData, OcrEngine};use screenpipe_engine::SCServer;use std::{net::SocketAddr, sync::Arc};@@ -166,6 +166,118 @@ mod tests {assert_eq!(last, start);}+ #[tokio::test]+ async fn test_large_production_stream_is_split_for_native_clients() {+ let (url, db, server_handle) = setup_stream_test_server().await;+ let device_name = "large-native-stream-monitor";+ db.insert_video_chunk("large-native-stream.mp4", device_name)+ .await+ .unwrap();++ let start = DateTime::parse_from_rfc3339("2026-06-29T00:00:00Z")+ .unwrap()+ .with_timezone(&Utc);+ let total_frames = 24usize;+ let seeded_frames: Vec<_> = (0..total_frames)+ .map(|idx| {+ (+ start + Duration::seconds(idx as i64),+ idx as i64,+ vec![FrameWindowData {+ app_name: Some("Large Capture".to_string()),+ window_name: Some(format!("Large window {idx}")),+ browser_url: None,+ focused: true,+ text: format!("native timeline frame {idx}"),+ text_json: "[]".to_string(),+ }],+ )+ })+ .collect();+ db.insert_multi_frames_with_ocr_batch(+ device_name,+ &seeded_frames,+ Arc::new(OcrEngine::Tesseract),+ )+ .await+ .unwrap();+ let audio_chunk_id = db+ .insert_audio_chunk("large-native-stream.wav", Some(start))+ .await+ .unwrap();+ db.insert_audio_transcription(+ audio_chunk_id,+ &"spoken words ".repeat(350_000),+ 0,+ "e2e",+ &AudioDevice {+ name: "large-native-stream-microphone".to_string(),+ device_type: DeviceType::Input,+ },+ None,+ Some(0.0),+ Some(total_frames as f64),+ Some(start),+ )+ .await+ .unwrap();++ let end = start + Duration::seconds(total_frames as i64 - 1);+ let (ws_stream, _) = tokio_tungstenite::connect_async(&url)+ .await+ .expect("websocket should connect");+ let (mut write, mut read) = ws_stream.split();+ write+ .send(Message::Text(+ serde_json::to_string(&StreamFramesLimitedRequest {+ start_time: start.to_rfc3339(),+ end_time: end.to_rfc3339(),+ order: "descending".to_string(),+ limit: total_frames,+ })+ .unwrap(),+ ))+ .await+ .expect("request should send");++ let (received_frames, message_sizes) = timeout(std::time::Duration::from_secs(30), async {+ let mut received = Vec::with_capacity(total_frames);+ let mut sizes = Vec::new();+ while received.len() < total_frames {+ let msg = read+ .next()+ .await+ .expect("websocket should stay open")+ .expect("message should read");+ let Message::Text(text) = msg else {+ continue;+ };+ if text == "\"keep-alive-text\"" {+ continue;+ }+ sizes.push(text.len());+ let mut batch: Vec<StreamTimeSeriesResponse> =+ serde_json::from_str(&text).expect("response batch should parse");+ received.append(&mut batch);+ }+ (received, sizes)+ })+ .await+ .expect("large production stream should remain connected");++ server_handle.abort();++ assert_eq!(received_frames.len(), total_frames);+ assert!(+ message_sizes.len() > 1,+ "a >64 MiB logical response must use multiple WebSocket messages"+ );+ assert!(+ message_sizes.iter().all(|size| *size <= 8 * 1024 * 1024),+ "every message must fit the native client's bounded receive window"+ );+ }+/// TEST 1: Reproduce the main issue - new frames after initial fetch are not pushed////// This test verifies the bug where: