Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix native timeline search and stream reliability

Louis Beaumonta4c1161d2c0c
diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
index a5d0f385c..4fe889056 100644
--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
@@ -636,6 +636,64 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {
);
});
+ it("clicks a standalone Search thumbnail and opens the overlay native Timeline at that exact frame", async function () {
+ const nativeAvailable = await invokeOrThrow<boolean>("native_timeline_is_available");
+ if (!nativeAvailable) this.skip();
+
+ await openHomeWindow();
+ await invokeOrThrow("open_search_window", {
+ query: null,
+ timelineOrigin: null,
+ });
+ await browser.waitUntil(
+ async () => (await browser.getWindowHandles()).includes("search"),
+ { timeout: t(20_000), timeoutMsg: "standalone Search window did not open" },
+ );
+ await browser.switchToWindow("search");
+
+ // This is the production path used by the global Search button: unlike a
+ // Search launched from inside Timeline, it has no timelineOrigin. The old
+ // fallback opened the overlay at its live edge and sent only a React event,
+ // which the native AppKit Timeline never receives.
+ const searchQuery = "vector search result number 5";
+ const searchResult = await readySearchThumbnail(searchQuery);
+ await $(searchResult.selector).click();
+ const expectedFrameId = searchResult.frameId;
+
+ const overlayLabel = await browser.waitUntil(
+ async () => {
+ const handles = await browser.getWindowHandles();
+ return handles.find((handle) => handle === "main" || handle === "main-window") ?? false;
+ },
+ {
+ timeout: t(20_000),
+ interval: 200,
+ timeoutMsg: "standalone Search did not open the overlay Timeline",
+ },
+ );
+ await browser.switchToWindow(overlayLabel as string);
+ await browser.waitUntil(
+ async () => {
+ const state = await nativeTimelineSearchState(overlayLabel as string);
+ return state?.attached === true &&
+ state.currentFrameId === expectedFrameId &&
+ state.displayedFrameId === expectedFrameId &&
+ state.loadedImageFrameId === expectedFrameId &&
+ Math.abs(
+ new Date(state.currentTimestamp ?? 0).getTime() -
+ new Date(searchResult.timestamp).getTime(),
+ ) < 2 &&
+ state.searchQuery === searchQuery &&
+ state.searchFrameIds.includes(expectedFrameId);
+ },
+ {
+ timeout: t(30_000),
+ interval: 200,
+ timeoutMsg: `standalone Search did not move the native Timeline to frame ${expectedFrameId}`,
+ },
+ );
+ });
+
it("clicks a real Search thumbnail and restores the overlay native Timeline at that exact frame", async function () {
const nativeAvailable = await invokeOrThrow<boolean>("native_timeline_is_available");
if (!nativeAvailable) this.skip();
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/commands.rs b/apps/screenpipe-app-tauri/src-tauri/src/commands.rs
index 7c6570d7e..a7376ce35 100644
--- a/apps/screenpipe-app-tauri/src-tauri/src/commands.rs
+++ b/apps/screenpipe-app-tauri/src-tauri/src/commands.rs
@@ -2352,9 +2352,9 @@ mod search_navigation_origin_tests {
/// Navigate from Search to the timeline that opened it.
///
-/// Native Swift timelines are addressed by their host-window label. Searches
-/// opened elsewhere retain the legacy React event so non-macOS and older
-/// surfaces continue to work.
+/// Native Swift timelines are addressed by their host-window label. Global
+/// Search defaults to the overlay's `main` label on macOS; non-macOS surfaces
+/// retain the legacy React event.
#[tauri::command]
#[specta::specta]
pub async fn search_navigate_to_timeline(
@@ -2369,52 +2369,56 @@ pub async fn search_navigate_to_timeline(
let timeline_origin = validated_timeline_origin(timeline_origin.as_deref())?;
#[cfg(target_os = "macos")]
- if let Some(origin) = timeline_origin {
- if crate::native_timeline::is_available() {
- match origin {
- "home" => ShowRewindWindow::Home { page: None }
- .show(&app_handle)
- .map_err(|e| e.to_string())?,
- "main" | "main-window" => ShowRewindWindow::Main
- .show(&app_handle)
- .map_err(|e| e.to_string())?,
- _ => unreachable!("timeline origin was validated"),
- };
+ if crate::native_timeline::is_available() {
+ // Global Search has no timeline origin. On macOS its destination is
+ // still the native Timeline attached to the overlay, not the hidden
+ // React fallback inside that overlay. The old origin-less branch only
+ // emitted `search-navigate-to-timestamp`, so the visible native child
+ // opened at the live edge and never received the clicked frame.
+ let origin = timeline_origin.unwrap_or("main");
+ match origin {
+ "home" => ShowRewindWindow::Home { page: None }
+ .show(&app_handle)
+ .map_err(|e| e.to_string())?,
+ "main" | "main-window" => ShowRewindWindow::Main
+ .show(&app_handle)
+ .map_err(|e| e.to_string())?,
+ _ => unreachable!("timeline origin was validated"),
+ };
- // The source timeline normally remains attached while Search is
- // visible. Retry as well so a restored overlay can finish mounting
- // its native child before the hand-off arrives.
- let _ = crate::native_timeline::navigate_to_search_result(
- &timestamp,
- frame_id,
- origin,
- search_terms.as_deref(),
- search_results_json.as_deref(),
- search_query.as_deref(),
- );
+ // The source timeline normally remains attached while Search is
+ // visible. Retry as well so a restored or newly-created overlay can
+ // finish mounting its native child before the hand-off arrives.
+ let _ = crate::native_timeline::navigate_to_search_result(
+ &timestamp,
+ frame_id,
+ origin,
+ search_terms.as_deref(),
+ search_results_json.as_deref(),
+ search_query.as_deref(),
+ );
- let app = app_handle.clone();
- let timestamp_retry = timestamp.clone();
- let origin_retry = origin.to_string();
- let search_terms_retry = search_terms.clone();
- let search_results_retry = search_results_json.clone();
- let search_query_retry = search_query.clone();
- tokio::spawn(async move {
- for _ in 0..4 {
- tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
- let _ = crate::native_timeline::navigate_to_search_result(
- &timestamp_retry,
- frame_id,
- &origin_retry,
- search_terms_retry.as_deref(),
- search_results_retry.as_deref(),
- search_query_retry.as_deref(),
- );
- }
- let _ = ShowRewindWindow::Search { query: None }.close(&app);
- });
- return Ok(());
- }
+ let app = app_handle.clone();
+ let timestamp_retry = timestamp.clone();
+ let origin_retry = origin.to_string();
+ let search_terms_retry = search_terms.clone();
+ let search_results_retry = search_results_json.clone();
+ let search_query_retry = search_query.clone();
+ tokio::spawn(async move {
+ for _ in 0..4 {
+ tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
+ let _ = crate::native_timeline::navigate_to_search_result(
+ &timestamp_retry,
+ frame_id,
+ &origin_retry,
+ search_terms_retry.as_deref(),
+ search_results_retry.as_deref(),
+ search_query_retry.as_deref(),
+ );
+ }
+ let _ = ShowRewindWindow::Search { query: None }.close(&app);
+ });
+ return Ok(());
}
// Show the Main timeline
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
index 3ee9ab619..f57a49ce8 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
@@ -308,6 +308,14 @@ final class FrameStreamClient: NSObject {
switch result {
case .failure(let error):
guard !self.stopped else { return }
+ let nsError = error as NSError
+ NSLog(
+ "[native-timeline] frame stream receive failed domain=%@ code=%ld description=%@ reason=%@",
+ nsError.domain,
+ nsError.code,
+ nsError.localizedDescription,
+ nsError.localizedFailureReason ?? "none"
+ )
self.delegate?.frameStream(didFail: error.localizedDescription)
self.scheduleReconnect()
case .success(let message):
@@ -336,8 +344,13 @@ final class FrameStreamClient: NSObject {
delegate?.frameStream(didReceive: update)
case .some(.serverError(let message)):
delegate?.frameStream(didFail: message)
- case .some(.keepAlive), .none:
+ case .some(.keepAlive):
break
+ case .none:
+ NSLog(
+ "[native-timeline] ignored undecodable frame stream message bytes=%ld",
+ text.utf8.count
+ )
}
}
diff --git a/crates/screenpipe-engine/src/routes/streaming.rs b/crates/screenpipe-engine/src/routes/streaming.rs
index c0ad952a1..bdddc01c1 100644
--- a/crates/screenpipe-engine/src/routes/streaming.rs
+++ b/crates/screenpipe-engine/src/routes/streaming.rs
@@ -47,6 +47,11 @@ pub struct StreamFramesRequest {
const MAX_STREAM_FRAME_LIMIT: usize = 10_000;
const DEFAULT_STREAM_FRAME_LIMIT: usize = MAX_STREAM_FRAME_LIMIT;
const STREAM_BATCH_CAPACITY: usize = 100;
+/// Safari/Chromium accept very large WebSocket messages, but native clients
+/// commonly enforce a much smaller receive ceiling. Production transcript and
+/// screen-text payloads can make 100 otherwise-small timeline rows exceed that
+/// ceiling, so the wire batch must be bounded by encoded bytes as well as rows.
+const MAX_STREAM_BATCH_BYTES: usize = 8 * 1024 * 1024;
const STREAM_BATCH_FLUSH_DELAY: Duration = Duration::from_millis(100);
fn stream_frame_limit(requested: Option<usize>) -> usize {
@@ -828,14 +833,52 @@ async fn send_batch(
sender: &mut futures::stream::SplitSink<WebSocket, Message>,
buffer: &mut Vec<StreamTimeSeriesResponse>,
) -> Result<(), Box<dyn std::error::Error>> {
+ for json in encode_stream_batches(buffer, MAX_STREAM_BATCH_BYTES)? {
+ sender.send(Message::Text(json)).await?;
+ }
+ Ok(())
+}
+
+/// Serialize buffered rows into independently decodable JSON arrays while
+/// keeping each WebSocket message below `max_bytes` whenever one row itself
+/// fits. The caller owns retrying the request if the socket closes mid-send.
+fn encode_stream_batches(
+ buffer: &mut Vec<StreamTimeSeriesResponse>,
+ max_bytes: usize,
+) -> Result<Vec<String>, serde_json::Error> {
if buffer.is_empty() {
- return Ok(());
+ return Ok(Vec::new());
}
- let json = serde_json::to_string(&buffer)?;
- sender.send(Message::Text(json)).await?;
- buffer.clear();
- Ok(())
+ let rows = std::mem::take(buffer);
+ let mut messages = Vec::new();
+ let mut message = String::from("[");
+ let mut row_count = 0usize;
+
+ for row in rows {
+ let encoded = serde_json::to_string(&row)?;
+ let separator_bytes = usize::from(row_count > 0);
+ let next_size = message.len() + separator_bytes + encoded.len() + 1;
+ if row_count > 0 && next_size > max_bytes {
+ message.push(']');
+ messages.push(message);
+ message = String::from("[");
+ row_count = 0;
+ }
+
+ if row_count > 0 {
+ message.push(',');
+ }
+ message.push_str(&encoded);
+ row_count += 1;
+ }
+
+ if row_count > 0 {
+ message.push(']');
+ messages.push(message);
+ }
+
+ Ok(messages)
}
#[cfg(test)]
@@ -888,6 +931,54 @@ mod tests {
}
}
+ #[test]
+ fn websocket_batches_are_split_by_encoded_size() {
+ let mut responses = Vec::new();
+ for frame_id in 0..24 {
+ let mut response = StreamTimeSeriesResponse::from(create_time_series_frame(
+ create_test_frame_data(1, 0),
+ ));
+ response.devices[0].frame_id = frame_id;
+ // `text` and its legacy `ocr_text` alias are both sent today. This
+ // deliberately makes the original 100-row batch larger than a
+ // native WebSocket receive ceiling without needing private data.
+ let payload = "x".repeat(512 * 1024);
+ response.devices[0].metadata.text = payload.clone();
+ response.devices[0].metadata.ocr_text = payload;
+ responses.push(response);
+ }
+
+ let messages = encode_stream_batches(&mut responses, MAX_STREAM_BATCH_BYTES).unwrap();
+ assert!(
+ responses.is_empty(),
+ "encoded rows must drain the pending buffer"
+ );
+ assert!(
+ messages.len() > 1,
+ "oversized production batch must be split"
+ );
+ assert!(
+ messages
+ .iter()
+ .all(|message| message.len() <= MAX_STREAM_BATCH_BYTES),
+ "every multi-row WebSocket message must fit the native receive ceiling"
+ );
+ let decoded_rows: usize = messages
+ .iter()
+ .map(|message| {
+ serde_json::from_str::<serde_json::Value>(message)
+ .unwrap()
+ .as_array()
+ .unwrap()
+ .len()
+ })
+ .sum();
+ assert_eq!(
+ decoded_rows, 24,
+ "splitting must not drop or duplicate rows"
+ );
+ }
+
/// TEST: Demonstrate and verify the audio duplication bug is fixed
/// With the fix, audio should appear only once, not duplicated per OCR entry
#[test]
diff --git a/crates/screenpipe-engine/tests/stream_frames_test.rs b/crates/screenpipe-engine/tests/stream_frames_test.rs
index 33f4dc19d..4b2f612e6 100644
--- a/crates/screenpipe-engine/tests/stream_frames_test.rs
+++ b/crates/screenpipe-engine/tests/stream_frames_test.rs
@@ -22,7 +22,7 @@ struct StreamTimeSeriesResponse {
mod tests {
use super::*;
use screenpipe_audio::audio_manager::AudioManagerBuilder;
- use screenpipe_db::{DatabaseManager, OcrEngine};
+ use screenpipe_db::{AudioDevice, DatabaseManager, DeviceType, FrameWindowData, OcrEngine};
use screenpipe_engine::SCServer;
use std::{net::SocketAddr, sync::Arc};
@@ -166,6 +166,118 @@ mod tests {
assert_eq!(last, start);
}
+ #[tokio::test]
+ async fn test_large_production_stream_is_split_for_native_clients() {
+ let (url, db, server_handle) = setup_stream_test_server().await;
+ let device_name = "large-native-stream-monitor";
+ db.insert_video_chunk("large-native-stream.mp4", device_name)
+ .await
+ .unwrap();
+
+ let start = DateTime::parse_from_rfc3339("2026-06-29T00:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let total_frames = 24usize;
+ let seeded_frames: Vec<_> = (0..total_frames)
+ .map(|idx| {
+ (
+ start + Duration::seconds(idx as i64),
+ idx as i64,
+ vec![FrameWindowData {
+ app_name: Some("Large Capture".to_string()),
+ window_name: Some(format!("Large window {idx}")),
+ browser_url: None,
+ focused: true,
+ text: format!("native timeline frame {idx}"),
+ text_json: "[]".to_string(),
+ }],
+ )
+ })
+ .collect();
+ db.insert_multi_frames_with_ocr_batch(
+ device_name,
+ &seeded_frames,
+ Arc::new(OcrEngine::Tesseract),
+ )
+ .await
+ .unwrap();
+ let audio_chunk_id = db
+ .insert_audio_chunk("large-native-stream.wav", Some(start))
+ .await
+ .unwrap();
+ db.insert_audio_transcription(
+ audio_chunk_id,
+ &"spoken words ".repeat(350_000),
+ 0,
+ "e2e",
+ &AudioDevice {
+ name: "large-native-stream-microphone".to_string(),
+ device_type: DeviceType::Input,
+ },
+ None,
+ Some(0.0),
+ Some(total_frames as f64),
+ Some(start),
+ )
+ .await
+ .unwrap();
+
+ let end = start + Duration::seconds(total_frames as i64 - 1);
+ let (ws_stream, _) = tokio_tungstenite::connect_async(&url)
+ .await
+ .expect("websocket should connect");
+ let (mut write, mut read) = ws_stream.split();
+ write
+ .send(Message::Text(
+ serde_json::to_string(&StreamFramesLimitedRequest {
+ start_time: start.to_rfc3339(),
+ end_time: end.to_rfc3339(),
+ order: "descending".to_string(),
+ limit: total_frames,
+ })
+ .unwrap(),
+ ))
+ .await
+ .expect("request should send");
+
+ let (received_frames, message_sizes) = timeout(std::time::Duration::from_secs(30), async {
+ let mut received = Vec::with_capacity(total_frames);
+ let mut sizes = Vec::new();
+ while received.len() < total_frames {
+ let msg = read
+ .next()
+ .await
+ .expect("websocket should stay open")
+ .expect("message should read");
+ let Message::Text(text) = msg else {
+ continue;
+ };
+ if text == "\"keep-alive-text\"" {
+ continue;
+ }
+ sizes.push(text.len());
+ let mut batch: Vec<StreamTimeSeriesResponse> =
+ serde_json::from_str(&text).expect("response batch should parse");
+ received.append(&mut batch);
+ }
+ (received, sizes)
+ })
+ .await
+ .expect("large production stream should remain connected");
+
+ server_handle.abort();
+
+ assert_eq!(received_frames.len(), total_frames);
+ assert!(
+ message_sizes.len() > 1,
+ "a >64 MiB logical response must use multiple WebSocket messages"
+ );
+ assert!(
+ message_sizes.iter().all(|size| *size <= 8 * 1024 * 1024),
+ "every message must fit the native client's bounded receive window"
+ );
+ }
+
/// TEST 1: Reproduce the main issue - new frames after initial fetch are not pushed
///
/// This test verifies the bug where: