YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix visible native timeline search jumps
Louis Beaumonte8d02b2efceb
diff --git a/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx b/apps/screenpipe-app-tauri/components/rewind/search-modal.tsxindex 8ed1c18c2..f5f1605b7 100644--- a/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx+++ b/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx@@ -3247,6 +3247,7 @@ export function SearchModal({ isOpen, onClose, onNavigateToTimestamp, embedded =key={result.frame_id}data-index={index}data-frame-id={result.frame_id}+ data-timestamp={result.timestamp}data-nav-index={pos}onClick={() => {if (thumbnailReady) handleSelectResult(result);diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.tsindex 6f0f262a4..a5d0f385c 100644--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts@@ -40,6 +40,9 @@ interface NativeTimelineSearchState {attached: boolean;queued: boolean;currentFrameId?: string;+ displayedFrameId?: string;+ loadedImageFrameId?: string;+ currentTimestamp?: string;searchQuery?: string;activeResultIndex?: number;searchFrameIds: string[];@@ -95,24 +98,30 @@ async function nativeTimelineSearchStateFromSearch(async function readySearchThumbnail(query: string,index = 0,-): Promise<{ frameId: string; selector: string }> {+): Promise<{ frameId: string; timestamp: string; selector: string }> {const input = await $('input[placeholder*="search memory"]');await input.waitForExist({ timeout: t(20_000) });await input.setValue(query);const selector = `[data-index='${index}']`;- const result = await $(selector);- await result.waitForExist({ timeout: t(20_000) });+ await $(selector).waitForExist({ timeout: t(20_000) });await browser.waitUntil(- async () => (await result.getAttribute("data-thumbnail-ready")) === "true",+ // Search cards are intentionally replaced as exact-thumbnail verification+ // settles. Reacquire the element on every poll so a valid rerender cannot+ // turn this user-click test into a stale-WebDriver-element failure.+ async () =>+ (await $(selector).getAttribute("data-thumbnail-ready")) === "true",{timeout: t(20_000),interval: 100,timeoutMsg: `Search thumbnail ${index} never became clickable`,},);+ const result = await $(selector);const frameId = await result.getAttribute("data-frame-id");if (!frameId) throw new Error("Search card did not expose its exact frame id");- return { frameId, selector };+ const timestamp = await result.getAttribute("data-timestamp");+ if (!timestamp) throw new Error("Search card did not expose its exact timestamp");+ return { frameId, timestamp, selector };}async function clickReadySearchThumbnail(query: string, index = 0): Promise<string> {@@ -610,6 +619,12 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {async () => {const state = await nativeTimelineSearchState("home");return state?.currentFrameId === expectedFrameId &&+ state.displayedFrameId === expectedFrameId &&+ state.loadedImageFrameId === expectedFrameId &&+ Math.abs(+ new Date(state.currentTimestamp ?? 0).getTime() -+ new Date(searchResult.timestamp).getTime(),+ ) < 2 &&state.searchQuery === searchQuery &&state.searchFrameIds.includes(expectedFrameId);},@@ -665,13 +680,21 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {{ timeout: t(20_000), timeoutMsg: "Search window did not open from overlay" },);await browser.switchToWindow("search");- const expectedFrameId = await clickReadySearchThumbnail("vector");+ const searchResult = await readySearchThumbnail("vector");+ await $(searchResult.selector).click();+ const expectedFrameId = searchResult.frameId;await browser.switchToWindow(overlayLabel as string);await browser.waitUntil(async () => {const state = await nativeTimelineSearchState(overlayLabel as string);return state?.currentFrameId === expectedFrameId &&+ state.displayedFrameId === expectedFrameId &&+ state.loadedImageFrameId === expectedFrameId &&+ Math.abs(+ new Date(state.currentTimestamp ?? 0).getTime() -+ new Date(searchResult.timestamp).getTime(),+ ) < 2 &&state.searchQuery === "vector" &&state.searchFrameIds.includes(expectedFrameId);},diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swiftindex 49b7aff67..3ee9ab619 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift@@ -208,6 +208,12 @@ protocol FrameStreamClientDelegate: AnyObject {/// Owns one `/stream/frames` socket, reconnecting forever with backoff.final class FrameStreamClient: NSObject {+ /// A production batch can exceed URLSessionWebSocketTask's 1 MiB default+ /// once OCR, accessibility text and multiple monitors are present. The web+ /// timeline accepts those batches; Swift must do the same instead of+ /// disconnecting mid-response and leaving the playhead on stale data.+ static let maximumMessageSize = 64 * 1024 * 1024+enum State: Equatable {case idlecase connecting@@ -285,6 +291,7 @@ final class FrameStreamClient: NSObject {}let socket = session.webSocketTask(with: urlRequest)+ socket.maximumMessageSize = Self.maximumMessageSizetask = socketsocket.resume()// The task reports "running" immediately; the first successful receivediff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swiftindex 30f1b7436..5bc39a31a 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift@@ -183,6 +183,11 @@ final class TimelineViewModel: ObservableObject {// Playhead@Published var currentIndex: Int = 0@Published private(set) var currentImage: NSImage?+ /// The exact device frame represented by `currentImage`. A time-series row+ /// can contain several monitors, so the row index alone is not enough to+ /// prove the canvas is showing the thumbnail the user clicked.+ @Published private(set) var currentImageFrameId: String?+ @Published private(set) var preferredFrameId: String?@Published private(set) var isLoadingImage = false/// Set when every tier failed for the current frame. Without this the/// canvas spins forever on an unreadable capture instead of saying so.@@ -341,7 +346,10 @@ final class TimelineViewModel: ObservableObject {stream.request(FrameStreamRequest(start: timestamp.addingTimeInterval(-radius),end: timestamp.addingTimeInterval(radius),- limit: 2500+ // Four seconds cannot legitimately need a day-sized response.+ // Keeping this small also makes the exact hit win promptly over a+ // large day backfill on the same websocket.+ limit: 64))}@@ -370,18 +378,20 @@ final class TimelineViewModel: ObservableObject {pendingBatch = []let result = TimelineMerge.merge(existing: frames, incoming: incoming)- let previousFrameId = currentFrame?.devices.first?.frameId+ let previousDisplayFrameId = displayFrameIdframes = result.framescurrentIndex = TimelineLiveEdge.shiftIndex(currentIndex, newFramesAtFront: result.newAtFront)var resolvedPendingSearch = falseif let pending = pendingSearchNavigation {if let frameId = pending.frameId,let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {+ preferredFrameId = frameIdcurrentIndex = indexpendingSearchNavigation = nilresolvedPendingSearch = true} else if pending.frameId == nil,let index = TimelineNavigation.indexNearest(pending.timestamp, in: frames) {+ preferredFrameId = nilcurrentIndex = indexpendingSearchNavigation = nilresolvedPendingSearch = true@@ -396,7 +406,9 @@ final class TimelineViewModel: ObservableObject {connectionError = nil// Only reload pixels when the frame under the playhead actually changed.- if currentFrame?.devices.first?.frameId != previousFrameId || currentImage == nil {+ if displayFrameId != previousDisplayFrameId+ || currentImageFrameId != displayFrameId+ || currentImage == nil {loadCurrentImage()}@@ -576,6 +588,25 @@ final class TimelineViewModel: ObservableObject {return frames.indices.contains(index) ? frames[index] : nil}+ /// Device within the displayed time-series row. Search supplies an exact+ /// frame id; without retaining it, multi-monitor hits silently rendered+ /// device 0 even when the clicked thumbnail belonged to device 1.+ var displayDeviceIndex: Int? {+ guard let frame = displayFrame else { return nil }+ if let preferredFrameId,+ let index = frame.devices.firstIndex(where: { $0.frameId == preferredFrameId }) {+ return index+ }+ return frame.devices.indices.first+ }++ var displayFrameId: String? {+ guard let frame = displayFrame,+ let index = displayDeviceIndex,+ frame.devices.indices.contains(index) else { return nil }+ return frame.devices[index].frameId+ }+var currentTimestamp: Date? {currentFrame.flatMap { TimelineFrames.date(of: $0) }}@@ -693,6 +724,11 @@ final class TimelineViewModel: ObservableObject {func setIndex(_ index: Int, pausePlayback: Bool = true) {let clamped = min(max(0, index), max(0, frames.count - 1))+ if let preferredFrameId,+ frames.indices.contains(clamped),+ !frames[clamped].devices.contains(where: { $0.frameId == preferredFrameId }) {+ self.preferredFrameId = nil+ }guard clamped != currentIndex else { return }if pausePlayback, isPlaying { pause() }currentIndex = clamped@@ -724,26 +760,33 @@ final class TimelineViewModel: ObservableObject {}private func loadCurrentImage() {- guard let frame = displayFrame else {+ guard let frame = displayFrame,+ let deviceIndex = displayDeviceIndex,+ frame.devices.indices.contains(deviceIndex) else {currentImage = nil+ currentImageFrameId = nilimageUnavailable = falsereturn}+ let targetFrameId = frame.devices[deviceIndex].frameIdimageLoadToken += 1let token = imageLoadTokenisLoadingImage = trueimageUnavailable = false+ // The previous pixels may stay cached, but they are no longer allowed+ // to render as if they belonged to this playhead position.+ currentImageFrameId = nilTask { [images] in- let image = await images.image(for: frame)+ let image = await images.image(for: frame, deviceIndex: deviceIndex)await MainActor.run {guard token == self.imageLoadToken else { return }- // Keep the previous image when a load fails so the canvas does- // not flash empty mid-scrub, but record the failure so the view- // can stop pretending it is still loading.if let image {self.currentImage = image+ self.currentImageFrameId = targetFrameIdself.imageUnavailable = false} else {+ self.currentImage = nil+ self.currentImageFrameId = nilself.imageUnavailable = true}self.isLoadingImage = false@@ -792,6 +835,7 @@ final class TimelineViewModel: ObservableObject {// later clear loading state for this current-day request.navigationGeneration += 1pendingSearchNavigation = nil+ preferredFrameId = nilisNavigating = falsecurrentDate = Date()currentIndex = 0@@ -815,6 +859,8 @@ final class TimelineViewModel: ObservableObject {currentIndex = 0frames = []currentImage = nil+ currentImageFrameId = nil+ preferredFrameId = nilisLoading = truerequestDay(date)// Never leave the spinner up forever if the day query stalls.@@ -1017,13 +1063,14 @@ final class TimelineViewModel: ObservableObject {if let frameId,let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {pendingSearchNavigation = nil- setIndex(index)+ selectSearchFrame(frameId, at: index)return}if frameId == nil,Calendar.current.isDate(timestamp, inSameDayAs: currentDate),let index = TimelineNavigation.indexNearest(timestamp, in: frames) {pendingSearchNavigation = nil+ preferredFrameId = nilsetIndex(index)return}@@ -1055,8 +1102,20 @@ final class TimelineViewModel: ObservableObject {var updated = reviewupdated.activeIndex = indexsearchReview = updated- if let frameIndex = TimelineNavigation.index(ofFrameId: review.frameIds[index], in: frames) {- setIndex(frameIndex)+ let frameId = review.frameIds[index]+ if let frameIndex = TimelineNavigation.index(ofFrameId: frameId, in: frames) {+ selectSearchFrame(frameId, at: frameIndex)+ }+ }++ private func selectSearchFrame(_ frameId: String, at index: Int) {+ let previousDisplayFrameId = displayFrameId+ preferredFrameId = frameId+ setIndex(index)+ // `setIndex` intentionally no-ops for the same time-series row. Device+ // selection can still change within that row, so reload explicitly.+ if displayFrameId != previousDisplayFrameId || currentImageFrameId != displayFrameId {+ loadCurrentImage()}}diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swiftindex f19076eb8..b0eb3662f 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift@@ -175,7 +175,8 @@ struct TimelineFrameCanvas: View {@ViewBuilderprivate var frameContent: some View {if let frame = model.displayFrame, TimelineFrames.hasVisual(frame) {- if let image = model.currentImage {+ if let image = model.currentImage,+ model.currentImageFrameId == model.displayFrameId {// Not `Image(nsImage:)`: the point of a recorded frame is to// read it and take the text out, which needs VisionKit sitting// on the pixels.diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swiftindex 8479a4a77..e10f0fbbb 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift@@ -413,11 +413,20 @@ final class TimelineWindowController: NSObject, NSWindowDelegate {"attached": true,"queued": pendingSearchNavigation[label] != nil,"searchFrameIds": model.searchReview?.frameIds ?? [],- "loadedFrameIds": model.frames.compactMap { $0.devices.first?.frameId }+ "loadedFrameIds": model.frames.flatMap { $0.devices.map(\.frameId) }]- if let frameId = model.currentFrame?.devices.first?.frameId {+ if let frameId = model.preferredFrameId ?? model.currentFrame?.devices.first?.frameId {state["currentFrameId"] = frameId}+ if let frameId = model.displayFrameId {+ state["displayedFrameId"] = frameId+ }+ if let frameId = model.currentImageFrameId {+ state["loadedImageFrameId"] = frameId+ }+ if let timestamp = model.currentTimestamp {+ state["currentTimestamp"] = TimelineTime.iso(timestamp)+ }if let review = model.searchReview {state["searchQuery"] = review.querystate["activeResultIndex"] = review.activeIndexdiff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swiftindex 1f6cc7f78..f3ad1b579 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift@@ -692,6 +692,14 @@ private func testSearchNavigationWaitsForExactFrame(model: TimelineViewModel) {for index in targetFrames.indices {targetFrames[index].devices[0].frameId = String(900_000 + index)}+ // A time-series row can contain several monitors. Put the clicked result+ // on the second device so this test fails if the canvas merely changes the+ // row index while continuing to render device 0.+ var clickedDevice = targetFrames[2].devices[0]+ targetFrames[2].devices[0].frameId = "900102"+ clickedDevice.deviceId = "second-monitor"+ clickedDevice.frameId = "900002"+ targetFrames[2].devices.append(clickedDevice)model.navigateToSearchResult(timestamp: targetDate.addingTimeInterval(-60),@@ -703,11 +711,12 @@ private func testSearchNavigationWaitsForExactFrame(model: TimelineViewModel) {expect(model.frames.isEmpty, "a result on another day must request that day")model.injectForTesting(frames: targetFrames)- expectEqual(- model.currentFrame?.devices.first?.frameId,- "900002",- "search navigation resolves the exact pending frame"+ expect(+ model.currentFrame?.devices.contains(where: { $0.frameId == "900002" }) == true,+ "search navigation resolves the exact pending time-series row")+ expectEqual(model.displayFrameId, "900002", "search renders the clicked monitor frame")+ expectEqual(model.displayDeviceIndex, 1, "search renders the clicked device within the row")expectEqual(model.searchReview?.activeIndex, 2, "search review starts on the clicked result")}diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swiftindex 2c66650c0..63cff27b3 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift@@ -205,6 +205,11 @@ private func testRequestEncoding() {let req = FrameStreamRequest(start: start, end: start.addingTimeInterval(3600))expectEqual(req.order, "descending", "default order")expectEqual(req.limit, 2500, "default limit matches the webview")+ expectEqual(+ FrameStreamClient.maximumMessageSize,+ 64 * 1024 * 1024,+ "native websocket accepts production-sized frame batches"+ )guard let data = try? JSONEncoder().encode(req),let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {