Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix visible native timeline search jumps

Louis Beaumonte8d02b2efceb
diff --git a/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx b/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx
index 8ed1c18c2..f5f1605b7 100644
--- a/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx
+++ b/apps/screenpipe-app-tauri/components/rewind/search-modal.tsx
@@ -3247,6 +3247,7 @@ export function SearchModal({ isOpen, onClose, onNavigateToTimestamp, embedded =
key={result.frame_id}
data-index={index}
data-frame-id={result.frame_id}
+ data-timestamp={result.timestamp}
data-nav-index={pos}
onClick={() => {
if (thumbnailReady) handleSelectResult(result);
diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
index 6f0f262a4..a5d0f385c 100644
--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
@@ -40,6 +40,9 @@ interface NativeTimelineSearchState {
attached: boolean;
queued: boolean;
currentFrameId?: string;
+ displayedFrameId?: string;
+ loadedImageFrameId?: string;
+ currentTimestamp?: string;
searchQuery?: string;
activeResultIndex?: number;
searchFrameIds: string[];
@@ -95,24 +98,30 @@ async function nativeTimelineSearchStateFromSearch(
async function readySearchThumbnail(
query: string,
index = 0,
-): Promise<{ frameId: string; selector: string }> {
+): Promise<{ frameId: string; timestamp: string; selector: string }> {
const input = await $('input[placeholder*="search memory"]');
await input.waitForExist({ timeout: t(20_000) });
await input.setValue(query);
const selector = `[data-index='${index}']`;
- const result = await $(selector);
- await result.waitForExist({ timeout: t(20_000) });
+ await $(selector).waitForExist({ timeout: t(20_000) });
await browser.waitUntil(
- async () => (await result.getAttribute("data-thumbnail-ready")) === "true",
+ // Search cards are intentionally replaced as exact-thumbnail verification
+ // settles. Reacquire the element on every poll so a valid rerender cannot
+ // turn this user-click test into a stale-WebDriver-element failure.
+ async () =>
+ (await $(selector).getAttribute("data-thumbnail-ready")) === "true",
{
timeout: t(20_000),
interval: 100,
timeoutMsg: `Search thumbnail ${index} never became clickable`,
},
);
+ const result = await $(selector);
const frameId = await result.getAttribute("data-frame-id");
if (!frameId) throw new Error("Search card did not expose its exact frame id");
- return { frameId, selector };
+ const timestamp = await result.getAttribute("data-timestamp");
+ if (!timestamp) throw new Error("Search card did not expose its exact timestamp");
+ return { frameId, timestamp, selector };
}
async function clickReadySearchThumbnail(query: string, index = 0): Promise<string> {
@@ -610,6 +619,12 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {
async () => {
const state = await nativeTimelineSearchState("home");
return state?.currentFrameId === expectedFrameId &&
+ state.displayedFrameId === expectedFrameId &&
+ state.loadedImageFrameId === expectedFrameId &&
+ Math.abs(
+ new Date(state.currentTimestamp ?? 0).getTime() -
+ new Date(searchResult.timestamp).getTime(),
+ ) < 2 &&
state.searchQuery === searchQuery &&
state.searchFrameIds.includes(expectedFrameId);
},
@@ -665,13 +680,21 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {
{ timeout: t(20_000), timeoutMsg: "Search window did not open from overlay" },
);
await browser.switchToWindow("search");
- const expectedFrameId = await clickReadySearchThumbnail("vector");
+ const searchResult = await readySearchThumbnail("vector");
+ await $(searchResult.selector).click();
+ const expectedFrameId = searchResult.frameId;
await browser.switchToWindow(overlayLabel as string);
await browser.waitUntil(
async () => {
const state = await nativeTimelineSearchState(overlayLabel as string);
return state?.currentFrameId === expectedFrameId &&
+ state.displayedFrameId === expectedFrameId &&
+ state.loadedImageFrameId === expectedFrameId &&
+ Math.abs(
+ new Date(state.currentTimestamp ?? 0).getTime() -
+ new Date(searchResult.timestamp).getTime(),
+ ) < 2 &&
state.searchQuery === "vector" &&
state.searchFrameIds.includes(expectedFrameId);
},
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
index 49b7aff67..3ee9ab619 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
@@ -208,6 +208,12 @@ protocol FrameStreamClientDelegate: AnyObject {
/// Owns one `/stream/frames` socket, reconnecting forever with backoff.
final class FrameStreamClient: NSObject {
+ /// A production batch can exceed URLSessionWebSocketTask's 1 MiB default
+ /// once OCR, accessibility text and multiple monitors are present. The web
+ /// timeline accepts those batches; Swift must do the same instead of
+ /// disconnecting mid-response and leaving the playhead on stale data.
+ static let maximumMessageSize = 64 * 1024 * 1024
+
enum State: Equatable {
case idle
case connecting
@@ -285,6 +291,7 @@ final class FrameStreamClient: NSObject {
}
let socket = session.webSocketTask(with: urlRequest)
+ socket.maximumMessageSize = Self.maximumMessageSize
task = socket
socket.resume()
// The task reports "running" immediately; the first successful receive
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
index 30f1b7436..5bc39a31a 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
@@ -183,6 +183,11 @@ final class TimelineViewModel: ObservableObject {
// Playhead
@Published var currentIndex: Int = 0
@Published private(set) var currentImage: NSImage?
+ /// The exact device frame represented by `currentImage`. A time-series row
+ /// can contain several monitors, so the row index alone is not enough to
+ /// prove the canvas is showing the thumbnail the user clicked.
+ @Published private(set) var currentImageFrameId: String?
+ @Published private(set) var preferredFrameId: String?
@Published private(set) var isLoadingImage = false
/// Set when every tier failed for the current frame. Without this the
/// canvas spins forever on an unreadable capture instead of saying so.
@@ -341,7 +346,10 @@ final class TimelineViewModel: ObservableObject {
stream.request(FrameStreamRequest(
start: timestamp.addingTimeInterval(-radius),
end: timestamp.addingTimeInterval(radius),
- limit: 2500
+ // Four seconds cannot legitimately need a day-sized response.
+ // Keeping this small also makes the exact hit win promptly over a
+ // large day backfill on the same websocket.
+ limit: 64
))
}
@@ -370,18 +378,20 @@ final class TimelineViewModel: ObservableObject {
pendingBatch = []
let result = TimelineMerge.merge(existing: frames, incoming: incoming)
- let previousFrameId = currentFrame?.devices.first?.frameId
+ let previousDisplayFrameId = displayFrameId
frames = result.frames
currentIndex = TimelineLiveEdge.shiftIndex(currentIndex, newFramesAtFront: result.newAtFront)
var resolvedPendingSearch = false
if let pending = pendingSearchNavigation {
if let frameId = pending.frameId,
let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {
+ preferredFrameId = frameId
currentIndex = index
pendingSearchNavigation = nil
resolvedPendingSearch = true
} else if pending.frameId == nil,
let index = TimelineNavigation.indexNearest(pending.timestamp, in: frames) {
+ preferredFrameId = nil
currentIndex = index
pendingSearchNavigation = nil
resolvedPendingSearch = true
@@ -396,7 +406,9 @@ final class TimelineViewModel: ObservableObject {
connectionError = nil
// Only reload pixels when the frame under the playhead actually changed.
- if currentFrame?.devices.first?.frameId != previousFrameId || currentImage == nil {
+ if displayFrameId != previousDisplayFrameId
+ || currentImageFrameId != displayFrameId
+ || currentImage == nil {
loadCurrentImage()
}
@@ -576,6 +588,25 @@ final class TimelineViewModel: ObservableObject {
return frames.indices.contains(index) ? frames[index] : nil
}
+ /// Device within the displayed time-series row. Search supplies an exact
+ /// frame id; without retaining it, multi-monitor hits silently rendered
+ /// device 0 even when the clicked thumbnail belonged to device 1.
+ var displayDeviceIndex: Int? {
+ guard let frame = displayFrame else { return nil }
+ if let preferredFrameId,
+ let index = frame.devices.firstIndex(where: { $0.frameId == preferredFrameId }) {
+ return index
+ }
+ return frame.devices.indices.first
+ }
+
+ var displayFrameId: String? {
+ guard let frame = displayFrame,
+ let index = displayDeviceIndex,
+ frame.devices.indices.contains(index) else { return nil }
+ return frame.devices[index].frameId
+ }
+
var currentTimestamp: Date? {
currentFrame.flatMap { TimelineFrames.date(of: $0) }
}
@@ -693,6 +724,11 @@ final class TimelineViewModel: ObservableObject {
func setIndex(_ index: Int, pausePlayback: Bool = true) {
let clamped = min(max(0, index), max(0, frames.count - 1))
+ if let preferredFrameId,
+ frames.indices.contains(clamped),
+ !frames[clamped].devices.contains(where: { $0.frameId == preferredFrameId }) {
+ self.preferredFrameId = nil
+ }
guard clamped != currentIndex else { return }
if pausePlayback, isPlaying { pause() }
currentIndex = clamped
@@ -724,26 +760,33 @@ final class TimelineViewModel: ObservableObject {
}
private func loadCurrentImage() {
- guard let frame = displayFrame else {
+ guard let frame = displayFrame,
+ let deviceIndex = displayDeviceIndex,
+ frame.devices.indices.contains(deviceIndex) else {
currentImage = nil
+ currentImageFrameId = nil
imageUnavailable = false
return
}
+ let targetFrameId = frame.devices[deviceIndex].frameId
imageLoadToken += 1
let token = imageLoadToken
isLoadingImage = true
imageUnavailable = false
+ // The previous pixels may stay cached, but they are no longer allowed
+ // to render as if they belonged to this playhead position.
+ currentImageFrameId = nil
Task { [images] in
- let image = await images.image(for: frame)
+ let image = await images.image(for: frame, deviceIndex: deviceIndex)
await MainActor.run {
guard token == self.imageLoadToken else { return }
- // Keep the previous image when a load fails so the canvas does
- // not flash empty mid-scrub, but record the failure so the view
- // can stop pretending it is still loading.
if let image {
self.currentImage = image
+ self.currentImageFrameId = targetFrameId
self.imageUnavailable = false
} else {
+ self.currentImage = nil
+ self.currentImageFrameId = nil
self.imageUnavailable = true
}
self.isLoadingImage = false
@@ -792,6 +835,7 @@ final class TimelineViewModel: ObservableObject {
// later clear loading state for this current-day request.
navigationGeneration += 1
pendingSearchNavigation = nil
+ preferredFrameId = nil
isNavigating = false
currentDate = Date()
currentIndex = 0
@@ -815,6 +859,8 @@ final class TimelineViewModel: ObservableObject {
currentIndex = 0
frames = []
currentImage = nil
+ currentImageFrameId = nil
+ preferredFrameId = nil
isLoading = true
requestDay(date)
// Never leave the spinner up forever if the day query stalls.
@@ -1017,13 +1063,14 @@ final class TimelineViewModel: ObservableObject {
if let frameId,
let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {
pendingSearchNavigation = nil
- setIndex(index)
+ selectSearchFrame(frameId, at: index)
return
}
if frameId == nil,
Calendar.current.isDate(timestamp, inSameDayAs: currentDate),
let index = TimelineNavigation.indexNearest(timestamp, in: frames) {
pendingSearchNavigation = nil
+ preferredFrameId = nil
setIndex(index)
return
}
@@ -1055,8 +1102,20 @@ final class TimelineViewModel: ObservableObject {
var updated = review
updated.activeIndex = index
searchReview = updated
- if let frameIndex = TimelineNavigation.index(ofFrameId: review.frameIds[index], in: frames) {
- setIndex(frameIndex)
+ let frameId = review.frameIds[index]
+ if let frameIndex = TimelineNavigation.index(ofFrameId: frameId, in: frames) {
+ selectSearchFrame(frameId, at: frameIndex)
+ }
+ }
+
+ private func selectSearchFrame(_ frameId: String, at index: Int) {
+ let previousDisplayFrameId = displayFrameId
+ preferredFrameId = frameId
+ setIndex(index)
+ // `setIndex` intentionally no-ops for the same time-series row. Device
+ // selection can still change within that row, so reload explicitly.
+ if displayFrameId != previousDisplayFrameId || currentImageFrameId != displayFrameId {
+ loadCurrentImage()
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift
index f19076eb8..b0eb3662f 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViews.swift
@@ -175,7 +175,8 @@ struct TimelineFrameCanvas: View {
@ViewBuilder
private var frameContent: some View {
if let frame = model.displayFrame, TimelineFrames.hasVisual(frame) {
- if let image = model.currentImage {
+ if let image = model.currentImage,
+ model.currentImageFrameId == model.displayFrameId {
// Not `Image(nsImage:)`: the point of a recorded frame is to
// read it and take the text out, which needs VisionKit sitting
// on the pixels.
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
index 8479a4a77..e10f0fbbb 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
@@ -413,11 +413,20 @@ final class TimelineWindowController: NSObject, NSWindowDelegate {
"attached": true,
"queued": pendingSearchNavigation[label] != nil,
"searchFrameIds": model.searchReview?.frameIds ?? [],
- "loadedFrameIds": model.frames.compactMap { $0.devices.first?.frameId }
+ "loadedFrameIds": model.frames.flatMap { $0.devices.map(\.frameId) }
]
- if let frameId = model.currentFrame?.devices.first?.frameId {
+ if let frameId = model.preferredFrameId ?? model.currentFrame?.devices.first?.frameId {
state["currentFrameId"] = frameId
}
+ if let frameId = model.displayFrameId {
+ state["displayedFrameId"] = frameId
+ }
+ if let frameId = model.currentImageFrameId {
+ state["loadedImageFrameId"] = frameId
+ }
+ if let timestamp = model.currentTimestamp {
+ state["currentTimestamp"] = TimelineTime.iso(timestamp)
+ }
if let review = model.searchReview {
state["searchQuery"] = review.query
state["activeResultIndex"] = review.activeIndex
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift
index 1f6cc7f78..f3ad1b579 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_interaction_tests.swift
@@ -692,6 +692,14 @@ private func testSearchNavigationWaitsForExactFrame(model: TimelineViewModel) {
for index in targetFrames.indices {
targetFrames[index].devices[0].frameId = String(900_000 + index)
}
+ // A time-series row can contain several monitors. Put the clicked result
+ // on the second device so this test fails if the canvas merely changes the
+ // row index while continuing to render device 0.
+ var clickedDevice = targetFrames[2].devices[0]
+ targetFrames[2].devices[0].frameId = "900102"
+ clickedDevice.deviceId = "second-monitor"
+ clickedDevice.frameId = "900002"
+ targetFrames[2].devices.append(clickedDevice)
model.navigateToSearchResult(
timestamp: targetDate.addingTimeInterval(-60),
@@ -703,11 +711,12 @@ private func testSearchNavigationWaitsForExactFrame(model: TimelineViewModel) {
expect(model.frames.isEmpty, "a result on another day must request that day")
model.injectForTesting(frames: targetFrames)
- expectEqual(
- model.currentFrame?.devices.first?.frameId,
- "900002",
- "search navigation resolves the exact pending frame"
+ expect(
+ model.currentFrame?.devices.contains(where: { $0.frameId == "900002" }) == true,
+ "search navigation resolves the exact pending time-series row"
)
+ expectEqual(model.displayFrameId, "900002", "search renders the clicked monitor frame")
+ expectEqual(model.displayDeviceIndex, 1, "search renders the clicked device within the row")
expectEqual(model.searchReview?.activeIndex, 2, "search review starts on the clicked result")
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift
index 2c66650c0..63cff27b3 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline_tests.swift
@@ -205,6 +205,11 @@ private func testRequestEncoding() {
let req = FrameStreamRequest(start: start, end: start.addingTimeInterval(3600))
expectEqual(req.order, "descending", "default order")
expectEqual(req.limit, 2500, "default limit matches the webview")
+ expectEqual(
+ FrameStreamClient.maximumMessageSize,
+ 64 * 1024 * 1024,
+ "native websocket accepts production-sized frame batches"
+ )
guard let data = try? JSONEncoder().encode(req),
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {