YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix native timeline search jumps outside loaded frames
Louis Beaumont0f7e6b915af9
diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.tsindex f7f6cda40..6f0f262a4 100644--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts@@ -43,6 +43,7 @@ interface NativeTimelineSearchState {searchQuery?: string;activeResultIndex?: number;searchFrameIds: string[];+ loadedFrameIds: string[];}async function nativeTimelineSearchState(@@ -69,24 +70,55 @@ async function nativeTimelineSearchState(}}-async function clickFirstReadySearchThumbnail(query: string): Promise<string> {+async function nativeTimelineSearchStateFromSearch(+ windowLabel: string,+): Promise<NativeTimelineSearchState | null> {+ return await browser.waitUntil(+ async () => {+ try {+ return await invokeOrThrow<NativeTimelineSearchState | null>(+ "plugin:e2e|native_timeline_search_state",+ { windowLabel },+ );+ } catch {+ return false;+ }+ },+ {+ timeout: t(10_000),+ interval: 100,+ timeoutMsg: "Search webview never exposed native Timeline state",+ },+ );+}++async function readySearchThumbnail(+ query: string,+ index = 0,+): Promise<{ frameId: string; selector: string }> {const input = await $('input[placeholder*="search memory"]');await input.waitForExist({ timeout: t(20_000) });await input.setValue(query);- const first = await $("[data-index='0']");- await first.waitForExist({ timeout: t(20_000) });+ const selector = `[data-index='${index}']`;+ const result = await $(selector);+ await result.waitForExist({ timeout: t(20_000) });await browser.waitUntil(- async () => (await first.getAttribute("data-thumbnail-ready")) === "true",+ async () => (await result.getAttribute("data-thumbnail-ready")) === "true",{timeout: t(20_000),interval: 100,- timeoutMsg: "first exact Search thumbnail never became clickable",+ timeoutMsg: `Search thumbnail ${index} never became clickable`,},);- const frameId = await first.getAttribute("data-frame-id");+ const frameId = await result.getAttribute("data-frame-id");if (!frameId) throw new Error("Search card did not expose its exact frame id");- await first.click();- return frameId;+ return { frameId, selector };+}++async function clickReadySearchThumbnail(query: string, index = 0): Promise<string> {+ const result = await readySearchThumbnail(query, index);+ await $(result.selector).click();+ return result.frameId;}describe("Search bugs over seeded data (reproduces #4645)", function () {@@ -562,14 +594,23 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {{ timeout: t(20_000), timeoutMsg: "Search window did not open from Home" },);await browser.switchToWindow("search");- const expectedFrameId = await clickFirstReadySearchThumbnail("vector");+ // Address one stored-OCR frame uniquely. It sits between the native+ // stream's two boundary frames, and the pre-click assertion below proves+ // the timeline has not loaded it yet.+ const searchQuery = "vector search result number 5";+ const searchResult = await readySearchThumbnail(searchQuery);+ const beforeClick = await nativeTimelineSearchStateFromSearch("home");+ expect(beforeClick?.attached).toBe(true);+ expect(beforeClick?.loadedFrameIds ?? []).not.toContain(searchResult.frameId);+ await $(searchResult.selector).click();+ const expectedFrameId = searchResult.frameId;await browser.switchToWindow("home");await browser.waitUntil(async () => {const state = await nativeTimelineSearchState("home");return state?.currentFrameId === expectedFrameId &&- state.searchQuery === "vector" &&+ state.searchQuery === searchQuery &&state.searchFrameIds.includes(expectedFrameId);},{@@ -624,7 +665,7 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {{ timeout: t(20_000), timeoutMsg: "Search window did not open from overlay" },);await browser.switchToWindow("search");- const expectedFrameId = await clickFirstReadySearchThumbnail("vector");+ const expectedFrameId = await clickReadySearchThumbnail("vector");await browser.switchToWindow(overlayLabel as string);await browser.waitUntil(diff --git a/apps/screenpipe-app-tauri/package.json b/apps/screenpipe-app-tauri/package.jsonindex 4f95e954c..82d1fdce7 100644--- a/apps/screenpipe-app-tauri/package.json+++ b/apps/screenpipe-app-tauri/package.json@@ -56,7 +56,7 @@"test:e2e:capture-stall-recovery:macos": "SCREENPIPE_E2E_SEED=onboarding,no-audio,ignore-disk-pressure,sck-capture-hang-once,capture-loop-silent-once,sck-lookup-hang-once SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/capture-stall-recovery.spec.ts","test:e2e:recording-health-focus-cold:macos": "SCREENPIPE_E2E_SEED=onboarding,no-audio,ignore-disk-pressure,focus-cold-heartbeat SCREENPIPE_PORT=3047 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/recording-health-focus-cold.spec.ts","test:e2e:recording-health-return-race": "SCREENPIPE_E2E_SEED=onboarding,no-recording,recording-health-return-race SCREENPIPE_PORT=3044 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/recording-health-return-race.spec.ts",- "test:e2e:search-bugs": "SCREENPIPE_E2E_SEED=onboarding,no-recording,search-fixture SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/search/search-bugs-4645.spec.ts",+ "test:e2e:search-bugs": "SCREENPIPE_E2E_SEED=onboarding,no-recording,search-fixture SCREENPIPE_E2E_TIMELINE_FRAME_LIMIT=2 SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/search/search-bugs-4645.spec.ts","typecheck": "bun x tsc --noEmit","bindings:check": "cargo test -p screenpipe-app tauri_bindings_are_current --manifest-path src-tauri/Cargo.toml -- --nocapture","bindings:generate": "UPDATE_TAURI_BINDINGS=1 cargo test -p screenpipe-app export_typescript_bindings --manifest-path src-tauri/Cargo.toml -- --nocapture",diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swiftindex 3fb5cc6e0..49b7aff67 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift@@ -64,11 +64,17 @@ struct FrameStreamRequest: Codable, Equatable {case limit}- init(start: Date, end: Date, order: String = "descending", limit: Int = 2500) {+ init(start: Date, end: Date, order: String = "descending", limit: Int? = nil) {self.startTime = TimelineTime.iso(start)self.endTime = TimelineTime.iso(end)self.order = order- self.limit = limit+ let environment = ProcessInfo.processInfo.environment+ let e2eLimit = environment["SCREENPIPE_E2E_SEED"] == nil+ ? nil+ : environment["SCREENPIPE_E2E_TIMELINE_FRAME_LIMIT"]+ .flatMap(Int.init)+ .flatMap { $0 > 0 ? $0 : nil }+ self.limit = limit ?? e2eLimit ?? 2500}}diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swiftindex 9fe05449c..30f1b7436 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift@@ -332,6 +332,19 @@ final class TimelineViewModel: ObservableObject {stream.request(FrameStreamRequest(start: range.start, end: range.end))}+ /// Search can return an exact frame that falls outside the day's capped+ /// 2,500-frame stream. Fetch a tiny range around the hit first; once the+ /// frame arrives, `flushPending` backfills the normal day without losing+ /// the selected frame from the merged stream.+ private func requestSearchWindow(around timestamp: Date) {+ let radius: TimeInterval = 2+ stream.request(FrameStreamRequest(+ start: timestamp.addingTimeInterval(-radius),+ end: timestamp.addingTimeInterval(radius),+ limit: 2500+ ))+ }+private func refreshHealth() {Task { [weak self] inguard let self else { return }@@ -360,15 +373,18 @@ final class TimelineViewModel: ObservableObject {let previousFrameId = currentFrame?.devices.first?.frameIdframes = result.framescurrentIndex = TimelineLiveEdge.shiftIndex(currentIndex, newFramesAtFront: result.newAtFront)+ var resolvedPendingSearch = falseif let pending = pendingSearchNavigation {if let frameId = pending.frameId,let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {currentIndex = indexpendingSearchNavigation = nil+ resolvedPendingSearch = true} else if pending.frameId == nil,let index = TimelineNavigation.indexNearest(pending.timestamp, in: frames) {currentIndex = indexpendingSearchNavigation = nil+ resolvedPendingSearch = true}}meetings = TimelineMeetingDetection.detect(frames: frames)@@ -385,6 +401,9 @@ final class TimelineViewModel: ObservableObject {}fetchTagsForViewport()+ if resolvedPendingSearch {+ requestDay(currentDate)+ }}// MARK: Tags@@ -1021,6 +1040,7 @@ final class TimelineViewModel: ObservableObject {supersedePendingNavigation: true,preservePendingSearchNavigation: true)+ requestSearchWindow(around: timestamp)}func stepSearchResult(_ delta: Int) {diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swiftindex e271fde56..8479a4a77 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift@@ -412,7 +412,8 @@ final class TimelineWindowController: NSObject, NSWindowDelegate {var state: [String: Any] = ["attached": true,"queued": pendingSearchNavigation[label] != nil,- "searchFrameIds": model.searchReview?.frameIds ?? []+ "searchFrameIds": model.searchReview?.frameIds ?? [],+ "loadedFrameIds": model.frames.compactMap { $0.devices.first?.frameId }]if let frameId = model.currentFrame?.devices.first?.frameId {state["currentFrameId"] = frameId