Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix native timeline search jumps outside loaded frames

Louis Beaumont0f7e6b915af9
diff --git a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
index f7f6cda40..6f0f262a4 100644
--- a/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
+++ b/apps/screenpipe-app-tauri/e2e/specs/search/search-bugs-4645.spec.ts
@@ -43,6 +43,7 @@ interface NativeTimelineSearchState {
searchQuery?: string;
activeResultIndex?: number;
searchFrameIds: string[];
+ loadedFrameIds: string[];
}
async function nativeTimelineSearchState(
@@ -69,24 +70,55 @@ async function nativeTimelineSearchState(
}
}
-async function clickFirstReadySearchThumbnail(query: string): Promise<string> {
+async function nativeTimelineSearchStateFromSearch(
+ windowLabel: string,
+): Promise<NativeTimelineSearchState | null> {
+ return await browser.waitUntil(
+ async () => {
+ try {
+ return await invokeOrThrow<NativeTimelineSearchState | null>(
+ "plugin:e2e|native_timeline_search_state",
+ { windowLabel },
+ );
+ } catch {
+ return false;
+ }
+ },
+ {
+ timeout: t(10_000),
+ interval: 100,
+ timeoutMsg: "Search webview never exposed native Timeline state",
+ },
+ );
+}
+
+async function readySearchThumbnail(
+ query: string,
+ index = 0,
+): Promise<{ frameId: string; selector: string }> {
const input = await $('input[placeholder*="search memory"]');
await input.waitForExist({ timeout: t(20_000) });
await input.setValue(query);
- const first = await $("[data-index='0']");
- await first.waitForExist({ timeout: t(20_000) });
+ const selector = `[data-index='${index}']`;
+ const result = await $(selector);
+ await result.waitForExist({ timeout: t(20_000) });
await browser.waitUntil(
- async () => (await first.getAttribute("data-thumbnail-ready")) === "true",
+ async () => (await result.getAttribute("data-thumbnail-ready")) === "true",
{
timeout: t(20_000),
interval: 100,
- timeoutMsg: "first exact Search thumbnail never became clickable",
+ timeoutMsg: `Search thumbnail ${index} never became clickable`,
},
);
- const frameId = await first.getAttribute("data-frame-id");
+ const frameId = await result.getAttribute("data-frame-id");
if (!frameId) throw new Error("Search card did not expose its exact frame id");
- await first.click();
- return frameId;
+ return { frameId, selector };
+}
+
+async function clickReadySearchThumbnail(query: string, index = 0): Promise<string> {
+ const result = await readySearchThumbnail(query, index);
+ await $(result.selector).click();
+ return result.frameId;
}
describe("Search bugs over seeded data (reproduces #4645)", function () {
@@ -562,14 +594,23 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {
{ timeout: t(20_000), timeoutMsg: "Search window did not open from Home" },
);
await browser.switchToWindow("search");
- const expectedFrameId = await clickFirstReadySearchThumbnail("vector");
+ // Address one stored-OCR frame uniquely. It sits between the native
+ // stream's two boundary frames, and the pre-click assertion below proves
+ // the timeline has not loaded it yet.
+ const searchQuery = "vector search result number 5";
+ const searchResult = await readySearchThumbnail(searchQuery);
+ const beforeClick = await nativeTimelineSearchStateFromSearch("home");
+ expect(beforeClick?.attached).toBe(true);
+ expect(beforeClick?.loadedFrameIds ?? []).not.toContain(searchResult.frameId);
+ await $(searchResult.selector).click();
+ const expectedFrameId = searchResult.frameId;
await browser.switchToWindow("home");
await browser.waitUntil(
async () => {
const state = await nativeTimelineSearchState("home");
return state?.currentFrameId === expectedFrameId &&
- state.searchQuery === "vector" &&
+ state.searchQuery === searchQuery &&
state.searchFrameIds.includes(expectedFrameId);
},
{
@@ -624,7 +665,7 @@ describe("Search bugs over seeded data (reproduces #4645)", function () {
{ timeout: t(20_000), timeoutMsg: "Search window did not open from overlay" },
);
await browser.switchToWindow("search");
- const expectedFrameId = await clickFirstReadySearchThumbnail("vector");
+ const expectedFrameId = await clickReadySearchThumbnail("vector");
await browser.switchToWindow(overlayLabel as string);
await browser.waitUntil(
diff --git a/apps/screenpipe-app-tauri/package.json b/apps/screenpipe-app-tauri/package.json
index 4f95e954c..82d1fdce7 100644
--- a/apps/screenpipe-app-tauri/package.json
+++ b/apps/screenpipe-app-tauri/package.json
@@ -56,7 +56,7 @@
"test:e2e:capture-stall-recovery:macos": "SCREENPIPE_E2E_SEED=onboarding,no-audio,ignore-disk-pressure,sck-capture-hang-once,capture-loop-silent-once,sck-lookup-hang-once SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/capture-stall-recovery.spec.ts",
"test:e2e:recording-health-focus-cold:macos": "SCREENPIPE_E2E_SEED=onboarding,no-audio,ignore-disk-pressure,focus-cold-heartbeat SCREENPIPE_PORT=3047 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/recording-health-focus-cold.spec.ts",
"test:e2e:recording-health-return-race": "SCREENPIPE_E2E_SEED=onboarding,no-recording,recording-health-return-race SCREENPIPE_PORT=3044 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/recording-health-return-race.spec.ts",
- "test:e2e:search-bugs": "SCREENPIPE_E2E_SEED=onboarding,no-recording,search-fixture SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/search/search-bugs-4645.spec.ts",
+ "test:e2e:search-bugs": "SCREENPIPE_E2E_SEED=onboarding,no-recording,search-fixture SCREENPIPE_E2E_TIMELINE_FRAME_LIMIT=2 SCREENPIPE_PORT=3041 bun run wdio run e2e/wdio.conf.ts --spec e2e/specs/search/search-bugs-4645.spec.ts",
"typecheck": "bun x tsc --noEmit",
"bindings:check": "cargo test -p screenpipe-app tauri_bindings_are_current --manifest-path src-tauri/Cargo.toml -- --nocapture",
"bindings:generate": "UPDATE_TAURI_BINDINGS=1 cargo test -p screenpipe-app export_typescript_bindings --manifest-path src-tauri/Cargo.toml -- --nocapture",
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
index 3fb5cc6e0..49b7aff67 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineAPI.swift
@@ -64,11 +64,17 @@ struct FrameStreamRequest: Codable, Equatable {
case limit
}
- init(start: Date, end: Date, order: String = "descending", limit: Int = 2500) {
+ init(start: Date, end: Date, order: String = "descending", limit: Int? = nil) {
self.startTime = TimelineTime.iso(start)
self.endTime = TimelineTime.iso(end)
self.order = order
- self.limit = limit
+ let environment = ProcessInfo.processInfo.environment
+ let e2eLimit = environment["SCREENPIPE_E2E_SEED"] == nil
+ ? nil
+ : environment["SCREENPIPE_E2E_TIMELINE_FRAME_LIMIT"]
+ .flatMap(Int.init)
+ .flatMap { $0 > 0 ? $0 : nil }
+ self.limit = limit ?? e2eLimit ?? 2500
}
}
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
index 9fe05449c..30f1b7436 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineViewModel.swift
@@ -332,6 +332,19 @@ final class TimelineViewModel: ObservableObject {
stream.request(FrameStreamRequest(start: range.start, end: range.end))
}
+ /// Search can return an exact frame that falls outside the day's capped
+ /// 2,500-frame stream. Fetch a tiny range around the hit first; once the
+ /// frame arrives, `flushPending` backfills the normal day without losing
+ /// the selected frame from the merged stream.
+ private func requestSearchWindow(around timestamp: Date) {
+ let radius: TimeInterval = 2
+ stream.request(FrameStreamRequest(
+ start: timestamp.addingTimeInterval(-radius),
+ end: timestamp.addingTimeInterval(radius),
+ limit: 2500
+ ))
+ }
+
private func refreshHealth() {
Task { [weak self] in
guard let self else { return }
@@ -360,15 +373,18 @@ final class TimelineViewModel: ObservableObject {
let previousFrameId = currentFrame?.devices.first?.frameId
frames = result.frames
currentIndex = TimelineLiveEdge.shiftIndex(currentIndex, newFramesAtFront: result.newAtFront)
+ var resolvedPendingSearch = false
if let pending = pendingSearchNavigation {
if let frameId = pending.frameId,
let index = TimelineNavigation.index(ofFrameId: frameId, in: frames) {
currentIndex = index
pendingSearchNavigation = nil
+ resolvedPendingSearch = true
} else if pending.frameId == nil,
let index = TimelineNavigation.indexNearest(pending.timestamp, in: frames) {
currentIndex = index
pendingSearchNavigation = nil
+ resolvedPendingSearch = true
}
}
meetings = TimelineMeetingDetection.detect(frames: frames)
@@ -385,6 +401,9 @@ final class TimelineViewModel: ObservableObject {
}
fetchTagsForViewport()
+ if resolvedPendingSearch {
+ requestDay(currentDate)
+ }
}
// MARK: Tags
@@ -1021,6 +1040,7 @@ final class TimelineViewModel: ObservableObject {
supersedePendingNavigation: true,
preservePendingSearchNavigation: true
)
+ requestSearchWindow(around: timestamp)
}
func stepSearchResult(_ delta: Int) {
diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
index e271fde56..8479a4a77 100644
--- a/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
+++ b/apps/screenpipe-app-tauri/src-tauri/swift/timeline/TimelineWindow.swift
@@ -412,7 +412,8 @@ final class TimelineWindowController: NSObject, NSWindowDelegate {
var state: [String: Any] = [
"attached": true,
"queued": pendingSearchNavigation[label] != nil,
- "searchFrameIds": model.searchReview?.frameIds ?? []
+ "searchFrameIds": model.searchReview?.frameIds ?? [],
+ "loadedFrameIds": model.frames.compactMap { $0.devices.first?.frameId }
]
if let frameId = model.currentFrame?.devices.first?.frameId {
state["currentFrameId"] = frameId