YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: merge timeline transcript backfills (#4555)
Louis Beaumont8b97ae7e58c6
diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts b/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.tsindex 7b42b19bb..c717e0a7d 100644--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts@@ -30,7 +30,8 @@ interface StreamTimeSeriesResponse {function createMockFrame(timestamp: string,deviceId: string = "test-device",- frameId: number = 1+ frameId: number = 1,+ audio: unknown[] = []): StreamTimeSeriesResponse {return {timestamp,@@ -45,7 +46,7 @@ function createMockFrame(ocr_text: "test text",browser_url: null,},- audio: [],+ audio,},],};@@ -110,6 +111,30 @@ describe("Timeline Store Logic - Frame Refresh Bug Tests", () => {expect(result.frames.length).toBe(1);});+ it("should merge same-timestamp transcript backfills into cached timeline frames", () => {+ const timestamp = "2026-06-25T20:19:01.273Z";+ const existingFrames = [createMockFrame(timestamp, "monitor_1", 1000192500)];+ const existingTimestamps = new Set([timestamp]);+ const transcript = {+ audio_chunk_id: 1000220712,+ transcription: "this transcript arrived after the cached frame rendered",+ device_name: "MacBook Pro Microphone",+ is_input: true,+ };++ const result = mergeTimelineFrames({+ existingFrames,+ existingTimestamps,+ incomingFrames: [+ createMockFrame(timestamp, "monitor_1", 1000192500, [transcript]),+ ],+ });++ expect(result.frames.length).toBe(1);+ expect(result.changed).toBe(true);+ expect(result.frames[0].devices[0].audio).toEqual([transcript]);+ });+/*** TEST 3: Multiple new frames should all be added*/diff --git a/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts b/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.tsindex 737a6b412..914b22aac 100644--- a/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts@@ -17,6 +17,157 @@ export interface TimelineFrameMergeResult<T extends TimelineFrameLike> {const compareFramesDesc = <T extends TimelineFrameLike>(a: T, b: T) =>b.timestamp.localeCompare(a.timestamp);+type TimelineDeviceLike = {+ device_id?: unknown;+ frame_id?: unknown;+ audio?: unknown[];+ [key: string]: unknown;+};++type TimelineAudioLike = {+ audio_chunk_id?: unknown;+ transcription?: unknown;+ [key: string]: unknown;+};++type TimelineFrameWithDevices = TimelineFrameLike & {+ devices?: TimelineDeviceLike[];+ [key: string]: unknown;+};++const isRecord = (value: unknown): value is Record<string, unknown> =>+ typeof value === "object" && value !== null;++const getDevices = <T extends TimelineFrameLike>(+ frame: T,+): TimelineDeviceLike[] | null => {+ if (!isRecord(frame) || !Array.isArray(frame.devices)) return null;+ return frame.devices.filter(isRecord) as TimelineDeviceLike[];+};++const deviceKey = (device: TimelineDeviceLike): string | null => {+ if (device.device_id != null) return `device:${String(device.device_id)}`;+ if (device.frame_id != null) return `frame:${String(device.frame_id)}`;+ return null;+};++const audioKey = (audio: unknown): string | null => {+ if (!isRecord(audio)) return null;+ const entry = audio as TimelineAudioLike;+ if (entry.audio_chunk_id != null) return `chunk:${String(entry.audio_chunk_id)}`;+ return null;+};++const transcriptionLength = (audio: unknown): number => {+ if (!isRecord(audio)) return 0;+ const transcription = (audio as TimelineAudioLike).transcription;+ return typeof transcription === "string" ? transcription.trim().length : 0;+};++function mergeAudioEntries(+ existingAudio: unknown[] | undefined,+ incomingAudio: unknown[] | undefined,+): { audio: unknown[]; changed: boolean } {+ if (!Array.isArray(incomingAudio) || incomingAudio.length === 0) {+ return { audio: existingAudio ?? [], changed: false };+ }++ const merged = Array.isArray(existingAudio) ? [...existingAudio] : [];+ const byKey = new Map<string, number>();++ for (let i = 0; i < merged.length; i++) {+ const key = audioKey(merged[i]);+ if (key) byKey.set(key, i);+ }++ let changed = false;+ for (const incoming of incomingAudio) {+ const key = audioKey(incoming);+ if (!key) {+ if (!merged.includes(incoming)) {+ merged.push(incoming);+ changed = true;+ }+ continue;+ }++ const existingIndex = byKey.get(key);+ if (existingIndex == null) {+ byKey.set(key, merged.length);+ merged.push(incoming);+ changed = true;+ continue;+ }++ const existing = merged[existingIndex];+ if (transcriptionLength(incoming) > transcriptionLength(existing)) {+ merged[existingIndex] = incoming;+ changed = true;+ }+ }++ return { audio: merged, changed };+}++function mergeSameTimestampFrame<T extends TimelineFrameLike>(+ existingFrame: T,+ incomingFrame: T,+): { frame: T; changed: boolean } {+ const incomingDevices = getDevices(incomingFrame);+ if (!incomingDevices?.length) return { frame: existingFrame, changed: false };++ const existingDevices = getDevices(existingFrame) ?? [];+ let nextDevices = existingDevices;+ let changed = false;+ const existingDeviceByKey = new Map<string, number>();++ for (let i = 0; i < existingDevices.length; i++) {+ const key = deviceKey(existingDevices[i]);+ if (key) existingDeviceByKey.set(key, i);+ }++ const ensureDevicesCopy = () => {+ if (nextDevices === existingDevices) nextDevices = [...existingDevices];+ };++ for (const incomingDevice of incomingDevices) {+ if (!Array.isArray(incomingDevice.audio) || incomingDevice.audio.length === 0) {+ continue;+ }++ const key = deviceKey(incomingDevice);+ const existingIndex = key ? existingDeviceByKey.get(key) : undefined;+ if (existingIndex == null) {+ ensureDevicesCopy();+ if (key) existingDeviceByKey.set(key, nextDevices.length);+ nextDevices.push(incomingDevice);+ changed = true;+ continue;+ }++ const existingDevice = nextDevices[existingIndex];+ const audioMerge = mergeAudioEntries(existingDevice.audio, incomingDevice.audio);+ if (!audioMerge.changed) continue;++ ensureDevicesCopy();+ nextDevices[existingIndex] = {+ ...existingDevice,+ audio: audioMerge.audio,+ };+ changed = true;+ }++ if (!changed) return { frame: existingFrame, changed: false };++ return {+ frame: {+ ...(existingFrame as unknown as TimelineFrameWithDevices),+ devices: nextDevices,+ } as unknown as T,+ changed: true,+ };+}+function mergeSortedDesc<T extends TimelineFrameLike>(existingFrames: T[],newFrames: T[],@@ -77,14 +228,53 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({}): TimelineFrameMergeResult<T> {const timestamps = replace ? new Set<string>() : existingTimestamps;const newUniqueFrames: T[] = [];+ const frameLocations = new Map<+ string,+ { source: "existing" | "new"; index: number }+ >();+ let updatedExistingFrames = existingFrames;+ let existingChanged = false;++ if (!replace) {+ for (let i = 0; i < existingFrames.length; i++) {+ frameLocations.set(existingFrames[i].timestamp, {+ source: "existing",+ index: i,+ });+ }+ }for (const frame of incomingFrames) {- if (timestamps.has(frame.timestamp)) continue;+ const location = frameLocations.get(frame.timestamp);+ if (location) {+ const currentFrame =+ location.source === "existing"+ ? updatedExistingFrames[location.index]+ : newUniqueFrames[location.index];+ const mergedFrame = mergeSameTimestampFrame(currentFrame, frame);+ if (!mergedFrame.changed) continue;++ if (location.source === "existing") {+ if (updatedExistingFrames === existingFrames) {+ updatedExistingFrames = [...existingFrames];+ }+ updatedExistingFrames[location.index] = mergedFrame.frame;+ existingChanged = true;+ } else {+ newUniqueFrames[location.index] = mergedFrame.frame;+ }+ continue;+ }+timestamps.add(frame.timestamp);+ frameLocations.set(frame.timestamp, {+ source: "new",+ index: newUniqueFrames.length,+ });newUniqueFrames.push(frame);}- if (newUniqueFrames.length === 0) {+ if (newUniqueFrames.length === 0 && !existingChanged) {return {frames: existingFrames,timestamps,@@ -94,11 +284,21 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({};}+ if (newUniqueFrames.length === 0) {+ return {+ frames: updatedExistingFrames,+ timestamps,+ newUniqueFrames,+ newAtFront: 0,+ changed: true,+ };+ }+const sortedNewFrames =newUniqueFrames.length > 1? [...newUniqueFrames].sort(compareFramesDesc): newUniqueFrames;- const previousNewest = replace ? undefined : existingFrames[0]?.timestamp;+ const previousNewest = replace ? undefined : updatedExistingFrames[0]?.timestamp;const newAtFront = previousNewest? sortedNewFrames.filter((frame) => frame.timestamp.localeCompare(previousNewest) > 0,@@ -108,7 +308,7 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({return {frames: replace? sortedNewFrames- : mergeSortedDesc(existingFrames, sortedNewFrames),+ : mergeSortedDesc(updatedExistingFrames, sortedNewFrames),timestamps,newUniqueFrames: sortedNewFrames,newAtFront,