Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix: merge timeline transcript backfills (#4555)

Louis Beaumont8b97ae7e58c6
diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts b/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts
index 7b42b19bb..c717e0a7d 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts
+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/timeline-store-logic.test.ts
@@ -30,7 +30,8 @@ interface StreamTimeSeriesResponse {
function createMockFrame(
timestamp: string,
deviceId: string = "test-device",
- frameId: number = 1
+ frameId: number = 1,
+ audio: unknown[] = []
): StreamTimeSeriesResponse {
return {
timestamp,
@@ -45,7 +46,7 @@ function createMockFrame(
ocr_text: "test text",
browser_url: null,
},
- audio: [],
+ audio,
},
],
};
@@ -110,6 +111,30 @@ describe("Timeline Store Logic - Frame Refresh Bug Tests", () => {
expect(result.frames.length).toBe(1);
});
+ it("should merge same-timestamp transcript backfills into cached timeline frames", () => {
+ const timestamp = "2026-06-25T20:19:01.273Z";
+ const existingFrames = [createMockFrame(timestamp, "monitor_1", 1000192500)];
+ const existingTimestamps = new Set([timestamp]);
+ const transcript = {
+ audio_chunk_id: 1000220712,
+ transcription: "this transcript arrived after the cached frame rendered",
+ device_name: "MacBook Pro Microphone",
+ is_input: true,
+ };
+
+ const result = mergeTimelineFrames({
+ existingFrames,
+ existingTimestamps,
+ incomingFrames: [
+ createMockFrame(timestamp, "monitor_1", 1000192500, [transcript]),
+ ],
+ });
+
+ expect(result.frames.length).toBe(1);
+ expect(result.changed).toBe(true);
+ expect(result.frames[0].devices[0].audio).toEqual([transcript]);
+ });
+
/**
* TEST 3: Multiple new frames should all be added
*/
diff --git a/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts b/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts
index 737a6b412..914b22aac 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts
+++ b/apps/screenpipe-app-tauri/lib/hooks/timeline-frame-merge.ts
@@ -17,6 +17,157 @@ export interface TimelineFrameMergeResult<T extends TimelineFrameLike> {
const compareFramesDesc = <T extends TimelineFrameLike>(a: T, b: T) =>
b.timestamp.localeCompare(a.timestamp);
+type TimelineDeviceLike = {
+ device_id?: unknown;
+ frame_id?: unknown;
+ audio?: unknown[];
+ [key: string]: unknown;
+};
+
+type TimelineAudioLike = {
+ audio_chunk_id?: unknown;
+ transcription?: unknown;
+ [key: string]: unknown;
+};
+
+type TimelineFrameWithDevices = TimelineFrameLike & {
+ devices?: TimelineDeviceLike[];
+ [key: string]: unknown;
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+ typeof value === "object" && value !== null;
+
+const getDevices = <T extends TimelineFrameLike>(
+ frame: T,
+): TimelineDeviceLike[] | null => {
+ if (!isRecord(frame) || !Array.isArray(frame.devices)) return null;
+ return frame.devices.filter(isRecord) as TimelineDeviceLike[];
+};
+
+const deviceKey = (device: TimelineDeviceLike): string | null => {
+ if (device.device_id != null) return `device:${String(device.device_id)}`;
+ if (device.frame_id != null) return `frame:${String(device.frame_id)}`;
+ return null;
+};
+
+const audioKey = (audio: unknown): string | null => {
+ if (!isRecord(audio)) return null;
+ const entry = audio as TimelineAudioLike;
+ if (entry.audio_chunk_id != null) return `chunk:${String(entry.audio_chunk_id)}`;
+ return null;
+};
+
+const transcriptionLength = (audio: unknown): number => {
+ if (!isRecord(audio)) return 0;
+ const transcription = (audio as TimelineAudioLike).transcription;
+ return typeof transcription === "string" ? transcription.trim().length : 0;
+};
+
+function mergeAudioEntries(
+ existingAudio: unknown[] | undefined,
+ incomingAudio: unknown[] | undefined,
+): { audio: unknown[]; changed: boolean } {
+ if (!Array.isArray(incomingAudio) || incomingAudio.length === 0) {
+ return { audio: existingAudio ?? [], changed: false };
+ }
+
+ const merged = Array.isArray(existingAudio) ? [...existingAudio] : [];
+ const byKey = new Map<string, number>();
+
+ for (let i = 0; i < merged.length; i++) {
+ const key = audioKey(merged[i]);
+ if (key) byKey.set(key, i);
+ }
+
+ let changed = false;
+ for (const incoming of incomingAudio) {
+ const key = audioKey(incoming);
+ if (!key) {
+ if (!merged.includes(incoming)) {
+ merged.push(incoming);
+ changed = true;
+ }
+ continue;
+ }
+
+ const existingIndex = byKey.get(key);
+ if (existingIndex == null) {
+ byKey.set(key, merged.length);
+ merged.push(incoming);
+ changed = true;
+ continue;
+ }
+
+ const existing = merged[existingIndex];
+ if (transcriptionLength(incoming) > transcriptionLength(existing)) {
+ merged[existingIndex] = incoming;
+ changed = true;
+ }
+ }
+
+ return { audio: merged, changed };
+}
+
+function mergeSameTimestampFrame<T extends TimelineFrameLike>(
+ existingFrame: T,
+ incomingFrame: T,
+): { frame: T; changed: boolean } {
+ const incomingDevices = getDevices(incomingFrame);
+ if (!incomingDevices?.length) return { frame: existingFrame, changed: false };
+
+ const existingDevices = getDevices(existingFrame) ?? [];
+ let nextDevices = existingDevices;
+ let changed = false;
+ const existingDeviceByKey = new Map<string, number>();
+
+ for (let i = 0; i < existingDevices.length; i++) {
+ const key = deviceKey(existingDevices[i]);
+ if (key) existingDeviceByKey.set(key, i);
+ }
+
+ const ensureDevicesCopy = () => {
+ if (nextDevices === existingDevices) nextDevices = [...existingDevices];
+ };
+
+ for (const incomingDevice of incomingDevices) {
+ if (!Array.isArray(incomingDevice.audio) || incomingDevice.audio.length === 0) {
+ continue;
+ }
+
+ const key = deviceKey(incomingDevice);
+ const existingIndex = key ? existingDeviceByKey.get(key) : undefined;
+ if (existingIndex == null) {
+ ensureDevicesCopy();
+ if (key) existingDeviceByKey.set(key, nextDevices.length);
+ nextDevices.push(incomingDevice);
+ changed = true;
+ continue;
+ }
+
+ const existingDevice = nextDevices[existingIndex];
+ const audioMerge = mergeAudioEntries(existingDevice.audio, incomingDevice.audio);
+ if (!audioMerge.changed) continue;
+
+ ensureDevicesCopy();
+ nextDevices[existingIndex] = {
+ ...existingDevice,
+ audio: audioMerge.audio,
+ };
+ changed = true;
+ }
+
+ if (!changed) return { frame: existingFrame, changed: false };
+
+ return {
+ frame: {
+ ...(existingFrame as unknown as TimelineFrameWithDevices),
+ devices: nextDevices,
+ } as unknown as T,
+ changed: true,
+ };
+}
+
function mergeSortedDesc<T extends TimelineFrameLike>(
existingFrames: T[],
newFrames: T[],
@@ -77,14 +228,53 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({
}): TimelineFrameMergeResult<T> {
const timestamps = replace ? new Set<string>() : existingTimestamps;
const newUniqueFrames: T[] = [];
+ const frameLocations = new Map<
+ string,
+ { source: "existing" | "new"; index: number }
+ >();
+ let updatedExistingFrames = existingFrames;
+ let existingChanged = false;
+
+ if (!replace) {
+ for (let i = 0; i < existingFrames.length; i++) {
+ frameLocations.set(existingFrames[i].timestamp, {
+ source: "existing",
+ index: i,
+ });
+ }
+ }
for (const frame of incomingFrames) {
- if (timestamps.has(frame.timestamp)) continue;
+ const location = frameLocations.get(frame.timestamp);
+ if (location) {
+ const currentFrame =
+ location.source === "existing"
+ ? updatedExistingFrames[location.index]
+ : newUniqueFrames[location.index];
+ const mergedFrame = mergeSameTimestampFrame(currentFrame, frame);
+ if (!mergedFrame.changed) continue;
+
+ if (location.source === "existing") {
+ if (updatedExistingFrames === existingFrames) {
+ updatedExistingFrames = [...existingFrames];
+ }
+ updatedExistingFrames[location.index] = mergedFrame.frame;
+ existingChanged = true;
+ } else {
+ newUniqueFrames[location.index] = mergedFrame.frame;
+ }
+ continue;
+ }
+
timestamps.add(frame.timestamp);
+ frameLocations.set(frame.timestamp, {
+ source: "new",
+ index: newUniqueFrames.length,
+ });
newUniqueFrames.push(frame);
}
- if (newUniqueFrames.length === 0) {
+ if (newUniqueFrames.length === 0 && !existingChanged) {
return {
frames: existingFrames,
timestamps,
@@ -94,11 +284,21 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({
};
}
+ if (newUniqueFrames.length === 0) {
+ return {
+ frames: updatedExistingFrames,
+ timestamps,
+ newUniqueFrames,
+ newAtFront: 0,
+ changed: true,
+ };
+ }
+
const sortedNewFrames =
newUniqueFrames.length > 1
? [...newUniqueFrames].sort(compareFramesDesc)
: newUniqueFrames;
- const previousNewest = replace ? undefined : existingFrames[0]?.timestamp;
+ const previousNewest = replace ? undefined : updatedExistingFrames[0]?.timestamp;
const newAtFront = previousNewest
? sortedNewFrames.filter(
(frame) => frame.timestamp.localeCompare(previousNewest) > 0,
@@ -108,7 +308,7 @@ export function mergeTimelineFrames<T extends TimelineFrameLike>({
return {
frames: replace
? sortedNewFrames
- : mergeSortedDesc(existingFrames, sortedNewFrames),
+ : mergeSortedDesc(updatedExistingFrames, sortedNewFrames),
timestamps,
newUniqueFrames: sortedNewFrames,
newAtFront,