YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: hold live transcript steady while reading older speech
Louis Beaumontc28da56f066b
diff --git a/apps/screenpipe-app-tauri/app/shortcut-reminder/page.tsx b/apps/screenpipe-app-tauri/app/shortcut-reminder/page.tsxindex b34bfe740..665cff07d 100644--- a/apps/screenpipe-app-tauri/app/shortcut-reminder/page.tsx+++ b/apps/screenpipe-app-tauri/app/shortcut-reminder/page.tsx@@ -29,6 +29,7 @@ import {import { useStarredSessions } from "@/components/starred-sessions/use-starred-sessions";import { useOverlayData } from "./use-overlay-data";import { useMeetingOverlay } from "./use-meeting-overlay";+import { TranscriptHistory } from "./transcript-history";import { AudioEqualizer } from "./audio-equalizer";import { formatShortcut, type ShortcutPlatform } from "./format-shortcut";import { DragStage } from "./drag-stage";@@ -76,7 +77,7 @@ const COLLAPSED_SIZE = { width: 22, height: 16 };const EXPANDED_SIZE = { width: 192, height: 62 };const SETTINGS_MENU_SIZE = { width: 164, height: 96 };const INCIDENT_SIZE = { width: 160, height: 40 };-const MEETING_SIZE = { width: 280, height: 80 };+const MEETING_SIZE = { width: 320, height: 160 };export default function ShortcutReminderPage() {@@ -519,11 +520,6 @@ export default function ShortcutReminderPage() {// Use sentence case for both the pill and its accessible name.const failureHeadlineSentence =failureHeadline.charAt(0).toUpperCase() + failureHeadline.slice(1);- const latestTranscript = meetingOverlay.items.at(-1);- const latestSpeaker = latestTranscript- ? latestTranscript.speakerName ||- (latestTranscript.deviceType === "input" ? "me" : "speaker")- : null;// While a drag is in flight the window covers every monitor, so the stage is// the whole surface. It outranks the health and meeting states rather than@@ -703,7 +699,7 @@ export default function ShortcutReminderPage() {style={{background: "rgba(0, 0, 0, 0.92)",display: "grid",- gridTemplateRows: "1fr 1px 1fr",+ gridTemplateRows: `${32 * overlayScale}px 1px minmax(0, 1fr)`,}}><div className="flex items-center min-w-0" style={{ padding: `0 ${padX}px`, gap: `${gap * 2}px` }}>@@ -763,14 +759,13 @@ export default function ShortcutReminderPage() {</button></div><div className="bg-white/15" />- <div className="flex items-center min-w-0" style={{ padding: `0 ${padX}px`, gap: `${gap * 2}px` }}>- <span className="font-mono text-white/40 shrink-0" style={{ fontSize: `${fontPx}px` }}>- {meetingOverlay.stopError ? ui("Stop failed") : latestSpeaker || ui("Live transcript")}- </span>- <span className="font-mono text-white/80 truncate" style={{ fontSize: `${fontPx}px` }}>- {meetingOverlay.stopError || latestTranscript?.text || ui("Listening for speech…")}- </span>- </div>+ {meetingOverlay.stopError || meetingOverlay.items.length === 0 ? (+ <div className="font-mono text-white/80" style={{ padding: padX, fontSize: fontPx }}>+ {meetingOverlay.stopError || ui("Listening for speech…")}+ </div>+ ) : (+ <TranscriptHistory key={liveMeetingId} items={meetingOverlay.items} scale={overlayScale} />+ )}</div></div>);diff --git a/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.test.tsx b/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.test.tsxnew file mode 100644index 000000000..90ca5533e--- /dev/null+++ b/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.test.tsx@@ -0,0 +1,69 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { cleanup, fireEvent, render, screen } from "@testing-library/react";+import { afterEach, describe, expect, it, vi } from "vitest";+import { TranscriptHistory } from "./transcript-history";+import type { MeetingOverlayTranscriptItem } from "./use-meeting-overlay";++const item = (i: number, text = `Turn ${i}`): MeetingOverlayTranscriptItem => ({+ meetingId: 1, itemId: `${i}`, deviceName: "fixture", deviceType: "output",+ speakerName: "Speaker", text, capturedAt: "2026-10-13T10:00:00Z", isFinal: true,+});+afterEach(() => { cleanup(); vi.restoreAllMocks(); });++describe("overlay transcript following", () => {+ it("holds even a small upward scroll through new and partial speech, then resumes at bottom", () => {+ let height = 600;+ vi.spyOn(HTMLElement.prototype, "scrollHeight", "get").mockImplementation(() => height);+ vi.spyOn(HTMLElement.prototype, "clientHeight", "get").mockReturnValue(120);+ const view = render(<TranscriptHistory items={[item(1)]} scale={1} />);+ const el = screen.getByRole("region", { name: "Live transcript" });+ expect(el.scrollTop).toBe(height);+ el.scrollTop = 478;+ fireEvent.scroll(el);+ height = 650;+ view.rerender(<TranscriptHistory items={[item(1), item(2)]} scale={1} />);+ expect(el.scrollTop).toBe(478);+ expect(screen.queryByText("Turn 2")).toBeNull();+ height = 700;+ view.rerender(<TranscriptHistory items={[item(1), item(2, "More partial speech wrapping onto another line")]} scale={1} />);+ expect(el.scrollTop).toBe(478);+ el.scrollTop = height - 120;+ fireEvent.scroll(el);+ height = 800;+ view.rerender(<TranscriptHistory items={[item(1), item(2), item(3)]} scale={1} />);+ expect(el.scrollTop).toBe(height);+ expect(screen.getByText("Turn 3")).toBeTruthy();+ });++ it("holds the reading snapshot when the bounded live feed drops older entries", () => {+ vi.spyOn(HTMLElement.prototype, "scrollHeight", "get").mockReturnValue(600);+ vi.spyOn(HTMLElement.prototype, "clientHeight", "get").mockReturnValue(120);+ const view = render(<TranscriptHistory items={[item(1), item(2)]} scale={1} />);+ const el = screen.getByRole("region");+ el.scrollTop = 100;+ fireEvent.scroll(el);+ view.rerender(<TranscriptHistory items={[item(2), item(3)]} scale={1} />);+ expect(el.scrollTop).toBe(100);+ expect(screen.getByText("Turn 1")).toBeTruthy();+ expect(screen.queryByText("Turn 3")).toBeNull();+ el.scrollTop = 480;+ fireEvent.scroll(el);+ expect(screen.queryByText("Turn 1")).toBeNull();+ expect(screen.getByText("Turn 3")).toBeTruthy();+ expect(el.scrollTop).toBe(600);+ });++ it("starts at the latest text after collapsing and reopening", () => {+ vi.spyOn(HTMLElement.prototype, "scrollHeight", "get").mockReturnValue(600);+ vi.spyOn(HTMLElement.prototype, "clientHeight", "get").mockReturnValue(120);+ const view = render(<TranscriptHistory items={[item(1)]} scale={1} />);+ const el = screen.getByRole("region");+ el.scrollTop = 0;+ fireEvent.scroll(el);+ view.unmount();+ render(<TranscriptHistory items={[item(1), item(2)]} scale={1} />);+ expect(screen.getByRole("region").scrollTop).toBe(600);+ });+});diff --git a/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.tsx b/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.tsxnew file mode 100644index 000000000..981327fea--- /dev/null+++ b/apps/screenpipe-app-tauri/app/shortcut-reminder/transcript-history.tsx@@ -0,0 +1,47 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { useLayoutEffect, useRef, useState } from "react";+import type { MeetingOverlayTranscriptItem } from "./use-meeting-overlay";++/** Mounted only while the card is open, so reopening resumes at the latest text. */+export function TranscriptHistory({ items, scale }: {+ items: MeetingOverlayTranscriptItem[];+ scale: number;+}) {+ const viewport = useRef<HTMLDivElement>(null);+ // Preserve the words as well as their pixel position when partial speech+ // reflows or older entries fall out of the bounded live feed.+ const [pausedItems, setPausedItems] = useState<MeetingOverlayTranscriptItem[] | null>(null);++ useLayoutEffect(() => {+ const el = viewport.current;+ if (el && pausedItems === null) el.scrollTop = el.scrollHeight;+ }, [items, pausedItems, scale]);++ return (+ <div+ ref={viewport}+ role="region"+ tabIndex={0}+ aria-label="Live transcript"+ className="min-h-0 overflow-y-auto font-mono text-white/80"+ style={{ padding: `${8 * scale}px ${10 * scale}px`, fontSize: `${8 * scale}px`, overflowAnchor: "none" }}+ onScroll={(event) => {+ const el = event.currentTarget;+ const atBottom = el.scrollHeight - el.clientHeight - el.scrollTop <= 0.5;+ setPausedItems((previous) => atBottom ? null : previous ?? items);+ }}+ onPointerDown={(event) => event.stopPropagation()}+ >+ {(pausedItems ?? items).map((item) => (+ <div key={`${item.deviceName}:${item.deviceType}:${item.itemId}`} className="flex items-baseline" style={{ gap: 7 * scale, marginBottom: 7 * scale }}>+ <span className="shrink-0 text-white/40 truncate text-right" style={{ width: 48 * scale }}>+ {item.speakerName || (item.deviceType === "input" ? "me" : "speaker")}+ </span>+ <span className="min-w-0 whitespace-pre-wrap break-words" style={{ opacity: item.isFinal ? 1 : 0.7 }}>{item.text}</span>+ </div>+ ))}+ </div>+ );+}diff --git a/apps/screenpipe-app-tauri/scripts/test-meeting-transcript-scroll.sh b/apps/screenpipe-app-tauri/scripts/test-meeting-transcript-scroll.shindex 7a35ab786..98f65bc28 100755--- a/apps/screenpipe-app-tauri/scripts/test-meeting-transcript-scroll.sh+++ b/apps/screenpipe-app-tauri/scripts/test-meeting-transcript-scroll.sh@@ -29,6 +29,7 @@ elsefixcrun swiftc \+ -DOVERLAY_PREVIEW \-swift-version 5 \-sdk "$sdk_path" \-target "$swift_target" \diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/meeting_transcript_scroll_tests.swift b/apps/screenpipe-app-tauri/src-tauri/swift/meeting_transcript_scroll_tests.swiftindex 360b6c34d..f46702c0e 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/meeting_transcript_scroll_tests.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/meeting_transcript_scroll_tests.swift@@ -30,6 +30,7 @@ struct MeetingTranscriptScrollTests {for scale in [CGFloat(1), 1.5, 2] {run(scale: scale)}+ testControllerReopen()print("meeting transcript scroll: \(checks) checks passed")}static func run(scale: CGFloat) {@@ -61,6 +62,14 @@ struct MeetingTranscriptScrollTests {expect(doc.frame.height > scroll.contentView.bounds.height * 3, "full history must overflow")expect(atBottom(), "open on latest transcript")expect(!panel.isKeyWindow, "test panel must never take focus")+ scroll.contentView.scroll(to: NSPoint(x: 0, y: scroll.contentView.bounds.minY - 2))+ scroll.reflectScrolledClipView(scroll.contentView)+ pump()+ let nearBottomOffset = scroll.contentView.bounds.minY+ host.rootView = MeetingTranscriptPreview(metrics: metrics, scale: scale, onOpenNote: {}, onStop: {}, onTogglePin: {})+ pump()+ expect(abs(scroll.contentView.bounds.minY - nearBottomOffset) < 1, "even a small upward scroll must survive a stream refresh")+scroll.contentView.scroll(to: NSPoint(x: 0, y: 0))scroll.reflectScrolledClipView(scroll.contentView)pump()@@ -78,11 +87,12 @@ struct MeetingTranscriptScrollTests {text: partial.text + " " + partial.text, capturedAt: partial.capturedAt, isFinal: false)pump()- expect(doc.frame.height > previousHeight, "growing partial text must wrap fully")+ expect(doc.frame.height == previousHeight, "partial text must not reflow the reading snapshot")expect(scroll.contentView.bounds.minY == 0, "partial updates must preserve reading position")scroll.contentView.scroll(to: NSPoint(x: 0, y: doc.frame.height - scroll.contentView.bounds.height))scroll.reflectScrolledClipView(scroll.contentView)pump()+ expect(doc.frame.height > previousHeight, "returning to bottom flushes queued partial speech without another event")metrics.meetingTranscriptItems.append(item(14))pump()expect(atBottom(), "returning to bottom must resume following new speech")@@ -97,8 +107,11 @@ struct MeetingTranscriptScrollTests {metrics.meetingTranscriptItems = (1...50).map(item)pump()expect(abs(scroll.contentView.bounds.minY - wheelOffset) <= 1, "snapshot preserves reading position")+ let pausedHeight = doc.frame.heightmetrics.meetingTranscriptItems = (2...51).map(item)pump()+ expect(doc.frame.height == pausedHeight, "rolling history does not replace the reading snapshot")+ expect(abs(scroll.contentView.bounds.minY - wheelOffset) <= 1, "rolling history keeps the same reading offset")expect(!atBottom(), "rolling history must not force reader to latest")metrics.activeMeetingId = 2pump()@@ -119,4 +132,40 @@ struct MeetingTranscriptScrollTests {expect(scrollView(host)?.contentView.bounds.minY == 0, "short transcript remains at top")panel.close()}+ static func testControllerReopen() {+ // Exercise the actual hide/show controller with synthetic streaming+ // speech behind other apps, with both network routes disconnected.+ setenv("SCREENPIPE_E2E_NON_INTRUSIVE", "1", 1)+ let controller = ShortcutReminderController()+ controller.show(shortcuts: "{\"metrics_ws_url\":\"ws://127.0.0.1:9/fixture\",\"events_ws_url\":\"ws://127.0.0.1:9/fixture\"}")+ controller.setPreviewPinnedMeeting()+ RunLoop.main.run(until: Date().addingTimeInterval(8))+ guard let panel = NSApplication.shared.windows.first(where: { window in+ window.isVisible && (window.contentView.map { scrollView($0) != nil } ?? false)+ }), let content = panel.contentView, let scroll = scrollView(content), let doc = scroll.documentView else {+ expect(false, "controller must show a scrollable transcript")+ return+ }+ expect(doc.frame.height > scroll.contentView.bounds.height, "controller fixture must overflow")+ scroll.contentView.scroll(to: NSPoint(x: 0, y: 0))+ scroll.reflectScrolledClipView(scroll.contentView)+ RunLoop.main.run(until: Date().addingTimeInterval(1))+ expect(scroll.contentView.bounds.minY == 0, "controller refresh preserves older speech")+ controller.toggleMeetingPin()+ pump()+ expect(!panel.isVisible, "unpin away from the card collapses it")+ controller.setPillHoveringExternally(true)+ pump()+ expect(panel.isVisible, "hover reopens the same transcript panel")+ guard let reopened = scrollView(content), let reopenedDoc = reopened.documentView else {+ expect(false, "reopened controller must have a transcript")+ return+ }+ expect(abs(reopenedDoc.frame.height - reopened.contentView.bounds.maxY) <= 1,+ "collapse then reopen must reset to latest speech")+ expect(!panel.isKeyWindow, "controller test must not take focus")+ controller.hide()+ pump()+ }+}diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift b/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swiftindex 7369cc891..5214bc4aa 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift@@ -1250,6 +1250,7 @@ private final class MeetingTranscriptScrollView: NSScrollView {private var hasContentLayout = falseprivate var contentNeedsLayout = trueprivate var lastViewportSize = NSSize.zero+ private var scrollObserver: NSObjectProtocol?init() {super.init(frame: .zero)@@ -1260,6 +1261,19 @@ private final class MeetingTranscriptScrollView: NSScrollView {scrollerStyle = .overlayscrollerKnobStyle = .lightdocumentView = hosting+ contentView.postsBoundsChangedNotifications = true+ scrollObserver = NotificationCenter.default.addObserver(+ forName: NSView.boundsDidChangeNotification, object: contentView, queue: .main+ ) { [weak self] _ in+ // Reaching the end must flush queued speech even if the stream has+ // gone quiet. Layout itself also changes bounds, so avoid recursion.+ guard let self, !self.layingOutContent else { return }+ self.needsLayout = true+ }+ }++ deinit {+ if let scrollObserver { NotificationCenter.default.removeObserver(scrollObserver) }}required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }@@ -1278,12 +1292,16 @@ private final class MeetingTranscriptScrollView: NSScrollView {private func layoutContent() {guard !layingOutContent, contentSize.width > 0, contentSize.height > 0 else { return }guard contentNeedsLayout || contentSize != lastViewportSize else { return }+ let oldOffset = contentView.bounds.origin+ let followsLatest = !hasContentLayout || hosting.frame.height - contentView.bounds.maxY <= 0.5+ // Keep a stable reading snapshot while paused. Otherwise partial text+ // reflow or eviction from the rolling feed moves the words under the+ // reader even when the numeric scroll offset stays unchanged.+ guard followsLatest || contentSize != lastViewportSize else { return }contentNeedsLayout = falselastViewportSize = contentSizelayingOutContent = truedefer { layingOutContent = false }- let oldOffset = contentView.bounds.origin- let followsLatest = !hasContentLayout || hosting.frame.height - contentView.bounds.maxY <= 4hosting.rootView = AnyView(content.frame(width: contentSize.width).fixedSize(horizontal: false, vertical: true))let height = hosting.fittingSize.heighthosting.setFrameSize(NSSize(width: contentSize.width, height: height))@@ -2576,6 +2594,12 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {if transcriptPanel == nil {createTranscriptPanel()}+ if transcriptPanel?.isVisible != true {+ // A reopened card starts at the latest speech. Streaming refreshes+ // keep the existing native scroll view and its reading position.+ transcriptHostingView?.removeFromSuperview()+ transcriptHostingView = nil+ }updateTranscriptContent()positionTranscriptPanel()gatedOrderFront(transcriptPanel)