YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(meetings): make the replay progress bar seekable and stop blanking padded meetings (#6670)
Louis Beaumont4e55cdcaba1f
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.test.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.test.tsxindex 3cda4c079..759258727 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.test.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.test.tsx@@ -209,6 +209,70 @@ describe("ReplayStrip", () => {expect(container.querySelector("audio, video")).toBeNull();});+ it("still shows the earliest frame when the cursor precedes the first capture", async () => {+ // Meeting bounds are padded (meeting_start can be minutes before the+ // first captured frame). Seeking to the very start must not blank the+ // player with "no screen images available" while frames exist.+ mocks.fetchFrameSamples.mockResolvedValue([+ { frameId: 1, timestamp: transcriptAt },+ { frameId: 2, timestamp: secondFrameAt },+ ]);++ renderReplayStrip();++ await waitFor(() =>+ expect(screen.getByTestId("replay-active-frame")).toHaveAttribute(+ "data-frame-id",+ "1",+ ),+ );++ // Home seeks to rangeStart (meetingStart), 2 minutes before frame 1.+ fireEvent.keyDown(screen.getByTestId("replay-scrubber"), { key: "Home" });++ expect(screen.getByTestId("replay-scrubber")).toHaveAttribute(+ "aria-valuenow",+ "0",+ );+ expect(screen.getByTestId("replay-active-frame")).toHaveAttribute(+ "data-frame-id",+ "1",+ );+ expect(+ screen.queryByText(/no screen images available/i),+ ).not.toBeInTheDocument();+ });++ it("seeks when clicking the control-bar progress slider", async () => {+ mocks.fetchFrameSamples.mockResolvedValue([+ { frameId: 1, timestamp: transcriptAt },+ { frameId: 2, timestamp: secondFrameAt },+ ]);++ renderReplayStrip();++ await waitFor(() =>+ expect(screen.getByTestId("replay-active-frame")).toHaveAttribute(+ "data-frame-id",+ "1",+ ),+ );++ const bar = screen.getByLabelText("silent replay progress");+ bar.getBoundingClientRect = () =>+ ({ left: 0, width: 200, top: 0, height: 28, right: 200, bottom: 28 }) as DOMRect;+ // jsdom has no PointerEvent; a MouseEvent named "pointerdown" carries+ // clientX and still triggers React's onPointerDown.+ fireEvent(bar, new MouseEvent("pointerdown", { clientX: 100, bubbles: true }));++ // Meeting is 10 minutes; clicking the middle seeks to ~300s.+ const valueNow = Number(+ screen.getByTestId("replay-scrubber").getAttribute("aria-valuenow"),+ );+ expect(valueNow).toBeGreaterThanOrEqual(295);+ expect(valueNow).toBeLessThanOrEqual(305);+ });+it("plays captured frames once in timestamp order and pauses cleanly", async () => {const start = "2026-06-12T10:00:00.000Z";const next = "2026-06-12T10:00:01.000Z";diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsxindex 31b3bc0e3..b2e8893c3 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/replay-strip.tsx@@ -88,6 +88,25 @@ export function formatReplayElapsed(ms: number): string {return `${minutes}:${String(seconds).padStart(2, "0")}`;}+// Pointer capture keeps a drag alive outside the element, but the id is only+// capturable while that pointer is active. Treat it as best effort so a+// non-capturable pointer can never break seeking.+function capturePointer(el: Element, pointerId: number) {+ try {+ el.setPointerCapture?.(pointerId);+ } catch {+ // pointer already released or never active+ }+}++function releasePointer(el: Element, pointerId: number) {+ try {+ el.releasePointerCapture?.(pointerId);+ } catch {+ // nothing captured+ }+}+export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps) {// Bounds from notable-quotes sample (cheap, already loaded).const sampleSorted = useMemo(@@ -183,6 +202,7 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripPropsconst updateCursor = useCallback((nextMs: number) => {+ if (!Number.isFinite(nextMs)) return;const clamped = Math.min(Math.max(nextMs, rangeStartMs), rangeEndMs);cursorRef.current = clamped;setCursorMs(clamped);@@ -336,7 +356,13 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps}, [renderableFrames, rangeStartMs, durationMs]);const activeFrame = useMemo<FrameSample | null>(() => {- return frameAtOrBefore(renderableFrames, cursorMs);+ // Meeting bounds are padded and capture can start after meeting_start,+ // so a cursor before the first captured frame is normal. Clamp to the+ // earliest frame instead of blanking the player with "no screen images+ // available" while frames exist.+ return (+ frameAtOrBefore(renderableFrames, cursorMs) ?? renderableFrames[0] ?? null+ );}, [renderableFrames, cursorMs]);const activeChunk = useMemo<MeetingAudioChunk | null>(() => {@@ -354,13 +380,13 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps100,);- const trackRef = useRef<HTMLDivElement | null>(null);const draggingRef = useRef(false);+ // Seek relative to whichever seek surface (thumbnail scrubber or+ // control-bar progress slider) received the pointer event.const seekFromEvent = useCallback(- (clientX: number) => {- const el = trackRef.current;- if (!el) return;+ (el: Element, clientX: number) => {const rect = el.getBoundingClientRect();+ if (rect.width <= 0) return;const ratio = Math.min(Math.max((clientX - rect.left) / rect.width, 0), 1);updateCursor(rangeStartMs + ratio * durationMs);},@@ -370,20 +396,22 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps(e: React.PointerEvent) => {setIsPlaying(false);draggingRef.current = true;- (e.target as Element).setPointerCapture?.(e.pointerId);- seekFromEvent(e.clientX);+ // Seek first: pointer capture is a best-effort drag enhancement and must+ // never be able to swallow the seek it precedes.+ seekFromEvent(e.currentTarget, e.clientX);+ capturePointer(e.target as Element, e.pointerId);},[seekFromEvent],);const onPointerMove = useCallback((e: React.PointerEvent) => {- if (draggingRef.current) seekFromEvent(e.clientX);+ if (draggingRef.current) seekFromEvent(e.currentTarget, e.clientX);},[seekFromEvent],);const onPointerUp = useCallback((e: React.PointerEvent) => {draggingRef.current = false;- (e.target as Element).releasePointerCapture?.(e.pointerId);+ releasePointer(e.target as Element, e.pointerId);}, []);const onScrubberKeyDown = useCallback(@@ -542,17 +570,26 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps{formatReplayElapsed(durationMs)}</span><div- role="progressbar"+ role="slider"+ tabIndex={0}aria-label="silent replay progress"aria-valuemin={0}aria-valuemax={Math.round(durationMs / 1000)}aria-valuenow={Math.round((cursorMs - rangeStartMs) / 1000)}- className="relative h-px flex-1 bg-white/30"+ aria-valuetext={`${formatReplayElapsed(cursorMs - rangeStartMs)} of ${formatReplayElapsed(durationMs)}`}+ className="relative flex h-full flex-1 cursor-pointer touch-none items-center focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-white"+ onPointerDown={onPointerDown}+ onPointerMove={onPointerMove}+ onPointerUp={onPointerUp}+ onPointerCancel={onPointerUp}+ onKeyDown={onScrubberKeyDown}>- <div- className="absolute inset-y-0 left-0 bg-white"- style={{ width: `${cursorPct}%` }}- />+ <div className="relative h-px w-full bg-white/30">+ <div+ className="absolute inset-y-0 left-0 bg-white"+ style={{ width: `${cursorPct}%` }}+ />+ </div></div><buttontype="button"@@ -619,7 +656,6 @@ export function ReplayStrip({ meetingId, segments, timeRange }: ReplayStripProps{/* Thumbnail strip + scrubber */}<div- ref={trackRef}data-testid="replay-scrubber"role="slider"tabIndex={0}diff --git a/apps/screenpipe-app-tauri/e2e/specs/meeting-replay-player.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/meeting-replay-player.spec.tsindex 8b89e2279..42de14b65 100644--- a/apps/screenpipe-app-tauri/e2e/specs/meeting-replay-player.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/meeting-replay-player.spec.ts@@ -120,7 +120,12 @@ describe("historical meeting silent replay player", function () {method: "PUT",body: JSON.stringify({title: TITLE,- meeting_start: new Date(firstMs - 1_000).toISOString(),+ // 30s of pre-capture padding: meeting bounds regularly start before+ // the first captured frame, and seeking into that region must clamp+ // to the earliest frame instead of showing "no screen images+ // available". Keep it under 60s — the next seeded fixture frame+ // lives at now-1min and must stay outside the replay window.+ meeting_start: new Date(firstMs - 30_000).toISOString(),meeting_end: new Date(lastMs + 1_000).toISOString(),note: "",}),@@ -214,6 +219,77 @@ describe("historical meeting silent replay player", function () {),).toBe(pausedFrameId);+ // Both regressions are position-dependent, and this harness cannot deliver+ // a positioned press: WebKit WebDriver synthesizes a bare click with+ // clientX 0 and no pointerdown/mousedown at all (verified against a live+ // listener), and browser.keys() never reaches a focused div. So drive the+ // real sliders in the real build with real PointerEvents at real+ // coordinates read off the live layout. Both surfaces are addressed by+ // attributes that predate the fix, so this spec fails on the buggy build.+ const scrubber = await waitForTestId("replay-scrubber", 5_000);+ const progress = await $('[aria-label="silent replay progress"]');+ await progress.waitForExist({ timeout: t(5_000) });+ const valueMax = Number(await scrubber.getAttribute("aria-valuemax"));+ expect(valueMax).toBeGreaterThan(0);+ const pressSlider = async (element: WebdriverIO.Element, ratio: number) => {+ await browser.execute(+ (el: HTMLElement, atRatio: number) => {+ const rect = el.getBoundingClientRect();+ const init = {+ bubbles: true,+ clientX: rect.left + rect.width * atRatio,+ clientY: rect.top + rect.height / 2,+ pointerId: 1,+ isPrimary: true,+ };+ el.dispatchEvent(new PointerEvent("pointerdown", init));+ el.dispatchEvent(new PointerEvent("pointerup", init));+ },+ element,+ ratio,+ );+ };++ // Regression (silent-blank bug): the start of the track is inside the+ // padded pre-capture region (meeting_start is 30s before the first+ // captured frame). Seeking there must clamp to the earliest frame instead+ // of blanking the player with "no screen images available". Driven through+ // the thumbnail scrubber, which already seeked correctly before the fix,+ // so this isolates the blanking bug from the dead-slider bug below.+ await pressSlider(scrubber, 0);+ await browser.waitUntil(+ async () => (await scrubber.getAttribute("aria-valuenow")) === "0",+ {+ timeout: t(5_000),+ timeoutMsg:+ "pressing the scrubber start did not seek to the meeting start",+ },+ );+ expect((await player.getText()).toLowerCase()).not.toContain(+ "no screen images available",+ );+ expect(+ await $("[data-testid='replay-active-frame']").getAttribute(+ "data-frame-id",+ ),+ ).toBe(firstFrameId);++ // Regression (unclickable slider bug): the control-bar progress line was a+ // bare progressbar with no seek handling, so pressing it did nothing at+ // any position. It is a real slider now - pressing its middle seeks to+ // ~50% of the meeting.+ await pressSlider(progress, 0.5);+ await browser.waitUntil(+ async () => {+ const now = Number(await scrubber.getAttribute("aria-valuenow"));+ return now >= valueMax * 0.4 && now <= valueMax * 0.6;+ },+ {+ timeout: t(5_000),+ timeoutMsg: "pressing the middle of the progress slider did not seek",+ },+ );+const selectedTabs = (await browser.execute(() =>Array.from(document.querySelectorAll<HTMLElement>(