Create polished demo videos without editing skills. Mac/Windows/Linux
fix(recording): account for mic pause gaps
wiiiii1233d6668fd60ae
diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.tsindex 903c7722..21192e91 100644--- a/electron/ipc/recording/diagnostics.test.ts+++ b/electron/ipc/recording/diagnostics.test.ts@@ -304,6 +304,73 @@ describe("getCompanionAudioFallbackPaths", () => {await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull();});+ it("classifies wall-clock mic chunk gaps covered by pause intervals", async () => {+ const { summarizeMicrophoneChunkTiming } = await import("./diagnostics");++ expect(+ summarizeMicrophoneChunkTiming(+ [+ {+ index: 0,+ size: 1024,+ elapsedMs: 250,+ deltaMs: null,+ recordedElapsedMs: 250,+ recordedDeltaMs: null,+ },+ {+ index: 1,+ size: 1024,+ elapsedMs: 8250,+ deltaMs: 8000,+ recordedElapsedMs: 500,+ recordedDeltaMs: 250,+ },+ ],+ [{ startElapsedMs: 250, endElapsedMs: 8250, durationMs: 7750 }],+ 250,+ ),+ ).toMatchObject({+ status: "pause-accounted",+ wallClockGapCount: 1,+ recordedGapCount: 0,+ pausedDurationMs: 7750,+ });+ });++ it("flags recorded mic chunk gaps that remain after pause accounting", async () => {+ const { summarizeMicrophoneChunkTiming } = await import("./diagnostics");++ expect(+ summarizeMicrophoneChunkTiming(+ [+ {+ index: 0,+ size: 1024,+ elapsedMs: 250,+ deltaMs: null,+ recordedElapsedMs: 250,+ recordedDeltaMs: null,+ },+ {+ index: 1,+ size: 1024,+ elapsedMs: 2500,+ deltaMs: 2250,+ recordedElapsedMs: 2500,+ recordedDeltaMs: 2250,+ },+ ],+ [],+ 250,+ ),+ ).toMatchObject({+ status: "needs-review",+ wallClockGapCount: 1,+ recordedGapCount: 1,+ });+ });+it("rejects tiny MP4 container-only outputs before they reach the editor", async () => {const videoPath = path.join(tempRoot, "recording-123.mp4");await fs.writeFile(videoPath, Buffer.alloc(261));diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.tsindex 081e2412..b8fb6860 100644--- a/electron/ipc/recording/diagnostics.ts+++ b/electron/ipc/recording/diagnostics.ts@@ -16,6 +16,21 @@ type CompanionAudioTimingMetadata = {startDelayMs?: number;};+export type MicrophoneChunkTimingEvent = {+ index: number;+ size: number;+ elapsedMs: number;+ deltaMs: number | null;+ recordedElapsedMs?: number;+ recordedDeltaMs?: number | null;+};++export type MicrophonePauseInterval = {+ startElapsedMs: number;+ endElapsedMs?: number;+ durationMs?: number;+};+export type VideoStreamDurationProbe = {durationSeconds: number | null;frameCount: number | null;@@ -102,6 +117,77 @@ export function getRecordingAudioMuxTimeoutMs(durationSeconds: number) {);}+function getFiniteNonNegativeNumber(value: unknown) {+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;+}++export function summarizeMicrophoneChunkTiming(+ chunkEvents: MicrophoneChunkTimingEvent[] | null | undefined,+ pauseIntervals: MicrophonePauseInterval[] | null | undefined = [],+ timesliceMs = 250,+) {+ const normalizedTimesliceMs =+ Number.isFinite(timesliceMs) && timesliceMs > 0 ? Math.round(timesliceMs) : 250;+ const thresholdMs = Math.max(600, Math.round(normalizedTimesliceMs * 2.5));+ const events = Array.isArray(chunkEvents) ? chunkEvents : [];+ const pauses = Array.isArray(pauseIntervals) ? pauseIntervals : [];+ const pausedDurationMs = pauses.reduce((total, interval) => {+ const durationMs = getFiniteNonNegativeNumber(interval.durationMs);+ return durationMs === null ? total : total + Math.round(durationMs);+ }, 0);++ const wallClockGaps = events+ .filter((event) => (event.deltaMs ?? 0) > thresholdMs)+ .map((event) => ({+ index: event.index,+ deltaMs: Math.round(event.deltaMs ?? 0),+ elapsedMs: Math.round(event.elapsedMs),+ }));+ const recordedGaps = events+ .filter((event) => {+ const recordedDeltaMs =+ getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0;+ return recordedDeltaMs > thresholdMs;+ })+ .map((event) => ({+ index: event.index,+ deltaMs: Math.round(+ getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0,+ ),+ recordedElapsedMs: Math.round(event.recordedElapsedMs ?? event.elapsedMs),+ }));+ const maxWallClockDeltaMs = events.reduce(+ (maxDeltaMs, event) => Math.max(maxDeltaMs, event.deltaMs ?? 0),+ 0,+ );+ const maxRecordedDeltaMs = events.reduce((maxDeltaMs, event) => {+ const recordedDeltaMs =+ getFiniteNonNegativeNumber(event.recordedDeltaMs) ?? event.deltaMs ?? 0;+ return Math.max(maxDeltaMs, recordedDeltaMs);+ }, 0);+ const status =+ recordedGaps.length > 0+ ? "needs-review"+ : wallClockGaps.length > 0 && pausedDurationMs > 0+ ? "pause-accounted"+ : "ok";++ return {+ status,+ timesliceMs: normalizedTimesliceMs,+ thresholdMs,+ eventCount: events.length,+ pauseIntervalCount: pauses.length,+ pausedDurationMs,+ maxWallClockDeltaMs: Math.round(maxWallClockDeltaMs),+ maxRecordedDeltaMs: Math.round(maxRecordedDeltaMs),+ wallClockGapCount: wallClockGaps.length,+ recordedGapCount: recordedGaps.length,+ wallClockGaps: wallClockGaps.slice(0, 10),+ recordedGaps: recordedGaps.slice(0, 10),+ };+}+/** Probe the duration of a media file (in seconds) using the container header. */export async function probeMediaDurationSeconds(filePath: string): Promise<number> {const ffmpegPath = getFfmpegBinaryPath();diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.tsindex 60db86ab..9321f5b8 100644--- a/electron/ipc/register/recording.ts+++ b/electron/ipc/register/recording.ts@@ -45,8 +45,11 @@ import {import {getCompanionAudioFallbackInfo,getFileSizeIfPresent,+ type MicrophoneChunkTimingEvent,+ type MicrophonePauseInterval,type RecordingDiagnosticsSnapshot,recordNativeCaptureDiagnostics,+ summarizeMicrophoneChunkTiming,validateRecordedVideo,writeRecordingDiagnosticsSnapshot,} from "../recording/diagnostics";@@ -189,7 +192,7 @@ function pickPrimitiveRecord(value: unknown) {return entries.length > 0 ? Object.fromEntries(entries) : null;}-function pickMicrophoneChunkEvents(value: unknown) {+function pickMicrophoneChunkEvents(value: unknown): MicrophoneChunkTimingEvent[] | null {if (!Array.isArray(value)) {return null;}@@ -200,7 +203,7 @@ function pickMicrophoneChunkEvents(value: unknown) {return null;}- const { index, size, elapsedMs, deltaMs } = event;+ const { index, size, elapsedMs, deltaMs, recordedElapsedMs, recordedDeltaMs } = event;if (typeof index !== "number" ||typeof size !== "number" ||@@ -214,6 +217,15 @@ function pickMicrophoneChunkEvents(value: unknown) {size: Math.round(size),elapsedMs: Math.round(elapsedMs),deltaMs: typeof deltaMs === "number" ? Math.round(deltaMs) : null,+ ...(typeof recordedElapsedMs === "number" &&+ Number.isFinite(recordedElapsedMs) &&+ recordedElapsedMs >= 0+ ? { recordedElapsedMs: Math.round(recordedElapsedMs) }+ : {}),+ recordedDeltaMs:+ typeof recordedDeltaMs === "number" && Number.isFinite(recordedDeltaMs)+ ? Math.max(0, Math.round(recordedDeltaMs))+ : null,};}).filter((event): event is NonNullable<typeof event> => event !== null);@@ -221,6 +233,37 @@ function pickMicrophoneChunkEvents(value: unknown) {return events.length > 0 ? events : null;}+function pickMicrophonePauseIntervals(value: unknown): MicrophonePauseInterval[] | null {+ if (!Array.isArray(value)) {+ return null;+ }++ const intervals = value+ .map((interval) => {+ if (!isRecord(interval) || typeof interval.startElapsedMs !== "number") {+ return null;+ }++ const startElapsedMs = Math.max(0, Math.round(interval.startElapsedMs));+ return {+ startElapsedMs,+ ...(typeof interval.endElapsedMs === "number" &&+ Number.isFinite(interval.endElapsedMs) &&+ interval.endElapsedMs >= startElapsedMs+ ? { endElapsedMs: Math.round(interval.endElapsedMs) }+ : {}),+ ...(typeof interval.durationMs === "number" &&+ Number.isFinite(interval.durationMs) &&+ interval.durationMs >= 0+ ? { durationMs: Math.round(interval.durationMs) }+ : {}),+ };+ })+ .filter((interval): interval is NonNullable<typeof interval> => interval !== null);++ return intervals.length > 0 ? intervals : null;+}+function pickAudioInputDevices(value: unknown) {if (!Array.isArray(value)) {return null;@@ -1415,6 +1458,7 @@ export function registerRecordingHandlers(audioInputDevices?: unknown;mediaRecorder?: unknown;chunkEvents?: unknown;+ pauseIntervals?: unknown;},) => {const baseName = videoPath.replace(/\.[^.]+$/, "");@@ -1478,6 +1522,15 @@ export function registerRecordingHandlers(}: null;const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents);+ const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals);+ const chunkTiming =+ chunkEvents || pauseIntervals+ ? summarizeMicrophoneChunkTiming(+ chunkEvents,+ pauseIntervals,+ mediaRecorder?.timesliceMs,+ )+ : null;const metadata = {...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0? { startDelayMs: Math.round(startDelayMs ?? 0) }@@ -1500,6 +1553,8 @@ export function registerRecordingHandlers(? { mediaRecorder }: {}),...(chunkEvents ? { chunkEvents } : {}),+ ...(pauseIntervals ? { pauseIntervals } : {}),+ ...(chunkTiming ? { chunkTiming } : {}),};if (Object.keys(metadata).length > 0) {try {diff --git a/src/hooks/useScreenRecorder.test.ts b/src/hooks/useScreenRecorder.test.tsindex 6a613459..e98554ad 100644--- a/src/hooks/useScreenRecorder.test.ts+++ b/src/hooks/useScreenRecorder.test.ts@@ -19,6 +19,7 @@ function createMockMediaRecorder(initialState: RecordingState = "inactive") {resume: vi.fn(() => {if (_state === "paused") _state = "recording";}),+ requestData: vi.fn(),stop: vi.fn(() => {_state = "inactive";}),@@ -151,6 +152,7 @@ function pauseRecording(webcamRecorder.pause();}if (micFallbackRecorder?.state === "recording") {+ micFallbackRecorder.requestData();micFallbackRecorder.pause();}return true;@@ -206,6 +208,7 @@ async function pauseNativeRecording(webcamRecorder.pause();}if (micFallbackRecorder?.state === "recording") {+ micFallbackRecorder.requestData();micFallbackRecorder.pause();}@@ -430,6 +433,7 @@ describe("useScreenRecorder state machine", () => {const result = pauseRecording(recorder, true, false, true, null, micFallback);expect(result).toBe(true);+ expect(micFallback.requestData).toHaveBeenCalled();expect(micFallback.state).toBe("paused");});@@ -613,6 +617,7 @@ describe("useScreenRecorder state machine", () => {const pausedResult = await pauseNativeRecording(webcam, { success: true }, micFallback);expect(pausedResult).toBe(true);expect(webcam.state).toBe("paused");+ expect(micFallback.requestData).toHaveBeenCalled();expect(micFallback.state).toBe("paused");expect(recorder.pause).not.toHaveBeenCalled();diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.tsindex d16683aa..3d7933ee 100644--- a/src/hooks/useScreenRecorder.ts+++ b/src/hooks/useScreenRecorder.ts@@ -84,6 +84,13 @@ type MicrophoneFallbackChunkEvent = {size: number;elapsedMs: number;deltaMs: number | null;+ recordedElapsedMs: number;+ recordedDeltaMs: number | null;+};+type MicrophoneFallbackPauseInterval = {+ startElapsedMs: number;+ endElapsedMs?: number;+ durationMs?: number;};type MicrophoneFallbackRecorderMetadata = {mimeType: string;@@ -99,6 +106,7 @@ type MicrophoneSidecarOptions = {audioInputDevices?: MicrophoneAudioInputDeviceSnapshot[];mediaRecorder?: MicrophoneFallbackRecorderMetadata;chunkEvents?: MicrophoneFallbackChunkEvent[];+ pauseIntervals?: MicrophoneFallbackPauseInterval[];};const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = {id: "screen:linux-portal",@@ -299,6 +307,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn {const micFallbackRecorderMetadata = useRef<MicrophoneFallbackRecorderMetadata | null>(null);const micFallbackChunkEvents = useRef<MicrophoneFallbackChunkEvent[]>([]);const micFallbackRecorderStartedAt = useRef<number | null>(null);+ const micFallbackPauseStartedAt = useRef<number | null>(null);+ const micFallbackPausedDurationMs = useRef(0);+ const micFallbackPauseIntervals = useRef<MicrophoneFallbackPauseInterval[]>([]);const browserMicrophoneProfile = useRef<BrowserMicrophoneProfile>(DEFAULT_BROWSER_MICROPHONE_PROFILE,);@@ -391,6 +402,32 @@ export function useScreenRecorder(): UseScreenRecorderReturn {});}, []);+ const getMicFallbackRecordedElapsedMs = useCallback((now = performance.now()) => {+ const startedAt = micFallbackRecorderStartedAt.current;+ if (startedAt === null) {+ return 0;+ }++ const currentPauseDurationMs =+ micFallbackPauseStartedAt.current === null+ ? 0+ : Math.max(0, now - micFallbackPauseStartedAt.current);+ return Math.max(+ 0,+ Math.round(+ now - startedAt - micFallbackPausedDurationMs.current - currentPauseDurationMs,+ ),+ );+ }, []);++ const resetMicFallbackTimingDiagnostics = useCallback(() => {+ micFallbackChunkEvents.current = [];+ micFallbackRecorderStartedAt.current = null;+ micFallbackPauseStartedAt.current = null;+ micFallbackPausedDurationMs.current = 0;+ micFallbackPauseIntervals.current = [];+ }, []);+const preparePermissions = useCallback(async (options: { startup?: boolean } = {}) => {const platform = await window.electronAPI.getPlatform();if (platform !== "darwin") {@@ -497,31 +534,40 @@ export function useScreenRecorder(): UseScreenRecorderReturn {micFallbackRequestedConstraints.current = null;micFallbackAudioInputDevices.current = null;micFallbackRecorderMetadata.current = null;- micFallbackChunkEvents.current = [];- micFallbackRecorderStartedAt.current = null;+ resetMicFallbackTimingDiagnostics();}- }, []);+ }, [resetMicFallbackTimingDiagnostics]);- const appendMicFallbackChunk = useCallback((event: BlobEvent) => {- if (event.data.size <= 0) {- return;- }+ const appendMicFallbackChunk = useCallback(+ (event: BlobEvent) => {+ if (event.data.size <= 0) {+ return;+ }- micFallbackChunks.current.push(event.data);- const startedAt = micFallbackRecorderStartedAt.current;- if (startedAt === null) {- return;- }+ micFallbackChunks.current.push(event.data);+ const startedAt = micFallbackRecorderStartedAt.current;+ if (startedAt === null) {+ return;+ }- const elapsedMs = Math.max(0, Math.round(performance.now() - startedAt));- const previous = micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1];- micFallbackChunkEvents.current.push({- index: micFallbackChunkEvents.current.length,- size: event.data.size,- elapsedMs,- deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null,- });- }, []);+ const now = performance.now();+ const elapsedMs = Math.max(0, Math.round(now - startedAt));+ const recordedElapsedMs = getMicFallbackRecordedElapsedMs(now);+ const previous =+ micFallbackChunkEvents.current[micFallbackChunkEvents.current.length - 1];+ micFallbackChunkEvents.current.push({+ index: micFallbackChunkEvents.current.length,+ size: event.data.size,+ elapsedMs,+ deltaMs: previous ? Math.max(0, elapsedMs - previous.elapsedMs) : null,+ recordedElapsedMs,+ recordedDeltaMs: previous+ ? Math.max(0, recordedElapsedMs - previous.recordedElapsedMs)+ : null,+ });+ },+ [getMicFallbackRecordedElapsedMs],+ );const resolveBrowserCaptureSource = useCallback(async (source: ProcessedDesktopSource) => {if (!source?.id?.startsWith("screen:")) {@@ -606,6 +652,27 @@ export function useScreenRecorder(): UseScreenRecorderReturn {[],);+ const closeMicFallbackPauseInterval = useCallback((now = performance.now()) => {+ const pauseStartedAt = micFallbackPauseStartedAt.current;+ if (pauseStartedAt === null) {+ return;+ }++ const durationMs = Math.max(0, Math.round(now - pauseStartedAt));+ micFallbackPausedDurationMs.current += durationMs;+ const startedAt = micFallbackRecorderStartedAt.current ?? now;+ const lastInterval =+ micFallbackPauseIntervals.current[micFallbackPauseIntervals.current.length - 1];+ if (lastInterval && lastInterval.endElapsedMs === undefined) {+ lastInterval.endElapsedMs = Math.max(+ lastInterval.startElapsedMs,+ Math.round(now - startedAt),+ );+ lastInterval.durationMs = durationMs;+ }+ micFallbackPauseStartedAt.current = null;+ }, []);+const stopMicFallbackRecorder = useCallback((): Promise<Blob | null> => {return new Promise((resolve) => {const recorder = micFallbackRecorder.current;@@ -614,6 +681,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {resolve(null);return;}+ closeMicFallbackPauseInterval();recorder.ondataavailable = appendMicFallbackChunk;recorder.onstop = () => {const blob =@@ -628,7 +696,38 @@ export function useScreenRecorder(): UseScreenRecorderReturn {};recorder.stop();});- }, [appendMicFallbackChunk]);+ }, [appendMicFallbackChunk, closeMicFallbackPauseInterval]);++ const pauseMicFallbackRecorder = useCallback(() => {+ const recorder = micFallbackRecorder.current;+ if (recorder?.state !== "recording") {+ return;+ }++ try {+ recorder.requestData();+ } catch (error) {+ console.warn("Failed to flush microphone fallback chunk before pause:", error);+ }++ recorder.pause();+ const now = performance.now();+ const startedAt = micFallbackRecorderStartedAt.current ?? now;+ micFallbackPauseStartedAt.current = now;+ micFallbackPauseIntervals.current.push({+ startElapsedMs: Math.max(0, Math.round(now - startedAt)),+ });+ }, []);++ const resumeMicFallbackRecorder = useCallback(() => {+ const recorder = micFallbackRecorder.current;+ if (recorder?.state !== "paused") {+ return;+ }++ closeMicFallbackPauseInterval();+ recorder.resume();+ }, [closeMicFallbackPauseInterval]);const storeMicrophoneSidecar = useCallback(async (@@ -644,7 +743,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {micFallbackRequestedConstraints.current = null;micFallbackAudioInputDevices.current = null;micFallbackRecorderMetadata.current = null;- micFallbackChunkEvents.current = [];+ resetMicFallbackTimingDiagnostics();return;}@@ -679,6 +778,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn {...(micFallbackChunkEvents.current.length > 0? { chunkEvents: [...micFallbackChunkEvents.current] }: {}),+ ...(micFallbackPauseIntervals.current.length > 0+ ? {+ pauseIntervals: micFallbackPauseIntervals.current.map(+ (interval) => ({ ...interval }),+ ),+ }+ : {}),};const result = await window.electronAPI.storeMicrophoneSidecar(arrayBuffer,@@ -706,10 +812,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {micFallbackRequestedConstraints.current = null;micFallbackAudioInputDevices.current = null;micFallbackRecorderMetadata.current = null;- micFallbackChunkEvents.current = [];+ resetMicFallbackTimingDiagnostics();}},- [],+ [resetMicFallbackTimingDiagnostics],);const stopWebcamRecorder = useCallback(async () => {@@ -1306,7 +1412,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {audioBitsPerSecond: AUDIO_BITRATE_VOICE,timesliceMs: RECORDER_TIMESLICE_MS,};- micFallbackChunkEvents.current = [];+ resetMicFallbackTimingDiagnostics();micFallbackRecorderStartedAt.current = performance.now();recorder.ondataavailable = appendMicFallbackChunk;micFallbackStartDelayMs.current = Math.max(@@ -1321,8 +1427,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {micFallbackRequestedConstraints.current = null;micFallbackAudioInputDevices.current = null;micFallbackRecorderMetadata.current = null;- micFallbackChunkEvents.current = [];- micFallbackRecorderStartedAt.current = null;+ resetMicFallbackTimingDiagnostics();console.warn("Browser microphone fallback failed:", micError);const permissionDenied =micError instanceof DOMException &&@@ -1667,9 +1772,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {if (webcamRecorder.current?.state === "recording") {webcamRecorder.current.pause();}- if (micFallbackRecorder.current?.state === "recording") {- micFallbackRecorder.current.pause();- }+ pauseMicFallbackRecorder();const boundaryMs = Date.now();markRecordingPaused(boundaryMs);setPaused(true);@@ -1697,7 +1800,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {}})();}- }, [markRecordingPaused, paused, recording]);+ }, [markRecordingPaused, pauseMicFallbackRecorder, paused, recording]);const resumeRecording = useCallback(() => {if (!recording || !paused) return;@@ -1715,9 +1818,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {if (webcamRecorder.current?.state === "paused") {webcamRecorder.current.resume();}- if (micFallbackRecorder.current?.state === "paused") {- micFallbackRecorder.current.resume();- }+ resumeMicFallbackRecorder();const boundaryMs = Date.now();markRecordingResumed(boundaryMs);setPaused(false);@@ -1745,7 +1846,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {}})();}- }, [markRecordingResumed, paused, recording]);+ }, [markRecordingResumed, paused, recording, resumeMicFallbackRecorder]);const cancelRecording = useCallback(() => {if (!recording) return;