Create polished demo videos without editing skills. Mac/Windows/Linux
fix(recording): use native audio start timestamps for Windows sync
Wiiid04fc00fd8e3
diff --git a/electron/ipc/ffmpeg/filters.test.ts b/electron/ipc/ffmpeg/filters.test.tsindex deab8017..bc28a3ca 100644--- a/electron/ipc/ffmpeg/filters.test.ts+++ b/electron/ipc/ffmpeg/filters.test.ts@@ -1,6 +1,10 @@import { describe, expect, it } from "vitest";-import { appendSyncedAudioFilter, getAudioSyncAdjustment } from "./filters";+import {+ appendSyncedAudioFilter,+ applyRecordedAudioStartDelay,+ getAudioSyncAdjustment,+} from "./filters";describe("getAudioSyncAdjustment", () => {it("does not speed up longer audio tracks that would advance speech", () => {@@ -30,6 +34,15 @@ describe("getAudioSyncAdjustment", () => {});});+ it("pads trailing silence instead of prepending extreme delay for very short audio tracks", () => {+ expect(getAudioSyncAdjustment(600, 480)).toEqual({+ mode: "pad",+ delayMs: 0,+ tempoRatio: 1,+ durationDeltaMs: 120000,+ });+ });+it("does not inject atempo when longer audio stays on the anchored path", () => {const filterParts: string[] = [];appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(120, 122.5));@@ -47,4 +60,51 @@ describe("getAudioSyncAdjustment", () => {"[1:a]atempo=0.975000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",]);});++ it("pads the tail for very short audio tracks", () => {+ const filterParts: string[] = [];+ appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(600, 480));++ expect(filterParts).toEqual([+ "[1:a]apad=pad_dur=120.000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",+ ]);+ });+});++describe("applyRecordedAudioStartDelay", () => {+ it("pads the tail when recorded metadata says audio started on time", () => {+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 110), 0)).toEqual({+ mode: "pad",+ delayMs: 0,+ tempoRatio: 1,+ durationDeltaMs: 10000,+ });+ });++ it("prefers a measured start delay over a tempo-only heuristic", () => {+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 119.8), 275)).toEqual({+ mode: "delay",+ delayMs: 275,+ tempoRatio: 1,+ durationDeltaMs: 200,+ });+ });++ it("applies a measured start delay even when durations already match", () => {+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 120), 275)).toEqual({+ mode: "delay",+ delayMs: 275,+ tempoRatio: 1,+ durationDeltaMs: 0,+ });+ });++ it("leaves tempo correction alone when recorded metadata says there was no late start", () => {+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 117), 0)).toEqual({+ mode: "tempo",+ delayMs: 0,+ tempoRatio: 0.975,+ durationDeltaMs: 3000,+ });+ });});diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.tsindex ce973f16..735e3c66 100644--- a/electron/ipc/ffmpeg/filters.ts+++ b/electron/ipc/ffmpeg/filters.ts@@ -1,5 +1,7 @@import type { AudioSyncAdjustment, PauseSegment } from "../types";+const MAX_AUDIO_SYNC_DELAY_MS = 15000;+export function buildAtempoFilters(tempoRatio: number): string[] {if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) {return [];@@ -58,9 +60,43 @@ export function getAudioSyncAdjustment(return { mode: "tempo", delayMs: 0, tempoRatio, durationDeltaMs };}+ if (durationDeltaMs > MAX_AUDIO_SYNC_DELAY_MS) {+ return { mode: "pad", delayMs: 0, tempoRatio: 1, durationDeltaMs };+ }+return { mode: "delay", delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs };}+export function applyRecordedAudioStartDelay(+ adjustment: AudioSyncAdjustment,+ recordedStartDelayMs?: number | null,+): AudioSyncAdjustment {+ if (!Number.isFinite(recordedStartDelayMs) || (recordedStartDelayMs ?? 0) < 0) {+ return adjustment;+ }++ const delayMs = Math.max(0, Math.round(recordedStartDelayMs ?? 0));+ if (delayMs > 20) {+ return {+ mode: "delay",+ delayMs,+ tempoRatio: 1,+ durationDeltaMs: adjustment.durationDeltaMs,+ };+ }++ if (adjustment.mode !== "delay" && adjustment.mode !== "pad") {+ return adjustment;+ }++ return {+ mode: "pad",+ delayMs: 0,+ tempoRatio: 1,+ durationDeltaMs: adjustment.durationDeltaMs,+ };+}+export function appendSyncedAudioFilter(filterParts: string[],inputLabel: string,@@ -77,6 +113,10 @@ export function appendSyncedAudioFilter(filters.push(...buildAtempoFilters(adjustment.tempoRatio));}+ if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) {+ filters.push(`apad=pad_dur=${formatFfmpegSeconds(adjustment.durationDeltaMs)}`);+ }+filters.push("aresample=async=1:first_pts=0", "asetpts=PTS-STARTPTS");filterParts.push(`${inputLabel}${filters.join(",")}[${outputLabel}]`);}diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.tsindex 9a775339..dca8f3fc 100644--- a/electron/ipc/recording/diagnostics.test.ts+++ b/electron/ipc/recording/diagnostics.test.ts@@ -166,6 +166,18 @@ describe("getCompanionAudioFallbackPaths", () => {});});+ it("ignores invalid sidecar timing metadata values", async () => {+ const micPath = path.join(tempRoot, "recording.mic.wav");+ await Promise.all([+ fs.writeFile(micPath, "mic"),+ fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: -250 })),+ ]);++ const { getCompanionAudioStartDelayMs } = await import("./diagnostics");++ await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull();+ });+it("rejects tiny MP4 container-only outputs before they reach the editor", async () => {const videoPath = path.join(tempRoot, "recording-123.mp4");await fs.writeFile(videoPath, Buffer.alloc(261));diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.tsindex 4588631e..fb7b6c4a 100644--- a/electron/ipc/recording/diagnostics.ts+++ b/electron/ipc/recording/diagnostics.ts@@ -119,6 +119,16 @@ async function readCompanionAudioTimingMetadata(}}+export async function getCompanionAudioStartDelayMs(companionPath: string) {+ const metadata = await readCompanionAudioTimingMetadata(companionPath);+ const startDelayMs = metadata?.startDelayMs;+ if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) {+ return null;+ }++ return Math.round(startDelayMs ?? 0);+}+export async function hasEmbeddedAudioStream(videoPath: string) {const ffmpegPath = getFfmpegBinaryPath();let stderr = "";@@ -172,22 +182,19 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) {const metadataEntries = await Promise.all(paths.map(async (audioPath) => {- const metadata = await readCompanionAudioTimingMetadata(audioPath);- const startDelayMs = metadata?.startDelayMs;- if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) {+ const startDelayMs = await getCompanionAudioStartDelayMs(audioPath);+ if (!Number.isFinite(startDelayMs)) {return null;}- return [audioPath, Math.round(startDelayMs ?? 0)] as const;+ return [audioPath, startDelayMs] as const;}),);return {paths,startDelayMsByPath: Object.fromEntries(- metadataEntries.filter(- (entry): entry is readonly [string, number] => entry !== null,- ),+ metadataEntries.filter((entry): entry is readonly [string, number] => entry !== null),),};}diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.tsindex 4a32f86c..612cc9f0 100644--- a/electron/ipc/recording/windows.ts+++ b/electron/ipc/recording/windows.ts@@ -7,24 +7,29 @@ import { BrowserWindow } from "electron";import { getFfmpegBinaryPath } from "../ffmpeg/binary";import {appendSyncedAudioFilter,+ applyRecordedAudioStartDelay,buildPausedAudioFilter,getAudioSyncAdjustment,normalizePauseSegments,} from "../ffmpeg/filters";import { getWindowsCaptureExePath } from "../paths/binaries";import {+ selectedSource,setWindowsCaptureProcess,+ setWindowsCaptureStopRequested,+ setWindowsNativeCaptureActive,windowsCaptureOutputBuffer,+ windowsCaptureStopRequested,windowsCaptureTargetPath,windowsNativeCaptureActive,- setWindowsNativeCaptureActive,- windowsCaptureStopRequested,- setWindowsCaptureStopRequested,- selectedSource,} from "../state";import type { AudioSyncAdjustment, PauseSegment } from "../types";import { moveFileWithOverwrite } from "../utils";-import { probeMediaDurationSeconds, validateRecordedVideo } from "./diagnostics";+import {+ getCompanionAudioStartDelayMs,+ probeMediaDurationSeconds,+ validateRecordedVideo,+} from "./diagnostics";import { emitRecordingInterrupted } from "./events";const execFileAsync = promisify(execFile);@@ -191,9 +196,21 @@ export async function muxNativeWindowsVideoWithAudio(if (videoDuration > 0) {for (let i = 0; i < audioFilePaths.length; i++) {const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i]);- const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration);+ const recordedStartDelayMs = await getCompanionAudioStartDelayMs(audioFilePaths[i]);+ const adjustment = applyRecordedAudioStartDelay(+ getAudioSyncAdjustment(videoDuration, audioDuration),+ recordedStartDelayMs,+ );audioAdjustments.set(audioInputs[i], adjustment);- if (adjustment.mode === "tempo") {+ if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "delay") {+ console.log(+ `[mux-win] ${audioInputs[i]} audio recorded a start delay of ${adjustment.delayMs}ms`,+ );+ } else if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "pad") {+ console.log(+ `[mux-win] ${audioInputs[i]} audio started on time but ends ${adjustment.durationDeltaMs}ms early — padding trailing silence`,+ );+ } else if (adjustment.mode === "tempo") {console.log(`[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,);@@ -201,6 +218,10 @@ export async function muxNativeWindowsVideoWithAudio(console.log(`[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,);+ } else if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) {+ console.log(+ `[mux-win] ${audioInputs[i]} audio is much shorter than video by ${adjustment.durationDeltaMs}ms — padding trailing silence`,+ );}}}@@ -326,7 +347,10 @@ export async function muxNativeWindowsVideoWithAudio(for (const audioPath of [systemAudioPath, micAudioPath]) {if (audioPath) {- await fs.rm(audioPath, { force: true }).catch(() => undefined);+ await Promise.all([+ fs.rm(audioPath, { force: true }).catch(() => undefined),+ fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined),+ ]);}}}diff --git a/electron/ipc/types.ts b/electron/ipc/types.tsindex 5922fb0e..7c7de9c1 100644--- a/electron/ipc/types.ts+++ b/electron/ipc/types.ts@@ -156,7 +156,7 @@ export type UiohookModuleNamespace = {};export type AudioSyncAdjustment = {- mode: "none" | "tempo" | "delay";+ mode: "none" | "tempo" | "delay" | "pad";delayMs: number;tempoRatio: number;durationDeltaMs: number;diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.jsonindex dbae2bae..0ae7bb77 100644--- a/electron/native/bin/win32-x64/helpers-manifest.json+++ b/electron/native/bin/win32-x64/helpers-manifest.json@@ -1,21 +1,21 @@{- "version": 1,- "platform": "win32",- "arch": "x64",- "helpers": {- "wgc-capture": {- "binaryName": "wgc-capture.exe",- "binarySha256": "013164c0a1391d334e5aa2fe0ff2e47b507407e36c7e44e7333a70a84b944643",- "sourceDir": "electron/native/wgc-capture",- "sourceFingerprint": "9e9bce082266ca5968cf5f0b535b469d47c4ec3a775a93726171c0dfdbcdaa44",- "updatedAt": "2026-03-29T02:15:34.516Z"- },- "cursor-monitor": {- "binaryName": "cursor-monitor.exe",- "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0",- "sourceDir": "electron/native/cursor-monitor",- "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e",- "updatedAt": "2026-03-29T02:15:38.286Z"- }- }+ "version": 1,+ "platform": "win32",+ "arch": "x64",+ "helpers": {+ "wgc-capture": {+ "binaryName": "wgc-capture.exe",+ "binarySha256": "bb4c2aa4141e81e1a05b54bd49dbb114eb3a5ff4a666bf7d26525a9da585128b",+ "sourceDir": "electron/native/wgc-capture",+ "sourceFingerprint": "5bb02a69049a02909d38188cae1dfdfb94fd49465b51ed6ab78a98092b7520cc",+ "updatedAt": "2026-04-25T09:17:16.237Z"+ },+ "cursor-monitor": {+ "binaryName": "cursor-monitor.exe",+ "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0",+ "sourceDir": "electron/native/cursor-monitor",+ "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e",+ "updatedAt": "2026-03-29T02:15:38.286Z"+ }+ }}diff --git a/electron/native/bin/win32-x64/wgc-capture.exe b/electron/native/bin/win32-x64/wgc-capture.exeindex e5d0df8f..d16a0724 100644Binary files a/electron/native/bin/win32-x64/wgc-capture.exe and b/electron/native/bin/win32-x64/wgc-capture.exe differdiff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cppindex c949e8a2..7fca95b8 100644--- a/electron/native/wgc-capture/src/main.cpp+++ b/electron/native/wgc-capture/src/main.cpp@@ -14,6 +14,7 @@#include <condition_variable>#include <chrono>#include <cstdio>+#include <fstream>static std::atomic<bool> g_stopRequested{false};static std::atomic<bool> g_pauseRequested{false};@@ -156,6 +157,43 @@ static std::wstring utf8ToWide(const std::string& str) {return wstr;}+static void writeCompanionAudioTimingMetadata(+ const std::string& audioPath,+ int64_t firstVideoTimestampHns,+ const WasapiCapture& capture+) {+ if (audioPath.empty() || firstVideoTimestampHns < 0) {+ return;+ }++ const int64_t firstPacketQpcHns = capture.firstPacketQpcHns();+ if (firstPacketQpcHns < 0) {+ return;+ }++ int64_t startDelayMs = (firstPacketQpcHns - firstVideoTimestampHns + 5000) / 10000;+ if (startDelayMs < 0) {+ startDelayMs = 0;+ }++ std::ofstream metadataFile(audioPath + ".json", std::ios::trunc);+ if (!metadataFile) {+ std::cerr << "WARNING: Failed to write audio timing metadata for " << audioPath << std::endl;+ return;+ }++ metadataFile << "{\"startDelayMs\":" << startDelayMs;+ const uint32_t discontinuityCount = capture.dataDiscontinuityCount();+ if (discontinuityCount > 0) {+ metadataFile << ",\"dataDiscontinuityCount\":" << discontinuityCount;+ }+ const uint32_t timestampErrorCount = capture.timestampErrorCount();+ if (timestampErrorCount > 0) {+ metadataFile << ",\"timestampErrorCount\":" << timestampErrorCount;+ }+ metadataFile << "}";+}+static void stdinListenerThread() {std::string line;while (std::getline(std::cin, line)) {@@ -250,9 +288,12 @@ int main(int argc, char* argv[]) {// Set up frame callbackstd::atomic<int64_t> frameCount{0};+ std::atomic<int64_t> firstVideoTimestampHns{-1};std::atomic<bool> recordingStartedAnnounced{false};session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {g_lastFrameTimestampHns = timestampHns;+ int64_t expectedFirstVideoTimestampHns = -1;+ firstVideoTimestampHns.compare_exchange_strong(expectedFirstVideoTimestampHns, timestampHns);if (g_stopRequested) return;if (g_pauseRequested) return;@@ -337,6 +378,19 @@ int main(int argc, char* argv[]) {if (audioActive) loopback.stop();if (micActive) micCapture.stop();+ if (audioActive) {+ writeCompanionAudioTimingMetadata(+ config.audioOutputPath,+ firstVideoTimestampHns.load(),+ loopback);+ }+ if (micActive) {+ writeCompanionAudioTimingMetadata(+ config.micOutputPath,+ firstVideoTimestampHns.load(),+ micCapture);+ }+if (frameCount.load() <= 0) {std::cerr << "ERROR: No video frames were captured before stop" << std::endl;DeleteFileW(outputPathW.c_str());@@ -350,9 +404,19 @@ int main(int argc, char* argv[]) {std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl;if (audioActive) {+ if (loopback.dataDiscontinuityCount() > 0 || loopback.timestampErrorCount() > 0) {+ std::cerr << "WARNING: System audio timing metadata includes discontinuities="+ << loopback.dataDiscontinuityCount()+ << " timestampErrors=" << loopback.timestampErrorCount() << std::endl;+ }std::cout << "Audio path: " << config.audioOutputPath << std::endl;}if (micActive) {+ if (micCapture.dataDiscontinuityCount() > 0 || micCapture.timestampErrorCount() > 0) {+ std::cerr << "WARNING: Microphone timing metadata includes discontinuities="+ << micCapture.dataDiscontinuityCount()+ << " timestampErrors=" << micCapture.timestampErrorCount() << std::endl;+ }std::cout << "Mic path: " << config.micOutputPath << std::endl;}std::cout.flush();diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cppindex 1659d013..e4008def 100644--- a/electron/native/wgc-capture/src/wasapi_loopback.cpp+++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp@@ -137,6 +137,9 @@ bool WasapiCapture::start() {}totalDataBytes_ = 0;+ firstPacketQpcHns_ = -1;+ dataDiscontinuityCount_ = 0;+ timestampErrorCount_ = 0;writeWavHeader(outputFile_, 0);HRESULT hr = audioClient_->Start();@@ -250,13 +253,37 @@ void WasapiCapture::captureThread() {BYTE* data = nullptr;UINT32 numFrames = 0;DWORD flags = 0;-- hr = captureClient_->GetBuffer(&data, &numFrames, &flags, nullptr, nullptr);+ UINT64 devicePosition = 0;+ UINT64 qpcPosition = 0;++ hr = captureClient_->GetBuffer(+ &data,+ &numFrames,+ &flags,+ &devicePosition,+ &qpcPosition);if (FAILED(hr)) {std::cerr << "WASAPI: GetBuffer failed hr=0x" << std::hex << hr << std::dec << std::endl;break;}+ if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) {+ dataDiscontinuityCount_.fetch_add(1);+ }+ if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) {+ timestampErrorCount_.fetch_add(1);+ }+ if (+ numFrames > 0 &&+ qpcPosition > 0 &&+ (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0+ ) {+ int64_t expected = -1;+ firstPacketQpcHns_.compare_exchange_strong(+ expected,+ static_cast<int64_t>(qpcPosition));+ }+UINT32 totalSamples = numFrames * channels;if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.hindex a4facf13..1792a863 100644--- a/electron/native/wgc-capture/src/wasapi_loopback.h+++ b/electron/native/wgc-capture/src/wasapi_loopback.h@@ -19,6 +19,9 @@ public:bool pause();bool resume();void stop();+ int64_t firstPacketQpcHns() const { return firstPacketQpcHns_.load(); }+ uint32_t dataDiscontinuityCount() const { return dataDiscontinuityCount_.load(); }+ uint32_t timestampErrorCount() const { return timestampErrorCount_.load(); }private:bool initializeCommon();@@ -41,4 +44,7 @@ private:DWORD streamFlags_ = 0;UINT32 bufferFrameCount_ = 0;+ std::atomic<int64_t> firstPacketQpcHns_{-1};+ std::atomic<uint32_t> dataDiscontinuityCount_{0};+ std::atomic<uint32_t> timestampErrorCount_{0};};