Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

fix(recording): use native audio start timestamps for Windows sync

Wiiid04fc00fd8e3
diff --git a/electron/ipc/ffmpeg/filters.test.ts b/electron/ipc/ffmpeg/filters.test.ts
index deab8017..bc28a3ca 100644
--- a/electron/ipc/ffmpeg/filters.test.ts
+++ b/electron/ipc/ffmpeg/filters.test.ts
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
-import { appendSyncedAudioFilter, getAudioSyncAdjustment } from "./filters";
+import {
+ appendSyncedAudioFilter,
+ applyRecordedAudioStartDelay,
+ getAudioSyncAdjustment,
+} from "./filters";
describe("getAudioSyncAdjustment", () => {
it("does not speed up longer audio tracks that would advance speech", () => {
@@ -30,6 +34,15 @@ describe("getAudioSyncAdjustment", () => {
});
});
+ it("pads trailing silence instead of prepending extreme delay for very short audio tracks", () => {
+ expect(getAudioSyncAdjustment(600, 480)).toEqual({
+ mode: "pad",
+ delayMs: 0,
+ tempoRatio: 1,
+ durationDeltaMs: 120000,
+ });
+ });
+
it("does not inject atempo when longer audio stays on the anchored path", () => {
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(120, 122.5));
@@ -47,4 +60,51 @@ describe("getAudioSyncAdjustment", () => {
"[1:a]atempo=0.975000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",
]);
});
+
+ it("pads the tail for very short audio tracks", () => {
+ const filterParts: string[] = [];
+ appendSyncedAudioFilter(filterParts, "[1:a]", "aout", getAudioSyncAdjustment(600, 480));
+
+ expect(filterParts).toEqual([
+ "[1:a]apad=pad_dur=120.000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",
+ ]);
+ });
+});
+
+describe("applyRecordedAudioStartDelay", () => {
+ it("pads the tail when recorded metadata says audio started on time", () => {
+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 110), 0)).toEqual({
+ mode: "pad",
+ delayMs: 0,
+ tempoRatio: 1,
+ durationDeltaMs: 10000,
+ });
+ });
+
+ it("prefers a measured start delay over a tempo-only heuristic", () => {
+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 119.8), 275)).toEqual({
+ mode: "delay",
+ delayMs: 275,
+ tempoRatio: 1,
+ durationDeltaMs: 200,
+ });
+ });
+
+ it("applies a measured start delay even when durations already match", () => {
+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 120), 275)).toEqual({
+ mode: "delay",
+ delayMs: 275,
+ tempoRatio: 1,
+ durationDeltaMs: 0,
+ });
+ });
+
+ it("leaves tempo correction alone when recorded metadata says there was no late start", () => {
+ expect(applyRecordedAudioStartDelay(getAudioSyncAdjustment(120, 117), 0)).toEqual({
+ mode: "tempo",
+ delayMs: 0,
+ tempoRatio: 0.975,
+ durationDeltaMs: 3000,
+ });
+ });
});
diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.ts
index ce973f16..735e3c66 100644
--- a/electron/ipc/ffmpeg/filters.ts
+++ b/electron/ipc/ffmpeg/filters.ts
@@ -1,5 +1,7 @@
import type { AudioSyncAdjustment, PauseSegment } from "../types";
+const MAX_AUDIO_SYNC_DELAY_MS = 15000;
+
export function buildAtempoFilters(tempoRatio: number): string[] {
if (!Number.isFinite(tempoRatio) || tempoRatio <= 0) {
return [];
@@ -58,9 +60,43 @@ export function getAudioSyncAdjustment(
return { mode: "tempo", delayMs: 0, tempoRatio, durationDeltaMs };
}
+ if (durationDeltaMs > MAX_AUDIO_SYNC_DELAY_MS) {
+ return { mode: "pad", delayMs: 0, tempoRatio: 1, durationDeltaMs };
+ }
+
return { mode: "delay", delayMs: durationDeltaMs, tempoRatio: 1, durationDeltaMs };
}
+export function applyRecordedAudioStartDelay(
+ adjustment: AudioSyncAdjustment,
+ recordedStartDelayMs?: number | null,
+): AudioSyncAdjustment {
+ if (!Number.isFinite(recordedStartDelayMs) || (recordedStartDelayMs ?? 0) < 0) {
+ return adjustment;
+ }
+
+ const delayMs = Math.max(0, Math.round(recordedStartDelayMs ?? 0));
+ if (delayMs > 20) {
+ return {
+ mode: "delay",
+ delayMs,
+ tempoRatio: 1,
+ durationDeltaMs: adjustment.durationDeltaMs,
+ };
+ }
+
+ if (adjustment.mode !== "delay" && adjustment.mode !== "pad") {
+ return adjustment;
+ }
+
+ return {
+ mode: "pad",
+ delayMs: 0,
+ tempoRatio: 1,
+ durationDeltaMs: adjustment.durationDeltaMs,
+ };
+}
+
export function appendSyncedAudioFilter(
filterParts: string[],
inputLabel: string,
@@ -77,6 +113,10 @@ export function appendSyncedAudioFilter(
filters.push(...buildAtempoFilters(adjustment.tempoRatio));
}
+ if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) {
+ filters.push(`apad=pad_dur=${formatFfmpegSeconds(adjustment.durationDeltaMs)}`);
+ }
+
filters.push("aresample=async=1:first_pts=0", "asetpts=PTS-STARTPTS");
filterParts.push(`${inputLabel}${filters.join(",")}[${outputLabel}]`);
}
diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts
index 9a775339..dca8f3fc 100644
--- a/electron/ipc/recording/diagnostics.test.ts
+++ b/electron/ipc/recording/diagnostics.test.ts
@@ -166,6 +166,18 @@ describe("getCompanionAudioFallbackPaths", () => {
});
});
+ it("ignores invalid sidecar timing metadata values", async () => {
+ const micPath = path.join(tempRoot, "recording.mic.wav");
+ await Promise.all([
+ fs.writeFile(micPath, "mic"),
+ fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: -250 })),
+ ]);
+
+ const { getCompanionAudioStartDelayMs } = await import("./diagnostics");
+
+ await expect(getCompanionAudioStartDelayMs(micPath)).resolves.toBeNull();
+ });
+
it("rejects tiny MP4 container-only outputs before they reach the editor", async () => {
const videoPath = path.join(tempRoot, "recording-123.mp4");
await fs.writeFile(videoPath, Buffer.alloc(261));
diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts
index 4588631e..fb7b6c4a 100644
--- a/electron/ipc/recording/diagnostics.ts
+++ b/electron/ipc/recording/diagnostics.ts
@@ -119,6 +119,16 @@ async function readCompanionAudioTimingMetadata(
}
}
+export async function getCompanionAudioStartDelayMs(companionPath: string) {
+ const metadata = await readCompanionAudioTimingMetadata(companionPath);
+ const startDelayMs = metadata?.startDelayMs;
+ if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) {
+ return null;
+ }
+
+ return Math.round(startDelayMs ?? 0);
+}
+
export async function hasEmbeddedAudioStream(videoPath: string) {
const ffmpegPath = getFfmpegBinaryPath();
let stderr = "";
@@ -172,22 +182,19 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) {
const metadataEntries = await Promise.all(
paths.map(async (audioPath) => {
- const metadata = await readCompanionAudioTimingMetadata(audioPath);
- const startDelayMs = metadata?.startDelayMs;
- if (!Number.isFinite(startDelayMs) || (startDelayMs ?? 0) < 0) {
+ const startDelayMs = await getCompanionAudioStartDelayMs(audioPath);
+ if (!Number.isFinite(startDelayMs)) {
return null;
}
- return [audioPath, Math.round(startDelayMs ?? 0)] as const;
+ return [audioPath, startDelayMs] as const;
}),
);
return {
paths,
startDelayMsByPath: Object.fromEntries(
- metadataEntries.filter(
- (entry): entry is readonly [string, number] => entry !== null,
- ),
+ metadataEntries.filter((entry): entry is readonly [string, number] => entry !== null),
),
};
}
diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts
index 4a32f86c..612cc9f0 100644
--- a/electron/ipc/recording/windows.ts
+++ b/electron/ipc/recording/windows.ts
@@ -7,24 +7,29 @@ import { BrowserWindow } from "electron";
import { getFfmpegBinaryPath } from "../ffmpeg/binary";
import {
appendSyncedAudioFilter,
+ applyRecordedAudioStartDelay,
buildPausedAudioFilter,
getAudioSyncAdjustment,
normalizePauseSegments,
} from "../ffmpeg/filters";
import { getWindowsCaptureExePath } from "../paths/binaries";
import {
+ selectedSource,
setWindowsCaptureProcess,
+ setWindowsCaptureStopRequested,
+ setWindowsNativeCaptureActive,
windowsCaptureOutputBuffer,
+ windowsCaptureStopRequested,
windowsCaptureTargetPath,
windowsNativeCaptureActive,
- setWindowsNativeCaptureActive,
- windowsCaptureStopRequested,
- setWindowsCaptureStopRequested,
- selectedSource,
} from "../state";
import type { AudioSyncAdjustment, PauseSegment } from "../types";
import { moveFileWithOverwrite } from "../utils";
-import { probeMediaDurationSeconds, validateRecordedVideo } from "./diagnostics";
+import {
+ getCompanionAudioStartDelayMs,
+ probeMediaDurationSeconds,
+ validateRecordedVideo,
+} from "./diagnostics";
import { emitRecordingInterrupted } from "./events";
const execFileAsync = promisify(execFile);
@@ -191,9 +196,21 @@ export async function muxNativeWindowsVideoWithAudio(
if (videoDuration > 0) {
for (let i = 0; i < audioFilePaths.length; i++) {
const audioDuration = await probeMediaDurationSeconds(audioFilePaths[i]);
- const adjustment = getAudioSyncAdjustment(videoDuration, audioDuration);
+ const recordedStartDelayMs = await getCompanionAudioStartDelayMs(audioFilePaths[i]);
+ const adjustment = applyRecordedAudioStartDelay(
+ getAudioSyncAdjustment(videoDuration, audioDuration),
+ recordedStartDelayMs,
+ );
audioAdjustments.set(audioInputs[i], adjustment);
- if (adjustment.mode === "tempo") {
+ if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "delay") {
+ console.log(
+ `[mux-win] ${audioInputs[i]} audio recorded a start delay of ${adjustment.delayMs}ms`,
+ );
+ } else if (Number.isFinite(recordedStartDelayMs) && adjustment.mode === "pad") {
+ console.log(
+ `[mux-win] ${audioInputs[i]} audio started on time but ends ${adjustment.durationDeltaMs}ms early — padding trailing silence`,
+ );
+ } else if (adjustment.mode === "tempo") {
console.log(
`[mux-win] ${audioInputs[i]} audio differs from video by ${adjustment.durationDeltaMs}ms — applying tempo ratio ${adjustment.tempoRatio.toFixed(6)}`,
);
@@ -201,6 +218,10 @@ export async function muxNativeWindowsVideoWithAudio(
console.log(
`[mux-win] ${audioInputs[i]} audio appears to start late by ${adjustment.delayMs}ms — adding leading silence`,
);
+ } else if (adjustment.mode === "pad" && adjustment.durationDeltaMs > 0) {
+ console.log(
+ `[mux-win] ${audioInputs[i]} audio is much shorter than video by ${adjustment.durationDeltaMs}ms — padding trailing silence`,
+ );
}
}
}
@@ -326,7 +347,10 @@ export async function muxNativeWindowsVideoWithAudio(
for (const audioPath of [systemAudioPath, micAudioPath]) {
if (audioPath) {
- await fs.rm(audioPath, { force: true }).catch(() => undefined);
+ await Promise.all([
+ fs.rm(audioPath, { force: true }).catch(() => undefined),
+ fs.rm(`${audioPath}.json`, { force: true }).catch(() => undefined),
+ ]);
}
}
}
diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts
index 5922fb0e..7c7de9c1 100644
--- a/electron/ipc/types.ts
+++ b/electron/ipc/types.ts
@@ -156,7 +156,7 @@ export type UiohookModuleNamespace = {
};
export type AudioSyncAdjustment = {
- mode: "none" | "tempo" | "delay";
+ mode: "none" | "tempo" | "delay" | "pad";
delayMs: number;
tempoRatio: number;
durationDeltaMs: number;
diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.json
index dbae2bae..0ae7bb77 100644
--- a/electron/native/bin/win32-x64/helpers-manifest.json
+++ b/electron/native/bin/win32-x64/helpers-manifest.json
@@ -1,21 +1,21 @@
{
- "version": 1,
- "platform": "win32",
- "arch": "x64",
- "helpers": {
- "wgc-capture": {
- "binaryName": "wgc-capture.exe",
- "binarySha256": "013164c0a1391d334e5aa2fe0ff2e47b507407e36c7e44e7333a70a84b944643",
- "sourceDir": "electron/native/wgc-capture",
- "sourceFingerprint": "9e9bce082266ca5968cf5f0b535b469d47c4ec3a775a93726171c0dfdbcdaa44",
- "updatedAt": "2026-03-29T02:15:34.516Z"
- },
- "cursor-monitor": {
- "binaryName": "cursor-monitor.exe",
- "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0",
- "sourceDir": "electron/native/cursor-monitor",
- "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e",
- "updatedAt": "2026-03-29T02:15:38.286Z"
- }
- }
+ "version": 1,
+ "platform": "win32",
+ "arch": "x64",
+ "helpers": {
+ "wgc-capture": {
+ "binaryName": "wgc-capture.exe",
+ "binarySha256": "bb4c2aa4141e81e1a05b54bd49dbb114eb3a5ff4a666bf7d26525a9da585128b",
+ "sourceDir": "electron/native/wgc-capture",
+ "sourceFingerprint": "5bb02a69049a02909d38188cae1dfdfb94fd49465b51ed6ab78a98092b7520cc",
+ "updatedAt": "2026-04-25T09:17:16.237Z"
+ },
+ "cursor-monitor": {
+ "binaryName": "cursor-monitor.exe",
+ "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0",
+ "sourceDir": "electron/native/cursor-monitor",
+ "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e",
+ "updatedAt": "2026-03-29T02:15:38.286Z"
+ }
+ }
}
diff --git a/electron/native/bin/win32-x64/wgc-capture.exe b/electron/native/bin/win32-x64/wgc-capture.exe
index e5d0df8f..d16a0724 100644
Binary files a/electron/native/bin/win32-x64/wgc-capture.exe and b/electron/native/bin/win32-x64/wgc-capture.exe differ
diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp
index c949e8a2..7fca95b8 100644
--- a/electron/native/wgc-capture/src/main.cpp
+++ b/electron/native/wgc-capture/src/main.cpp
@@ -14,6 +14,7 @@
#include <condition_variable>
#include <chrono>
#include <cstdio>
+#include <fstream>
static std::atomic<bool> g_stopRequested{false};
static std::atomic<bool> g_pauseRequested{false};
@@ -156,6 +157,43 @@ static std::wstring utf8ToWide(const std::string& str) {
return wstr;
}
+static void writeCompanionAudioTimingMetadata(
+ const std::string& audioPath,
+ int64_t firstVideoTimestampHns,
+ const WasapiCapture& capture
+) {
+ if (audioPath.empty() || firstVideoTimestampHns < 0) {
+ return;
+ }
+
+ const int64_t firstPacketQpcHns = capture.firstPacketQpcHns();
+ if (firstPacketQpcHns < 0) {
+ return;
+ }
+
+ int64_t startDelayMs = (firstPacketQpcHns - firstVideoTimestampHns + 5000) / 10000;
+ if (startDelayMs < 0) {
+ startDelayMs = 0;
+ }
+
+ std::ofstream metadataFile(audioPath + ".json", std::ios::trunc);
+ if (!metadataFile) {
+ std::cerr << "WARNING: Failed to write audio timing metadata for " << audioPath << std::endl;
+ return;
+ }
+
+ metadataFile << "{\"startDelayMs\":" << startDelayMs;
+ const uint32_t discontinuityCount = capture.dataDiscontinuityCount();
+ if (discontinuityCount > 0) {
+ metadataFile << ",\"dataDiscontinuityCount\":" << discontinuityCount;
+ }
+ const uint32_t timestampErrorCount = capture.timestampErrorCount();
+ if (timestampErrorCount > 0) {
+ metadataFile << ",\"timestampErrorCount\":" << timestampErrorCount;
+ }
+ metadataFile << "}";
+}
+
static void stdinListenerThread() {
std::string line;
while (std::getline(std::cin, line)) {
@@ -250,9 +288,12 @@ int main(int argc, char* argv[]) {
// Set up frame callback
std::atomic<int64_t> frameCount{0};
+ std::atomic<int64_t> firstVideoTimestampHns{-1};
std::atomic<bool> recordingStartedAnnounced{false};
session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) {
g_lastFrameTimestampHns = timestampHns;
+ int64_t expectedFirstVideoTimestampHns = -1;
+ firstVideoTimestampHns.compare_exchange_strong(expectedFirstVideoTimestampHns, timestampHns);
if (g_stopRequested) return;
if (g_pauseRequested) return;
@@ -337,6 +378,19 @@ int main(int argc, char* argv[]) {
if (audioActive) loopback.stop();
if (micActive) micCapture.stop();
+ if (audioActive) {
+ writeCompanionAudioTimingMetadata(
+ config.audioOutputPath,
+ firstVideoTimestampHns.load(),
+ loopback);
+ }
+ if (micActive) {
+ writeCompanionAudioTimingMetadata(
+ config.micOutputPath,
+ firstVideoTimestampHns.load(),
+ micCapture);
+ }
+
if (frameCount.load() <= 0) {
std::cerr << "ERROR: No video frames were captured before stop" << std::endl;
DeleteFileW(outputPathW.c_str());
@@ -350,9 +404,19 @@ int main(int argc, char* argv[]) {
std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl;
if (audioActive) {
+ if (loopback.dataDiscontinuityCount() > 0 || loopback.timestampErrorCount() > 0) {
+ std::cerr << "WARNING: System audio timing metadata includes discontinuities="
+ << loopback.dataDiscontinuityCount()
+ << " timestampErrors=" << loopback.timestampErrorCount() << std::endl;
+ }
std::cout << "Audio path: " << config.audioOutputPath << std::endl;
}
if (micActive) {
+ if (micCapture.dataDiscontinuityCount() > 0 || micCapture.timestampErrorCount() > 0) {
+ std::cerr << "WARNING: Microphone timing metadata includes discontinuities="
+ << micCapture.dataDiscontinuityCount()
+ << " timestampErrors=" << micCapture.timestampErrorCount() << std::endl;
+ }
std::cout << "Mic path: " << config.micOutputPath << std::endl;
}
std::cout.flush();
diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cpp
index 1659d013..e4008def 100644
--- a/electron/native/wgc-capture/src/wasapi_loopback.cpp
+++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp
@@ -137,6 +137,9 @@ bool WasapiCapture::start() {
}
totalDataBytes_ = 0;
+ firstPacketQpcHns_ = -1;
+ dataDiscontinuityCount_ = 0;
+ timestampErrorCount_ = 0;
writeWavHeader(outputFile_, 0);
HRESULT hr = audioClient_->Start();
@@ -250,13 +253,37 @@ void WasapiCapture::captureThread() {
BYTE* data = nullptr;
UINT32 numFrames = 0;
DWORD flags = 0;
-
- hr = captureClient_->GetBuffer(&data, &numFrames, &flags, nullptr, nullptr);
+ UINT64 devicePosition = 0;
+ UINT64 qpcPosition = 0;
+
+ hr = captureClient_->GetBuffer(
+ &data,
+ &numFrames,
+ &flags,
+ &devicePosition,
+ &qpcPosition);
if (FAILED(hr)) {
std::cerr << "WASAPI: GetBuffer failed hr=0x" << std::hex << hr << std::dec << std::endl;
break;
}
+ if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) {
+ dataDiscontinuityCount_.fetch_add(1);
+ }
+ if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) {
+ timestampErrorCount_.fetch_add(1);
+ }
+ if (
+ numFrames > 0 &&
+ qpcPosition > 0 &&
+ (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0
+ ) {
+ int64_t expected = -1;
+ firstPacketQpcHns_.compare_exchange_strong(
+ expected,
+ static_cast<int64_t>(qpcPosition));
+ }
+
UINT32 totalSamples = numFrames * channels;
if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.h
index a4facf13..1792a863 100644
--- a/electron/native/wgc-capture/src/wasapi_loopback.h
+++ b/electron/native/wgc-capture/src/wasapi_loopback.h
@@ -19,6 +19,9 @@ public:
bool pause();
bool resume();
void stop();
+ int64_t firstPacketQpcHns() const { return firstPacketQpcHns_.load(); }
+ uint32_t dataDiscontinuityCount() const { return dataDiscontinuityCount_.load(); }
+ uint32_t timestampErrorCount() const { return timestampErrorCount_.load(); }
private:
bool initializeCommon();
@@ -41,4 +44,7 @@ private:
DWORD streamFlags_ = 0;
UINT32 bufferFrameCount_ = 0;
+ std::atomic<int64_t> firstPacketQpcHns_{-1};
+ std::atomic<uint32_t> dataDiscontinuityCount_{0};
+ std::atomic<uint32_t> timestampErrorCount_{0};
};