Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

refactor : video-editor/audio to put all the timeline and video editor logic about audio here. reducing further video editor

Alan Trebugeaisfa9e53d7ffae
diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx
index 71c9ce53..657398a2 100644
--- a/src/components/video-editor/SettingsPanel.tsx
+++ b/src/components/video-editor/SettingsPanel.tsx
@@ -469,8 +469,15 @@ interface SettingsPanelProps {
selectedClipId?: string | null;
selectedClipSpeed?: number | null;
selectedClipMuted?: boolean | null;
+ hasClipSourceAudio?: boolean;
+ showClipSourceAudioTrack?: boolean;
onClipSpeedChange?: (speed: number) => void;
onClipMutedChange?: (muted: boolean) => void;
+ onShowClipSourceAudioTrackChange?: (show: boolean) => void;
+ sourceAudioTrackMeta?: Array<{ id: string; label: string }>;
+ sourceAudioTrackSettings?: Record<string, { volume: number; normalize: boolean }>;
+ onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void;
+ onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void;
onClipDelete?: (id: string) => void;
selectedAudioId?: string | null;
selectedAudioVolume?: number | null;
@@ -862,8 +869,15 @@ export function SettingsPanel({
selectedClipId,
selectedClipSpeed,
selectedClipMuted,
+ hasClipSourceAudio = false,
+ showClipSourceAudioTrack = false,
onClipSpeedChange,
onClipMutedChange,
+ onShowClipSourceAudioTrackChange,
+ sourceAudioTrackMeta = [],
+ sourceAudioTrackSettings = {},
+ onSourceAudioTrackVolumeChange,
+ onSourceAudioTrackNormalizeChange,
onClipDelete,
selectedAudioId,
selectedAudioVolume,
@@ -3049,12 +3063,38 @@ export function SettingsPanel({
<span className="text-[10px] text-muted-foreground">
{tSettings("clip.muteAudio", "Mute Audio")}
</span>
- <Switch
- checked={selectedClipMuted ?? false}
- onCheckedChange={(v) => onClipMutedChange?.(v)}
- className="data-[state=checked]:bg-[#06b6d4] scale-75"
- />
+ <div className="flex items-center gap-2">
+ <span
+ className={cn(
+ "rounded-full px-1.5 py-0.5 text-[9px] font-medium uppercase tracking-wider",
+ selectedClipMuted
+ ? "bg-red-500/15 text-red-300"
+ : "bg-emerald-500/15 text-emerald-300",
+ )}
+ >
+ {selectedClipMuted
+ ? tSettings("clip.mutedState", "Muted")
+ : tSettings("clip.audioOnState", "Audio On")}
+ </span>
+ <Switch
+ checked={!(selectedClipMuted ?? false)}
+ onCheckedChange={(v) => onClipMutedChange?.(!v)}
+ className="data-[state=checked]:bg-[#06b6d4] scale-75"
+ />
+ </div>
</div>
+ {hasClipSourceAudio && (
+ <div className="flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
+ <span className="text-[10px] text-muted-foreground">
+ {tSettings("clip.separateSourceAudio", "Separate Clip Audio")}
+ </span>
+ <Switch
+ checked={showClipSourceAudioTrack}
+ onCheckedChange={(v) => onShowClipSourceAudioTrackChange?.(v)}
+ className="data-[state=checked]:bg-[#06b6d4] scale-75"
+ />
+ </div>
+ )}
<div className="flex items-center gap-3">
<SectionLabel>{tSettings("speed.label", "Speed")}</SectionLabel>
</div>
@@ -3562,10 +3602,12 @@ export function SettingsPanel({
<div
className={cn(
"flex-shrink-0 border-t border-foreground/10 bg-editor-header p-4 pt-3",
- !selectedAudioId && "hidden",
+ !selectedAudioId &&
+ !(selectedClipId && hasClipSourceAudio && showClipSourceAudioTrack) &&
+ "hidden",
)}
>
- {selectedAudioId && (
+ {selectedAudioId ? (
<div>
<div className="mb-3 flex items-center justify-between">
<span className="text-sm font-medium text-foreground">
@@ -3596,7 +3638,57 @@ export function SettingsPanel({
{tSettings("audio.deleteRegion", "Delete Audio")}
</Button>
</div>
- )}
+ ) : selectedClipId && hasClipSourceAudio && showClipSourceAudioTrack ? (
+ <div className="flex flex-col gap-3">
+ <div className="text-sm font-medium text-foreground">
+ {tSettings("audio.sourceTracksTitle", "Clip Source Audio")}
+ </div>
+ {sourceAudioTrackMeta.map((track) => {
+ const settings = sourceAudioTrackSettings[track.id] ?? {
+ volume: 1,
+ normalize: false,
+ };
+ return (
+ <div
+ key={track.id}
+ className="rounded-lg border border-foreground/10 bg-foreground/[0.03] px-3 py-2"
+ >
+ <div className="mb-2 flex items-center justify-between">
+ <span className="text-[11px] font-medium text-foreground">
+ {track.label}
+ </span>
+ <span className="text-[10px] text-muted-foreground">
+ {Math.round(settings.volume * 100)}%
+ </span>
+ </div>
+ <div className="mb-2 flex items-center justify-between rounded-lg bg-foreground/[0.03] px-2.5 py-1.5">
+ <span className="text-[10px] text-muted-foreground">
+ {tSettings("audio.normalize", "Normalize")}
+ </span>
+ <Switch
+ checked={settings.normalize}
+ onCheckedChange={(v) =>
+ onSourceAudioTrackNormalizeChange?.(track.id, v)
+ }
+ className="data-[state=checked]:bg-[#2563EB] scale-75"
+ />
+ </div>
+ <SliderControl
+ label={tSettings("audio.volume", "Volume")}
+ value={settings.volume}
+ defaultValue={1}
+ min={0}
+ max={2}
+ step={0.01}
+ onChange={(v) => onSourceAudioTrackVolumeChange?.(track.id, v)}
+ formatValue={(v) => `${Math.round(v * 100)}%`}
+ parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
+ />
+ </div>
+ );
+ })}
+ </div>
+ ) : null}
</div>
</div>
);
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx
index fa5fc5ac..6208ca3f 100644
--- a/src/components/video-editor/VideoEditor.tsx
+++ b/src/components/video-editor/VideoEditor.tsx
@@ -80,14 +80,7 @@ import {
canUseInMemoryExportSaveFallback,
describeBlockedInMemoryExportSave,
} from "@/lib/exporter/exportSavePolicy";
-import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
-import {
- clampMediaTimeToDuration,
- enablePitchPreservingPlayback,
- estimateCompanionAudioStartDelaySeconds,
- getMediaSyncPlaybackRate,
-} from "@/lib/mediaTiming";
import { matchesShortcut } from "@/lib/shortcuts";
import { cn } from "@/lib/utils";
import {
@@ -147,6 +140,11 @@ import {
validateProjectData,
} from "./projectPersistence";
import { SettingsPanel } from "./SettingsPanel";
+import { SOURCE_AUDIO_NORMALIZE_GAIN, getSourceTrackIdFromPath } from "./audio/sourceAudioTracks";
+import { useAudioPreviewSync } from "./audio/useAudioPreviewSync";
+import { useSourceAudioFallback } from "./audio/useSourceAudioFallback";
+import { getActiveClipIdAtSourceTime, isClipMutedById } from "./audio/clipAudio";
+import { useSourceAudioTrackSettings } from "./audio/useSourceAudioTrackSettings";
import {
APP_HEADER_ICON_BUTTON_CLASS,
DiscordLinkButton,
@@ -347,12 +345,6 @@ async function writeSmokeExportReport(
const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000;
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
-const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
-const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
-const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
-const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
-const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
-const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
const EXPORT_ERROR_TOAST_DURATION_MS = 20000;
@@ -677,6 +669,8 @@ export default function VideoEditor() {
const [selectedAnnotationId, setSelectedAnnotationId] = useState<string | null>(null);
const [audioRegions, setAudioRegions] = useState<AudioRegion[]>([]);
const [selectedAudioId, setSelectedAudioId] = useState<string | null>(null);
+ const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false);
+ const [showClipSourceAudioTrack, setShowClipSourceAudioTrack] = useState(false);
const [autoCaptions, setAutoCaptions] = useState<CaptionCue[]>([]);
const [autoCaptionSettings, setAutoCaptionSettings] = useState<AutoCaptionSettings>(
DEFAULT_AUTO_CAPTION_SETTINGS,
@@ -700,9 +694,6 @@ export default function VideoEditor() {
const [exportError, setExportError] = useState<string | null>(null);
const [showExportDropdown, setShowExportDropdown] = useState(false);
const [previewVolume, setPreviewVolume] = useState(1);
- const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
- const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
- useState<Record<string, number>>({});
const applySessionPresentation = useCallback(
(
session:
@@ -1778,62 +1769,49 @@ export default function VideoEditor() {
() => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null),
[videoPath, videoSourcePath],
);
+ const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } =
+ useSourceAudioFallback({
+ currentSourcePath,
+ summarizeErrorMessage,
+ });
const { hasEmbeddedSourceAudio, externalAudioPaths: previewSourceAudioFallbackPaths } = useMemo(
() => resolveSourceAudioFallbackPaths(currentSourcePath, sourceAudioFallbackPaths),
[currentSourcePath, sourceAudioFallbackPaths],
);
const shouldMutePreviewVideo =
!hasEmbeddedSourceAudio && previewSourceAudioFallbackPaths.length > 0;
-
- useEffect(() => {
- let cancelled = false;
- setSourceAudioFallbackPaths([]);
- setSourceAudioFallbackStartDelayMsByPath({});
-
- if (!currentSourcePath) {
- return () => {
- cancelled = true;
- };
- }
-
- void (async () => {
- try {
- const result =
- await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
- if (cancelled) {
- return;
- }
- if (!result.success) {
- setSourceAudioFallbackPaths([]);
- setSourceAudioFallbackStartDelayMsByPath({});
- toast.warning(
- result.error
- ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
- : "Could not load companion audio sources. Playback and export may miss microphone audio.",
- { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
- );
- return;
- }
-
- toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
- setSourceAudioFallbackPaths(result.paths ?? []);
- setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
- } catch (error) {
- if (!cancelled) {
- setSourceAudioFallbackPaths([]);
- setSourceAudioFallbackStartDelayMsByPath({});
- toast.warning(
- `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
- { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
- );
- }
- }
- })();
-
- return () => {
- cancelled = true;
- };
- }, [currentSourcePath]);
+ const activeClipIdAtCurrentTime = useMemo(
+ () => getActiveClipIdAtSourceTime(currentTime, clipRegions),
+ [clipRegions, currentTime],
+ );
+ const {
+ sourceAudioTrackMeta,
+ activeSourceAudioTrackSettings,
+ selectedClipSourceAudioTrackSettings,
+ onSourceAudioTracksMetaChange,
+ onSelectedClipSourceAudioTrackVolumeChange,
+ onSelectedClipSourceAudioTrackNormalizeChange,
+ } = useSourceAudioTrackSettings({
+ selectedClipId,
+ activeClipId: activeClipIdAtCurrentTime,
+ });
+ const embeddedSourcePreviewGain = useMemo(() => {
+ const settings = activeSourceAudioTrackSettings.mixed ?? { volume: 1, normalize: false };
+ const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
+ return Math.max(0, Math.min(2, settings.volume * normalizeGain));
+ }, [activeSourceAudioTrackSettings]);
+ const getSourceTrackPreviewGain = useCallback(
+ (audioPath: string) => {
+ const trackId = getSourceTrackIdFromPath(audioPath);
+ const settings = activeSourceAudioTrackSettings[trackId] ?? { volume: 1, normalize: false };
+ const normalizeGain = settings.normalize ? SOURCE_AUDIO_NORMALIZE_GAIN : 1;
+ return Math.max(0, Math.min(2, settings.volume * normalizeGain));
+ },
+ [activeSourceAudioTrackSettings],
+ );
+ const isCurrentClipMuted = useMemo(() => {
+ return isClipMutedById(activeClipIdAtCurrentTime, clipRegions);
+ }, [activeClipIdAtCurrentTime, clipRegions]);
const projectDisplayName = useMemo(() => {
const fileName =
@@ -4096,289 +4074,24 @@ export default function VideoEditor() {
}
}, [selectedAudioId, audioRegions]);
- // Audio playback sync: manage Audio elements that play in sync with video
- const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
- const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
- const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
- const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
- const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
- const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
- const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
-
- useEffect(() => {
- let cancelled = false;
- const existing = audioElementsRef.current;
- const currentIds = new Set(audioRegions.map((r) => r.id));
-
- // Remove old audio elements
- for (const [id, audio] of existing) {
- if (!currentIds.has(id)) {
- audio.pause();
- audio.src = "";
- audioElementRevokersRef.current.get(id)?.();
- audioElementRevokersRef.current.delete(id);
- audioElementResourcesRef.current.delete(id);
- existing.delete(id);
- }
- }
-
- // Create/update audio elements
- for (const region of audioRegions) {
- let audio = existing.get(region.id);
- if (!audio) {
- audio = new Audio();
- audio.preload = "auto";
- existing.set(region.id, audio);
- }
-
- if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) {
- audio.pause();
- audio.src = "";
- audioElementRevokersRef.current.get(region.id)?.();
- audioElementRevokersRef.current.delete(region.id);
- audioElementResourcesRef.current.set(region.id, region.audioPath);
-
- void (async () => {
- const resolved = await resolveMediaElementSource(region.audioPath);
- const latestAudio = existing.get(region.id);
-
- if (
- cancelled ||
- latestAudio !== audio ||
- audioElementResourcesRef.current.get(region.id) !== region.audioPath
- ) {
- resolved.revoke();
- return;
- }
-
- audioElementRevokersRef.current.set(region.id, resolved.revoke);
- latestAudio.src = resolved.src;
- })();
- }
-
- audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume));
- }
-
- return () => {
- cancelled = true;
- };
- }, [audioRegions, previewVolume]);
-
- useEffect(() => {
- let cancelled = false;
- const existing = sourceAudioElementsRef.current;
- const currentIds = new Set(previewSourceAudioFallbackPaths);
-
- for (const [id, audio] of existing) {
- if (!currentIds.has(id)) {
- audio.pause();
- audio.src = "";
- sourceAudioElementRevokersRef.current.get(id)?.();
- sourceAudioElementRevokersRef.current.delete(id);
- sourceAudioElementResourcesRef.current.delete(id);
- existing.delete(id);
- }
- }
-
- for (const audioPath of previewSourceAudioFallbackPaths) {
- let audio = existing.get(audioPath);
- if (!audio) {
- audio = new Audio();
- audio.preload = "auto";
- existing.set(audioPath, audio);
- }
- audio.dataset.sourceAudioPath = audioPath;
-
- if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
- audio.pause();
- audio.src = "";
- sourceAudioElementRevokersRef.current.get(audioPath)?.();
- sourceAudioElementRevokersRef.current.delete(audioPath);
- sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
-
- void (async () => {
- try {
- const resolved = await resolveMediaElementSource(audioPath);
- const latestAudio = existing.get(audioPath);
-
- if (
- cancelled ||
- latestAudio !== audio ||
- sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
- ) {
- resolved.revoke();
- return;
- }
-
- sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
- latestAudio.src = resolved.src;
- } catch (error) {
- if (cancelled) {
- return;
- }
-
- sourceAudioElementRevokersRef.current.get(audioPath)?.();
- sourceAudioElementRevokersRef.current.delete(audioPath);
- sourceAudioElementResourcesRef.current.delete(audioPath);
- const latestAudio = existing.get(audioPath);
- if (latestAudio === audio) {
- latestAudio.pause();
- latestAudio.src = "";
- }
- toast.warning(
- `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
- { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
- );
- }
- })();
- }
-
- audio.volume = Math.max(0, Math.min(1, previewVolume));
- }
-
- if (previewSourceAudioFallbackPaths.length === 0) {
- lastSourceAudioSyncTimeRef.current = null;
- }
-
- return () => {
- cancelled = true;
- };
- }, [previewSourceAudioFallbackPaths, previewVolume]);
-
- useEffect(() => {
- return () => {
- for (const audio of audioElementsRef.current.values()) {
- audio.pause();
- audio.src = "";
- }
- for (const revoke of audioElementRevokersRef.current.values()) {
- revoke();
- }
- audioElementsRef.current.clear();
- audioElementRevokersRef.current.clear();
- audioElementResourcesRef.current.clear();
- for (const audio of sourceAudioElementsRef.current.values()) {
- audio.pause();
- audio.src = "";
- }
- for (const revoke of sourceAudioElementRevokersRef.current.values()) {
- revoke();
- }
- sourceAudioElementsRef.current.clear();
- sourceAudioElementRevokersRef.current.clear();
- sourceAudioElementResourcesRef.current.clear();
- lastSourceAudioSyncTimeRef.current = null;
- };
- }, []);
-
- // Sync audio playback with video currentTime and isPlaying state
- useEffect(() => {
- const currentTimeMs = currentTime * 1000;
- const activeSpeedRegion = effectiveSpeedRegions.find(
- (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
- );
- const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
-
- for (const region of audioRegions) {
- const audio = audioElementsRef.current.get(region.id);
- if (!audio) continue;
-
- const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
-
- if (isPlaying && isInRegion) {
- enablePitchPreservingPlayback(audio);
- const audioOffset = (currentTimeMs - region.startMs) / 1000;
- // Only seek if significantly out of sync (> 200ms)
- if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
- audio.currentTime = audioOffset;
- }
- const syncedPlaybackRate = getMediaSyncPlaybackRate({
- basePlaybackRate: targetPlaybackRate,
- currentTime: audio.currentTime,
- targetTime: audioOffset,
- });
- if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
- audio.playbackRate = syncedPlaybackRate;
- }
- if (audio.paused) {
- audio.play().catch(() => undefined);
- }
- } else {
- if (!audio.paused) {
- audio.pause();
- }
- }
- }
- }, [isPlaying, currentTime, audioRegions, effectiveSpeedRegions]);
-
- useEffect(() => {
- if (previewSourceAudioFallbackPaths.length === 0) {
- lastSourceAudioSyncTimeRef.current = null;
- return;
- }
-
- const activeSpeedRegion = effectiveSpeedRegions.find(
- (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
- );
- const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
- const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
- const timelineJumped =
- previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
- const driftThreshold = isPlaying
- ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
- : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
-
- for (const audio of sourceAudioElementsRef.current.values()) {
- enablePitchPreservingPlayback(audio);
- const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
- const startDelaySeconds = estimateCompanionAudioStartDelaySeconds(
- duration,
- audioDuration,
- sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""],
- );
- const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
- const targetTime = clampMediaTimeToDuration(
- currentTime - startDelaySeconds,
- audioDuration,
- );
-
- if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
- try {
- audio.currentTime = targetTime;
- } catch {
- // no-op
- }
- }
-
- const syncedPlaybackRate = getMediaSyncPlaybackRate({
- basePlaybackRate: targetPlaybackRate,
- currentTime: audio.currentTime,
- targetTime,
- toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
- correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
- maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
- });
- if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
- audio.playbackRate = syncedPlaybackRate;
- }
-
- const atEnd = audioDuration !== null && targetTime >= audioDuration;
- if (isPlaying && !beforeAudioStart && !atEnd) {
- audio.play().catch(() => undefined);
- } else if (!audio.paused) {
- audio.pause();
- }
- }
-
- lastSourceAudioSyncTimeRef.current = currentTime;
- }, [
+ useAudioPreviewSync({
+ audioRegions,
+ previewVolume,
+ isPlaying,
currentTime,
duration,
- isPlaying,
+ effectiveSpeedRegions,
previewSourceAudioFallbackPaths,
sourceAudioFallbackStartDelayMsByPath,
- effectiveSpeedRegions,
- ]);
+ isCurrentClipMuted,
+ getSourceTrackPreviewGain,
+ onSourceFallbackLoadError: (error) => {
+ toast.warning(
+ `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`,
+ { duration: 10000 },
+ );
+ },
+ });
const showExportSuccessToast = useCallback((filePath: string) => {
toast.success(`Exported successfully to ${filePath}`, {
@@ -6049,6 +5762,17 @@ export default function VideoEditor() {
selectedClipId && handleClipMutedChange(muted)
}
onClipDelete={handleClipDelete}
+ hasClipSourceAudio={hasClipSourceAudio}
+ showClipSourceAudioTrack={showClipSourceAudioTrack}
+ onShowClipSourceAudioTrackChange={setShowClipSourceAudioTrack}
+ sourceAudioTrackMeta={sourceAudioTrackMeta}
+ sourceAudioTrackSettings={selectedClipSourceAudioTrackSettings}
+ onSourceAudioTrackVolumeChange={
+ onSelectedClipSourceAudioTrackVolumeChange
+ }
+ onSourceAudioTrackNormalizeChange={
+ onSelectedClipSourceAudioTrackNormalizeChange
+ }
selectedAudioId={selectedAudioId}
selectedAudioVolume={
selectedAudioId
@@ -6353,7 +6077,17 @@ export default function VideoEditor() {
cursorClickBounceDuration
}
cursorSway={cursorSway}
- volume={shouldMutePreviewVideo ? 0 : previewVolume}
+ volume={
+ shouldMutePreviewVideo || isCurrentClipMuted
+ ? 0
+ : Math.max(
+ 0,
+ Math.min(
+ 1,
+ previewVolume * embeddedSourcePreviewGain,
+ ),
+ )
+ }
suspendRendering={shouldSuspendPreviewRendering}
/>
</div>
@@ -6628,6 +6362,17 @@ export default function VideoEditor() {
selectedAnnotationId={selectedAnnotationId}
onSelectAnnotation={handleSelectAnnotation}
aspectRatio={aspectRatio}
+ showSourceAudioTrack={showClipSourceAudioTrack}
+ sourceAudioTrackSettings={activeSourceAudioTrackSettings}
+ onSourceAudioAvailabilityChange={(available) => {
+ setHasClipSourceAudio(available);
+ if (!available) {
+ setShowClipSourceAudioTrack(false);
+ }
+ }}
+ onSourceAudioTracksMetaChange={(tracks) => {
+ onSourceAudioTracksMetaChange(tracks);
+ }}
/>
</div>
</div>
diff --git a/src/components/video-editor/audio/clipAudio.ts b/src/components/video-editor/audio/clipAudio.ts
new file mode 100644
index 00000000..d539ad4b
--- /dev/null
+++ b/src/components/video-editor/audio/clipAudio.ts
@@ -0,0 +1,19 @@
+import { mapSourceTimeToTimelineTime } from "../types";
+import type { ClipRegion } from "../types";
+
+export function getActiveClipIdAtSourceTime(
+ sourceTimeSeconds: number,
+ clipRegions: ClipRegion[],
+): string | null {
+ const sourceMs = sourceTimeSeconds * 1000;
+ const timelineMs = mapSourceTimeToTimelineTime(sourceMs, clipRegions);
+ const activeClip = clipRegions.find(
+ (clip) => timelineMs >= clip.startMs && timelineMs < clip.endMs,
+ );
+ return activeClip?.id ?? null;
+}
+
+export function isClipMutedById(clipId: string | null, clipRegions: ClipRegion[]): boolean {
+ if (!clipId) return false;
+ return clipRegions.find((clip) => clip.id === clipId)?.muted ?? false;
+}
diff --git a/src/components/video-editor/audio/sourceAudioTracks.ts b/src/components/video-editor/audio/sourceAudioTracks.ts
new file mode 100644
index 00000000..8b9998c3
--- /dev/null
+++ b/src/components/video-editor/audio/sourceAudioTracks.ts
@@ -0,0 +1,9 @@
+export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
+export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35;
+
+export function getSourceTrackIdFromPath(audioPath: string): "mic" | "system" | "mixed" {
+ const normalized = audioPath.toLowerCase();
+ if (normalized.includes(".mic.")) return "mic";
+ if (normalized.includes(".system.")) return "system";
+ return "mixed";
+}
diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts
new file mode 100644
index 00000000..620ae2e6
--- /dev/null
+++ b/src/components/video-editor/audio/useAudioPreviewSync.ts
@@ -0,0 +1,322 @@
+import { useEffect, useRef } from "react";
+import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
+import {
+ clampMediaTimeToDuration,
+ enablePitchPreservingPlayback,
+ estimateCompanionAudioStartDelaySeconds,
+ getMediaSyncPlaybackRate,
+} from "@/lib/mediaTiming";
+import type { AudioRegion, SpeedRegion } from "../types";
+
+const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18;
+const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01;
+const SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS = 0.08;
+const SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS = 8;
+const SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT = 0.015;
+
+interface UseAudioPreviewSyncParams {
+ audioRegions: AudioRegion[];
+ previewVolume: number;
+ isPlaying: boolean;
+ currentTime: number;
+ duration: number;
+ effectiveSpeedRegions: SpeedRegion[];
+ previewSourceAudioFallbackPaths: string[];
+ sourceAudioFallbackStartDelayMsByPath: Record<string, number>;
+ isCurrentClipMuted: boolean;
+ getSourceTrackPreviewGain: (audioPath: string) => number;
+ onSourceFallbackLoadError: (error: unknown) => void;
+}
+
+export function useAudioPreviewSync({
+ audioRegions,
+ previewVolume,
+ isPlaying,
+ currentTime,
+ duration,
+ effectiveSpeedRegions,
+ previewSourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
+ isCurrentClipMuted,
+ getSourceTrackPreviewGain,
+ onSourceFallbackLoadError,
+}: UseAudioPreviewSyncParams) {
+ const audioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
+ const audioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
+ const audioElementResourcesRef = useRef<Map<string, string>>(new Map());
+ const sourceAudioElementsRef = useRef<Map<string, HTMLAudioElement>>(new Map());
+ const sourceAudioElementRevokersRef = useRef<Map<string, () => void>>(new Map());
+ const sourceAudioElementResourcesRef = useRef<Map<string, string>>(new Map());
+ const lastSourceAudioSyncTimeRef = useRef<number | null>(null);
+
+ useEffect(() => {
+ let cancelled = false;
+ const existing = audioElementsRef.current;
+ const currentIds = new Set(audioRegions.map((r) => r.id));
+
+ for (const [id, audio] of existing) {
+ if (!currentIds.has(id)) {
+ audio.pause();
+ audio.src = "";
+ audioElementRevokersRef.current.get(id)?.();
+ audioElementRevokersRef.current.delete(id);
+ audioElementResourcesRef.current.delete(id);
+ existing.delete(id);
+ }
+ }
+
+ for (const region of audioRegions) {
+ let audio = existing.get(region.id);
+ if (!audio) {
+ audio = new Audio();
+ audio.preload = "auto";
+ existing.set(region.id, audio);
+ }
+
+ if (audioElementResourcesRef.current.get(region.id) !== region.audioPath) {
+ audio.pause();
+ audio.src = "";
+ audioElementRevokersRef.current.get(region.id)?.();
+ audioElementRevokersRef.current.delete(region.id);
+ audioElementResourcesRef.current.set(region.id, region.audioPath);
+
+ void (async () => {
+ const resolved = await resolveMediaElementSource(region.audioPath);
+ const latestAudio = existing.get(region.id);
+
+ if (
+ cancelled ||
+ latestAudio !== audio ||
+ audioElementResourcesRef.current.get(region.id) !== region.audioPath
+ ) {
+ resolved.revoke();
+ return;
+ }
+
+ audioElementRevokersRef.current.set(region.id, resolved.revoke);
+ latestAudio.src = resolved.src;
+ })();
+ }
+
+ audio.volume = Math.max(0, Math.min(1, region.volume * previewVolume));
+ }
+
+ return () => {
+ cancelled = true;
+ };
+ }, [audioRegions, previewVolume]);
+
+ useEffect(() => {
+ let cancelled = false;
+ const existing = sourceAudioElementsRef.current;
+ const currentIds = new Set(previewSourceAudioFallbackPaths);
+
+ for (const [id, audio] of existing) {
+ if (!currentIds.has(id)) {
+ audio.pause();
+ audio.src = "";
+ sourceAudioElementRevokersRef.current.get(id)?.();
+ sourceAudioElementRevokersRef.current.delete(id);
+ sourceAudioElementResourcesRef.current.delete(id);
+ existing.delete(id);
+ }
+ }
+
+ for (const audioPath of previewSourceAudioFallbackPaths) {
+ let audio = existing.get(audioPath);
+ if (!audio) {
+ audio = new Audio();
+ audio.preload = "auto";
+ existing.set(audioPath, audio);
+ }
+ audio.dataset.sourceAudioPath = audioPath;
+
+ if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) {
+ audio.pause();
+ audio.src = "";
+ sourceAudioElementRevokersRef.current.get(audioPath)?.();
+ sourceAudioElementRevokersRef.current.delete(audioPath);
+ sourceAudioElementResourcesRef.current.set(audioPath, audioPath);
+
+ void (async () => {
+ try {
+ const resolved = await resolveMediaElementSource(audioPath);
+ const latestAudio = existing.get(audioPath);
+
+ if (
+ cancelled ||
+ latestAudio !== audio ||
+ sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath
+ ) {
+ resolved.revoke();
+ return;
+ }
+
+ sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke);
+ latestAudio.src = resolved.src;
+ } catch (error) {
+ if (cancelled) {
+ return;
+ }
+
+ sourceAudioElementRevokersRef.current.get(audioPath)?.();
+ sourceAudioElementRevokersRef.current.delete(audioPath);
+ sourceAudioElementResourcesRef.current.delete(audioPath);
+ const latestAudio = existing.get(audioPath);
+ if (latestAudio === audio) {
+ latestAudio.pause();
+ latestAudio.src = "";
+ }
+ onSourceFallbackLoadError(error);
+ }
+ })();
+ }
+
+ audio.volume = isCurrentClipMuted
+ ? 0
+ : Math.max(0, Math.min(1, previewVolume * getSourceTrackPreviewGain(audioPath)));
+ }
+
+ if (previewSourceAudioFallbackPaths.length === 0) {
+ lastSourceAudioSyncTimeRef.current = null;
+ }
+
+ return () => {
+ cancelled = true;
+ };
+ }, [
+ getSourceTrackPreviewGain,
+ isCurrentClipMuted,
+ onSourceFallbackLoadError,
+ previewSourceAudioFallbackPaths,
+ previewVolume,
+ ]);
+
+ useEffect(() => {
+ return () => {
+ for (const audio of audioElementsRef.current.values()) {
+ audio.pause();
+ audio.src = "";
+ }
+ for (const revoke of audioElementRevokersRef.current.values()) {
+ revoke();
+ }
+ audioElementsRef.current.clear();
+ audioElementRevokersRef.current.clear();
+ audioElementResourcesRef.current.clear();
+ for (const audio of sourceAudioElementsRef.current.values()) {
+ audio.pause();
+ audio.src = "";
+ }
+ for (const revoke of sourceAudioElementRevokersRef.current.values()) {
+ revoke();
+ }
+ sourceAudioElementsRef.current.clear();
+ sourceAudioElementRevokersRef.current.clear();
+ sourceAudioElementResourcesRef.current.clear();
+ lastSourceAudioSyncTimeRef.current = null;
+ };
+ }, []);
+
+ useEffect(() => {
+ const currentTimeMs = currentTime * 1000;
+ const activeSpeedRegion = effectiveSpeedRegions.find(
+ (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs,
+ );
+ const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
+
+ for (const region of audioRegions) {
+ const audio = audioElementsRef.current.get(region.id);
+ if (!audio) continue;
+
+ const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs;
+
+ if (isPlaying && isInRegion) {
+ enablePitchPreservingPlayback(audio);
+ const audioOffset = (currentTimeMs - region.startMs) / 1000;
+ if (Math.abs(audio.currentTime - audioOffset) > 0.2) {
+ audio.currentTime = audioOffset;
+ }
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: targetPlaybackRate,
+ currentTime: audio.currentTime,
+ targetTime: audioOffset,
+ });
+ if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
+ audio.playbackRate = syncedPlaybackRate;
+ }
+ if (audio.paused) {
+ audio.play().catch(() => undefined);
+ }
+ } else if (!audio.paused) {
+ audio.pause();
+ }
+ }
+ }, [audioRegions, currentTime, effectiveSpeedRegions, isPlaying]);
+
+ useEffect(() => {
+ if (previewSourceAudioFallbackPaths.length === 0) {
+ lastSourceAudioSyncTimeRef.current = null;
+ return;
+ }
+
+ const activeSpeedRegion = effectiveSpeedRegions.find(
+ (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs,
+ );
+ const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1;
+ const previousTimelineTime = lastSourceAudioSyncTimeRef.current;
+ const timelineJumped =
+ previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25;
+ const driftThreshold = isPlaying
+ ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS
+ : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS;
+
+ for (const audio of sourceAudioElementsRef.current.values()) {
+ enablePitchPreservingPlayback(audio);
+ const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null;
+ const startDelaySeconds = estimateCompanionAudioStartDelaySeconds(
+ duration,
+ audioDuration,
+ sourceAudioFallbackStartDelayMsByPath[audio.dataset.sourceAudioPath ?? ""],
+ );
+ const beforeAudioStart = currentTime + 0.001 < startDelaySeconds;
+ const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration);
+
+ if (timelineJumped || Math.abs(audio.currentTime - targetTime) > driftThreshold) {
+ try {
+ audio.currentTime = targetTime;
+ } catch {
+ // no-op
+ }
+ }
+
+ const syncedPlaybackRate = getMediaSyncPlaybackRate({
+ basePlaybackRate: targetPlaybackRate,
+ currentTime: audio.currentTime,
+ targetTime,
+ toleranceSeconds: SOURCE_AUDIO_PREVIEW_RATE_TOLERANCE_SECONDS,
+ correctionWindowSeconds: SOURCE_AUDIO_PREVIEW_RATE_CORRECTION_WINDOW_SECONDS,
+ maxAdjustment: SOURCE_AUDIO_PREVIEW_MAX_RATE_ADJUSTMENT,
+ });
+ if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) {
+ audio.playbackRate = syncedPlaybackRate;
+ }
+
+ const atEnd = audioDuration !== null && targetTime >= audioDuration;
+ if (isPlaying && !beforeAudioStart && !atEnd) {
+ audio.play().catch(() => undefined);
+ } else if (!audio.paused) {
+ audio.pause();
+ }
+ }
+
+ lastSourceAudioSyncTimeRef.current = currentTime;
+ }, [
+ currentTime,
+ duration,
+ effectiveSpeedRegions,
+ isPlaying,
+ previewSourceAudioFallbackPaths,
+ sourceAudioFallbackStartDelayMsByPath,
+ ]);
+}
diff --git a/src/components/video-editor/audio/useSourceAudioFallback.ts b/src/components/video-editor/audio/useSourceAudioFallback.ts
new file mode 100644
index 00000000..64b4caa8
--- /dev/null
+++ b/src/components/video-editor/audio/useSourceAudioFallback.ts
@@ -0,0 +1,68 @@
+import { useEffect, useState } from "react";
+import { toast } from "sonner";
+import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "./sourceAudioTracks";
+
+interface UseSourceAudioFallbackParams {
+ currentSourcePath: string | null;
+ summarizeErrorMessage: (message: string) => string;
+}
+
+export function useSourceAudioFallback({
+ currentSourcePath,
+ summarizeErrorMessage,
+}: UseSourceAudioFallbackParams) {
+ const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState<string[]>([]);
+ const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] =
+ useState<Record<string, number>>({});
+
+ useEffect(() => {
+ let cancelled = false;
+ setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
+
+ if (!currentSourcePath) {
+ return () => {
+ cancelled = true;
+ };
+ }
+
+ void (async () => {
+ try {
+ const result = await window.electronAPI.getVideoAudioFallbackPaths(currentSourcePath);
+ if (cancelled) {
+ return;
+ }
+ if (!result.success) {
+ setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
+ toast.warning(
+ result.error
+ ? `Could not load companion audio sources: ${summarizeErrorMessage(result.error)}`
+ : "Could not load companion audio sources. Playback and export may miss microphone audio.",
+ { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
+ );
+ return;
+ }
+
+ toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID);
+ setSourceAudioFallbackPaths(result.paths ?? []);
+ setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {});
+ } catch (error) {
+ if (!cancelled) {
+ setSourceAudioFallbackPaths([]);
+ setSourceAudioFallbackStartDelayMsByPath({});
+ toast.warning(
+ `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`,
+ { id: SOURCE_AUDIO_FALLBACK_TOAST_ID, duration: 10000 },
+ );
+ }
+ }
+ })();
+
+ return () => {
+ cancelled = true;
+ };
+ }, [currentSourcePath, summarizeErrorMessage]);
+
+ return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath };
+}
diff --git a/src/components/video-editor/audio/useSourceAudioTrackSettings.ts b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts
new file mode 100644
index 00000000..27ea0daf
--- /dev/null
+++ b/src/components/video-editor/audio/useSourceAudioTrackSettings.ts
@@ -0,0 +1,112 @@
+import { useCallback, useMemo, useState } from "react";
+
+export type SourceAudioTrackSetting = { volume: number; normalize: boolean };
+export type SourceAudioTrackSettings = Record<string, SourceAudioTrackSetting>;
+export type SourceAudioTrackMeta = Array<{ id: string; label: string }>;
+
+interface UseSourceAudioTrackSettingsParams {
+ selectedClipId: string | null;
+ activeClipId: string | null;
+}
+
+export interface UseSourceAudioTrackSettingsResult {
+ sourceAudioTrackMeta: SourceAudioTrackMeta;
+ activeSourceAudioTrackSettings: SourceAudioTrackSettings;
+ selectedClipSourceAudioTrackSettings: SourceAudioTrackSettings;
+ onSourceAudioTracksMetaChange: (tracks: SourceAudioTrackMeta) => void;
+ onSelectedClipSourceAudioTrackVolumeChange: (id: string, volume: number) => void;
+ onSelectedClipSourceAudioTrackNormalizeChange: (id: string, normalize: boolean) => void;
+}
+
+export function useSourceAudioTrackSettings({
+ selectedClipId,
+ activeClipId,
+}: UseSourceAudioTrackSettingsParams): UseSourceAudioTrackSettingsResult {
+ const [sourceAudioTrackMeta, setSourceAudioTrackMeta] = useState<SourceAudioTrackMeta>([]);
+ const [sourceAudioTrackSettingsByClip, setSourceAudioTrackSettingsByClip] = useState<
+ Record<string, SourceAudioTrackSettings>
+ >({});
+ const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState<
+ SourceAudioTrackSettings
+ >({});
+
+ const activeSourceAudioTrackSettings = useMemo(() => {
+ if (!activeClipId) {
+ return defaultSourceAudioTrackSettings;
+ }
+ return {
+ ...defaultSourceAudioTrackSettings,
+ ...(sourceAudioTrackSettingsByClip[activeClipId] ?? {}),
+ };
+ }, [activeClipId, defaultSourceAudioTrackSettings, sourceAudioTrackSettingsByClip]);
+
+ const selectedClipSourceAudioTrackSettings = useMemo(() => {
+ if (!selectedClipId) {
+ return defaultSourceAudioTrackSettings;
+ }
+ return {
+ ...defaultSourceAudioTrackSettings,
+ ...(sourceAudioTrackSettingsByClip[selectedClipId] ?? {}),
+ };
+ }, [defaultSourceAudioTrackSettings, selectedClipId, sourceAudioTrackSettingsByClip]);
+
+ const onSourceAudioTracksMetaChange = useCallback((tracks: SourceAudioTrackMeta) => {
+ setSourceAudioTrackMeta(tracks);
+ setDefaultSourceAudioTrackSettings((prev) => {
+ const next: SourceAudioTrackSettings = {};
+ for (const track of tracks) {
+ next[track.id] = prev[track.id] ?? { volume: 1, normalize: false };
+ }
+ return next;
+ });
+ }, []);
+
+ const onSelectedClipSourceAudioTrackVolumeChange = useCallback(
+ (id: string, volume: number) => {
+ if (!selectedClipId) return;
+ setSourceAudioTrackSettingsByClip((prev) => {
+ const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
+ return {
+ ...prev,
+ [selectedClipId]: {
+ ...prevClip,
+ [id]: {
+ volume: Math.max(0, Math.min(2, volume)),
+ normalize: prevClip[id]?.normalize ?? false,
+ },
+ },
+ };
+ });
+ },
+ [defaultSourceAudioTrackSettings, selectedClipId],
+ );
+
+ const onSelectedClipSourceAudioTrackNormalizeChange = useCallback(
+ (id: string, normalize: boolean) => {
+ if (!selectedClipId) return;
+ setSourceAudioTrackSettingsByClip((prev) => {
+ const prevClip = prev[selectedClipId] ?? defaultSourceAudioTrackSettings;
+ return {
+ ...prev,
+ [selectedClipId]: {
+ ...prevClip,
+ [id]: {
+ volume: prevClip[id]?.volume ?? 1,
+ normalize,
+ },
+ },
+ };
+ });
+ },
+ [defaultSourceAudioTrackSettings, selectedClipId],
+ );
+
+ return {
+ sourceAudioTrackMeta,
+ activeSourceAudioTrackSettings,
+ selectedClipSourceAudioTrackSettings,
+ onSourceAudioTracksMetaChange,
+ onSelectedClipSourceAudioTrackVolumeChange,
+ onSelectedClipSourceAudioTrackNormalizeChange,
+ };
+}
diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/audio/waveform/WaveformGenerator.ts
similarity index 95%
rename from src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts
rename to src/components/video-editor/audio/waveform/WaveformGenerator.ts
index 128a65ad..a51475d8 100644
--- a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts
+++ b/src/components/video-editor/audio/waveform/WaveformGenerator.ts
@@ -1,6 +1,6 @@
import WorkerConstructor from "./waveform.worker?worker";
-import type { AudioPeaksData } from "../../core/timelineTypes";
-import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../core/constants";
+import type { AudioPeaksData } from "../../timeline/core/timelineTypes";
+import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants";
export class WaveformGenerator {
private audioContext: AudioContext;
diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/audio/waveform/waveform.worker.ts
similarity index 100%
rename from src/components/video-editor/timeline/components/waveform/waveform.worker.ts
rename to src/components/video-editor/audio/waveform/waveform.worker.ts
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx
index cf7005e4..6fc22499 100644
--- a/src/components/video-editor/timeline/TimelineEditor.tsx
+++ b/src/components/video-editor/timeline/TimelineEditor.tsx
@@ -17,6 +17,7 @@ import {
} from "@/utils/aspectRatioUtils";
import { formatShortcut } from "@/utils/platformUtils";
import { loadEditorPreferences, saveEditorPreferences } from "../editorPreferences";
+import { fromFileUrl } from "../projectPersistence";
import type {
AnnotationRegion,
AudioRegion,
@@ -35,6 +36,7 @@ import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime";
import { useTimelineRange } from "./hooks/useTimelineRange";
import TimelineCanvas from "./components/viewport/TimelineCanvas";
import TimelineToolbar from "./components/toolbar/TimelineToolbar";
+import type { AudioPeaksData } from "./core/timelineTypes";
export interface TimelineEditorProps {
videoDuration: number;
@@ -80,6 +82,35 @@ export interface TimelineEditorProps {
isCropped?: boolean;
videoPath?: string | null;
hideToolbar?: boolean;
+ showSourceAudioTrack?: boolean;
+ onSourceAudioAvailabilityChange?: (available: boolean) => void;
+ sourceAudioTrackSettings?: Record<string, { volume: number; normalize: boolean }>;
+ onSourceAudioTracksMetaChange?: (tracks: Array<{ id: string; label: string }>) => void;
+}
+
+function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null {
+ if (!input) return null;
+ try {
+ const url = new URL(input);
+ const isLocalMediaServer =
+ (url.protocol === "http:" || url.protocol === "https:") &&
+ (url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
+ url.pathname === "/video";
+ if (!isLocalMediaServer) return null;
+ return url.searchParams.get("path");
+ } catch {
+ return null;
+ }
+}
+
+function buildSourceSidecarPath(source: string, suffix: "mic" | "system"): string {
+ const normalized = source.replace(/\\/g, "/");
+ const lastSlash = normalized.lastIndexOf("/");
+ const dir = lastSlash >= 0 ? normalized.slice(0, lastSlash + 1) : "";
+ const fileName = lastSlash >= 0 ? normalized.slice(lastSlash + 1) : normalized;
+ const dotIndex = fileName.lastIndexOf(".");
+ const baseName = dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
+ return `${dir}${baseName}.${suffix}.wav`;
}
export interface TimelineEditorHandle {
@@ -138,6 +169,10 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
isCropped = false,
videoPath,
hideToolbar = false,
+ showSourceAudioTrack = false,
+ onSourceAudioAvailabilityChange,
+ sourceAudioTrackSettings = {},
+ onSourceAudioTracksMetaChange,
},
ref,
) {
@@ -229,9 +264,65 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
- const audioPeaks = useTimelineAudioPeaks(videoPath, {
+ const sourceAudioPeaks = useTimelineAudioPeaks(videoPath, {
enableSourceSidecarFallback: true,
});
+ const localSourcePath = useMemo(() => {
+ if (!videoPath) return null;
+ return (
+ extractLocalPathFromMediaServerUrl(videoPath) ||
+ (/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath)
+ );
+ }, [videoPath]);
+ const micSidecarPath = useMemo(
+ () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "mic") : null),
+ [localSourcePath],
+ );
+ const systemSidecarPath = useMemo(
+ () => (localSourcePath ? buildSourceSidecarPath(localSourcePath, "system") : null),
+ [localSourcePath],
+ );
+ const micSidecarPeaks = useTimelineAudioPeaks(micSidecarPath);
+ const systemSidecarPeaks = useTimelineAudioPeaks(systemSidecarPath);
+ const sourceAudioTracks = useMemo<Array<{ id: string; label: string; peaks: AudioPeaksData }>>(() => {
+ if (systemSidecarPeaks || micSidecarPeaks) {
+ const tracks: Array<{ id: string; label: string; peaks: AudioPeaksData }> = [];
+ if (systemSidecarPeaks) tracks.push({ id: "system", label: "Source System", peaks: systemSidecarPeaks });
+ if (micSidecarPeaks) tracks.push({ id: "mic", label: "Source Mic", peaks: micSidecarPeaks });
+ return tracks;
+ }
+ return sourceAudioPeaks ? [{ id: "mixed", label: "Source", peaks: sourceAudioPeaks }] : [];
+ }, [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks]);
+ useEffect(() => {
+ onSourceAudioTracksMetaChange?.(sourceAudioTracks.map((t) => ({ id: t.id, label: t.label })));
+ }, [onSourceAudioTracksMetaChange, sourceAudioTracks]);
+ const displaySourceAudioTracks = useMemo(() => {
+ return sourceAudioTracks.map((track) => {
+ const settings = sourceAudioTrackSettings[track.id] ?? { volume: 1, normalize: false };
+ const volume = Math.max(0, Math.min(2, settings.volume));
+ const normalize = settings.normalize;
+ const input = track.peaks.peaks;
+ const adjusted = new Float32Array(input.length);
+ for (let i = 0; i < input.length; i++) {
+ let amp = input[i];
+ if (normalize) {
+ amp = Math.sqrt(amp);
+ }
+ adjusted[i] = Math.max(0, Math.min(1, amp * volume));
+ }
+ return {
+ id: track.id,
+ label: track.label,
+ peaks: {
+ durationMs: track.peaks.durationMs,
+ peaks: adjusted,
+ } satisfies AudioPeaksData,
+ };
+ });
+ }, [sourceAudioTrackSettings, sourceAudioTracks]);
+ useEffect(() => {
+ onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0);
+ }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]);
useEffect(() => {
if (aspectRatio === "native") {
@@ -474,7 +565,8 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
selectAllBlocksActive={selectAllBlocksActive}
onClearBlockSelection={clearSelectedBlocks}
keyframes={keyframes}
- audioPeaks={audioPeaks}
+ sourceAudioTracks={displaySourceAudioTracks}
+ showSourceAudioTrack={showSourceAudioTrack}
liveSpanPreviewById={liveZoomPreview.previewSpans}
liveHiddenItemIds={Array.from(liveZoomPreview.hiddenZoomIds)}
/>
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
index d0dae8c2..94d4cd88 100644
--- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
+++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
@@ -57,7 +57,8 @@ interface TimelineCanvasProps {
selectAllBlocksActive?: boolean;
onClearBlockSelection?: () => void;
keyframes?: { id: string; time: number }[];
- audioPeaks?: AudioPeaksData | null;
+ sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>;
+ showSourceAudioTrack?: boolean;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
}
@@ -220,7 +221,8 @@ interface TimelineCanvasRowsProps {
onSelectClip?: (id: string | null) => void;
onSelectAnnotation?: (id: string | null) => void;
onSelectAudio?: (id: string | null) => void;
- audioPeaks?: AudioPeaksData | null;
+ sourceAudioTracks?: Array<{ id: string; label: string; peaks: AudioPeaksData }>;
+ showSourceAudioTrack?: boolean;
liveSpanPreviewById?: Record<string, { start: number; end: number }>;
liveHiddenItemIds?: string[];
direction: string;
@@ -282,7 +284,8 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
onSelectClip,
onSelectAnnotation,
onSelectAudio,
- audioPeaks,
+ sourceAudioTracks = [],
+ showSourceAudioTrack = false,
liveSpanPreviewById,
liveHiddenItemIds,
direction,
@@ -365,26 +368,27 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
</Item>
))}
</Row>
- {audioPeaks && (
- <Row id={SOURCE_AUDIO_ROW_ID}>
- {clipItems.map((item) => (
- <Item
- key={`source-audio-${item.id}`}
- id={`source-audio-${item.id}`}
- rowId={SOURCE_AUDIO_ROW_ID}
- span={liveSpanPreviewById?.[item.id] ?? item.span}
- disabled
- isSelected={selectAllBlocksActive || item.id === selectedClipId}
- onSelect={() => onSelectClip?.(item.id)}
- variant="audio"
- waveformPeaks={audioPeaks}
- waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span}
- >
- Source
- </Item>
- ))}
- </Row>
- )}
+ {showSourceAudioTrack &&
+ sourceAudioTracks.map((track) => (
+ <Row key={track.id} id={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}>
+ {clipItems.map((item) => (
+ <Item
+ key={`source-audio-${track.id}-${item.id}`}
+ id={`source-audio-${track.id}-${item.id}`}
+ rowId={`${SOURCE_AUDIO_ROW_ID}-${track.id}`}
+ span={liveSpanPreviewById?.[item.id] ?? item.span}
+ disabled
+ isSelected={selectAllBlocksActive || item.id === selectedClipId}
+ onSelect={() => onSelectClip?.(item.id)}
+ variant="audio"
+ waveformPeaks={track.peaks}
+ waveformSegmentSpan={liveSpanPreviewById?.[item.id] ?? item.span}
+ >
+ {track.label}
+ </Item>
+ ))}
+ </Row>
+ ))}
<Row
id={ZOOM_ROW_ID}
@@ -492,7 +496,8 @@ export default function TimelineCanvas({
selectAllBlocksActive = false,
onClearBlockSelection,
keyframes = [],
- audioPeaks,
+ sourceAudioTracks = [],
+ showSourceAudioTrack = false,
liveSpanPreviewById,
liveHiddenItemIds,
}: TimelineCanvasProps) {
@@ -646,9 +651,9 @@ export default function TimelineCanvas({
if (isAnnotationTrackRowId(item.rowId)) annotationRowIds.add(item.rowId);
if (isAudioTrackRowId(item.rowId)) audioRowIds.add(item.rowId);
}
- const sourceAudioRows = audioPeaks ? 1 : 0;
+ const sourceAudioRows = showSourceAudioTrack ? sourceAudioTracks.length : 0;
return 2 + sourceAudioRows + annotationRowIds.size + audioRowIds.size;
- }, [items, audioPeaks]);
+ }, [items, showSourceAudioTrack, sourceAudioTracks.length]);
const timelineRowsMinHeightPx = getTimelineRowsMinHeightPx(timelineRowCount);
const timelineContentMinHeightPx = getTimelineContentMinHeightPx(timelineRowCount);
const timelineViewportStretchFactor = getTimelineViewportStretchFactor(timelineRowCount);
@@ -724,7 +729,8 @@ export default function TimelineCanvas({
onSelectClip={onSelectClip}
onSelectAnnotation={onSelectAnnotation}
onSelectAudio={onSelectAudio}
- audioPeaks={audioPeaks}
+ sourceAudioTracks={sourceAudioTracks}
+ showSourceAudioTrack={showSourceAudioTrack}
liveSpanPreviewById={liveSpanPreviewById}
liveHiddenItemIds={liveHiddenItemIds}
direction={direction}
diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
index 499bab2b..fa750e29 100644
--- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
+++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
@@ -1,7 +1,7 @@
import { useEffect, useRef, useState } from "react";
import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
import { fromFileUrl } from "../../projectPersistence";
-import { waveformGenerator } from "../components/waveform/WaveformGenerator";
+import { waveformGenerator } from "../../audio/waveform/WaveformGenerator";
import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../core/constants";
import type { AudioPeaksData } from "../core/timelineTypes";