The open-source CapCut alternative
feat: separate audio from elements
Maze Wintherf1d5932c5417
diff --git a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsxindex 4dbc1a17..5ac0fbb7 100644--- a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx+++ b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx@@ -1,5 +1,7 @@+import { Button } from "@/components/ui/button";import { NumberField } from "@/components/ui/number-field";import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/lib/timeline/audio-constants";+import { isSourceAudioSeparated } from "@/lib/timeline/audio-separation";import { DEFAULTS } from "@/lib/timeline/defaults";import {clamp,@@ -10,6 +12,7 @@ import {} from "@/utils/math";import type { AudioElement, VideoElement } from "@/lib/timeline";import { resolveNumberAtTime } from "@/lib/animation";+import { useEditor } from "@/hooks/use-editor";import { useElementPlayhead } from "../hooks/use-element-playhead";import { useKeyframedNumberProperty } from "../hooks/use-keyframed-number-property";import { KeyframeToggle } from "../components/keyframe-toggle";@@ -34,6 +37,7 @@ export function AudioTab({element: AudioElement | VideoElement;trackId: string;}) {+ const editor = useEditor();const { localTime, isPlayheadWithinElementRange } = useElementPlayhead({startTime: element.startTime,duration: element.duration,@@ -80,46 +84,68 @@ export function AudioTab({rightValue: DEFAULTS.element.volume,}): (element.volume ?? DEFAULTS.element.volume) === DEFAULTS.element.volume;+ const isSeparated =+ element.type === "video" && isSourceAudioSeparated({ element });return (- <Section collapsible sectionKey={`${element.id}:audio`}>- <SectionHeader>- <SectionTitle>Audio</SectionTitle>- </SectionHeader>- <SectionContent>- <SectionFields>- <SectionField- label="Volume"- beforeLabel={- <KeyframeToggle- isActive={volume.isKeyframedAtTime}- isDisabled={!isPlayheadWithinElementRange}- title="Toggle volume keyframe"- onToggle={volume.toggleKeyframe}- />+ <>+ {isSeparated && (+ <div className="mx-4 mt-4 rounded-md border bg-muted/30 p-3">+ <p className="text-sm">Audio has been separated.</p>+ <Button+ className="mt-3"+ size="sm"+ variant="secondary"+ onClick={() =>+ editor.timeline.toggleSourceAudioSeparation({+ trackId,+ elementId: element.id,+ })}>- <NumberField- icon={<HugeiconsIcon icon={VolumeHighIcon} />}- value={volume.displayValue}- onFocus={volume.onFocus}- onChange={volume.onChange}- onBlur={volume.onBlur}- dragSensitivity="slow"- scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }}- onScrub={volume.scrubTo}- onScrubEnd={volume.commitScrub}- onReset={() =>- volume.commitValue({- value: DEFAULTS.element.volume,- })+ Recover audio+ </Button>+ </div>+ )}+ <Section collapsible sectionKey={`${element.id}:audio`}>+ <SectionHeader>+ <SectionTitle>Audio</SectionTitle>+ </SectionHeader>+ <SectionContent>+ <SectionFields>+ <SectionField+ label="Volume"+ beforeLabel={+ <KeyframeToggle+ isActive={volume.isKeyframedAtTime}+ isDisabled={!isPlayheadWithinElementRange}+ title="Toggle volume keyframe"+ onToggle={volume.toggleKeyframe}+ />}- isDefault={isDefault}- suffix="dB"- />- </SectionField>- </SectionFields>- </SectionContent>- </Section>+ >+ <NumberField+ icon={<HugeiconsIcon icon={VolumeHighIcon} />}+ value={volume.displayValue}+ onFocus={volume.onFocus}+ onChange={volume.onChange}+ onBlur={volume.onBlur}+ dragSensitivity="slow"+ scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }}+ onScrub={volume.scrubTo}+ onScrubEnd={volume.commitScrub}+ onReset={() =>+ volume.commitValue({+ value: DEFAULTS.element.volume,+ })+ }+ isDefault={isDefault}+ suffix="dB"+ />+ </SectionField>+ </SectionFields>+ </SectionContent>+ </Section>+ </>);}diff --git a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx b/apps/web/src/components/editor/panels/timeline/timeline-element.tsxindex f196da44..bb30b0b0 100644--- a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx+++ b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx@@ -39,6 +39,11 @@ import type {} from "@/lib/timeline";import type { MediaAsset } from "@/lib/media/types";import { mediaSupportsAudio } from "@/lib/media/media-utils";+import {+ canToggleSourceAudio,+ getSourceAudioActionLabel,+ isSourceAudioSeparated,+} from "@/lib/timeline/audio-separation";import {getActionDefinition,type TAction,@@ -61,7 +66,9 @@ import {Search01Icon,Exchange01Icon,KeyframeIcon,+ Link02Icon,MagicWand05Icon,+ Unlink02Icon,} from "@hugeicons/core-free-icons";import { HugeiconsIcon } from "@hugeicons/react";import { uppercase } from "@/utils/string";@@ -280,6 +287,19 @@ export function TimelineElement({};const isMuted = canElementHaveAudio(element) && element.muted === true;+ const canToggleCurrentSourceAudio =+ selectedElements.length === 1 &&+ isCurrentElementSelected &&+ canToggleSourceAudio({+ element,+ mediaAsset,+ });+ const sourceAudioLabel =+ element.type === "video"+ ? getSourceAudioActionLabel({ element })+ : "Extract audio";+ const isElementSourceAudioSeparated =+ element.type === "video" && isSourceAudioSeparated({ element });return (<ContextMenu>@@ -334,6 +354,21 @@ export function TimelineElement({isMuted={isMuted}/>)}+ {canToggleCurrentSourceAudio && (+ <ContextMenuItem+ icon={+ <HugeiconsIcon+ icon={isElementSourceAudioSeparated ? Unlink02Icon : Link02Icon}+ />+ }+ onClick={(event: React.MouseEvent) => {+ event.stopPropagation();+ invokeAction("toggle-source-audio");+ }}+ >+ {sourceAudioLabel}+ </ContextMenuItem>+ )}{canElementBeHidden(element) && (<VisibilityMenuItemelement={element}diff --git a/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx b/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsxindex f398264c..abe36f88 100644--- a/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx+++ b/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx@@ -1,4 +1,5 @@import { useEditor } from "@/hooks/use-editor";+import { useElementSelection } from "@/hooks/timeline/element/use-element-selection";import {TooltipProvider,Tooltip,@@ -6,7 +7,6 @@ import {TooltipContent,} from "@/components/ui/tooltip";import { Button } from "@/components/ui/button";-import { SplitSquareHorizontal } from "lucide-react";import {SplitButton,SplitButtonLeft,@@ -18,6 +18,12 @@ import { TIMELINE_CONSTANTS } from "@/constants/timeline-constants";import { sliderToZoom, zoomToSlider } from "@/lib/timeline/zoom-utils";import { ScenesView } from "@/components/editor/scenes-view";import { type TActionWithOptionalArgs, invokeAction } from "@/lib/actions";+import {+ canToggleSourceAudio,+ getSourceAudioActionLabel,+ isSourceAudioSeparated,+} from "@/lib/timeline/audio-separation";+import { hasMediaId } from "@/lib/timeline";import { cn } from "@/utils/ui";import { useTimelineStore } from "@/stores/timeline-store";import { ScrollArea } from "@/components/ui/scroll-area";@@ -33,8 +39,10 @@ import {Copy01Icon,AlignLeftIcon,AlignRightIcon,+ Link02Icon,Layers01Icon,Chart03Icon,+ Unlink02Icon,} from "@hugeicons/core-free-icons";import { HugeiconsIcon } from "@hugeicons/react";import { OcRippleIcon } from "@/components/icons";@@ -78,9 +86,47 @@ export function TimelineToolbar({}function ToolbarLeftSection() {+ const editor = useEditor();+ const mediaAssets = useEditor((currentEditor) => currentEditor.media.getAssets());+ const { selectedElements } = useElementSelection();const isCurrentlyBookmarked = useEditor((e) =>e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }),);+ const selectedElement =+ selectedElements.length === 1+ ? (editor.timeline.getElementsWithTracks({+ elements: selectedElements,+ })[0] ?? null)+ : null;+ const selectedMediaAsset = (() => {+ if (!selectedElement) {+ return null;+ }++ const { element } = selectedElement;+ if (!hasMediaId(element)) {+ return null;+ }++ return mediaAssets.find((asset) => asset.id === element.mediaId) ?? null;+ })();+ const canToggleSelectedSourceAudio =+ !!selectedElement &&+ canToggleSourceAudio({+ element: selectedElement.element,+ mediaAsset: selectedMediaAsset,+ });+ const sourceAudioLabel =+ selectedElement?.element.type === "video"+ ? getSourceAudioActionLabel({+ element: selectedElement.element,+ })+ : "Extract audio";+ const isSelectedSourceAudioSeparated =+ selectedElement?.element.type === "video" &&+ isSourceAudioSeparated({+ element: selectedElement.element,+ });const handleAction = ({action,@@ -117,10 +163,16 @@ function ToolbarLeftSection() {/><ToolbarButton- icon={<SplitSquareHorizontal />}- tooltip="Separate audio (coming soon)"- disabled={true}- onClick={({ event: _event }) => {}}+ icon={+ <HugeiconsIcon+ icon={isSelectedSourceAudioSeparated ? Unlink02Icon : Link02Icon}+ />+ }+ tooltip={sourceAudioLabel}+ disabled={!canToggleSelectedSourceAudio}+ onClick={({ event }) =>+ handleAction({ action: "toggle-source-audio", event })+ }/><ToolbarButton@@ -275,6 +327,7 @@ function ToolbarButton({<Buttonvariant={isActive ? "secondary" : "text"}size="icon"+ disabled={disabled}onClick={(event) => onClick({ event })}className={cn("rounded-sm",diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.tsindex 263de41f..db8c684f 100644--- a/apps/web/src/core/managers/timeline-manager.ts+++ b/apps/web/src/core/managers/timeline-manager.ts@@ -45,6 +45,7 @@ import {ToggleMaskInvertedCommand,UpsertEffectParamKeyframeCommand,RemoveEffectParamKeyframeCommand,+ ToggleSourceAudioSeparationCommand,} from "@/lib/commands/timeline";import { BatchCommand } from "@/lib/commands";import type { InsertElementParams } from "@/lib/commands/timeline/element/insert-element";@@ -281,6 +282,20 @@ export class TimelineManager {this.editor.command.execute({ command });}+ toggleSourceAudioSeparation({+ trackId,+ elementId,+ }: {+ trackId: string;+ elementId: string;+ }): void {+ const command = new ToggleSourceAudioSeparationCommand({+ trackId,+ elementId,+ });+ this.editor.command.execute({ command });+ }+updateElements({updates,pushHistory = true,diff --git a/apps/web/src/hooks/actions/use-editor-actions.ts b/apps/web/src/hooks/actions/use-editor-actions.tsindex bfac931b..087ea687 100644--- a/apps/web/src/hooks/actions/use-editor-actions.ts+++ b/apps/web/src/hooks/actions/use-editor-actions.ts@@ -6,9 +6,13 @@ import { useActionHandler } from "@/hooks/actions/use-action-handler";import { useEditor } from "../use-editor";import { useElementSelection } from "../timeline/element/use-element-selection";import { useKeyframeSelection } from "../timeline/element/use-keyframe-selection";-import { getElementsAtTime } from "@/lib/timeline";+import {+ getElementsAtTime,+ hasMediaId,+} from "@/lib/timeline";import { cancelInteraction } from "@/lib/cancel-interaction";import { invokeAction } from "@/lib/actions";+import { canToggleSourceAudio } from "@/lib/timeline/audio-separation";import {activateScope,clearActiveScope,@@ -266,6 +270,48 @@ export function useEditorActions() {undefined,);+ useActionHandler(+ "toggle-source-audio",+ () => {+ if (selectedElements.length !== 1) {+ return;+ }++ const selectedElement = editor.timeline.getElementsWithTracks({+ elements: selectedElements,+ })[0];+ if (!selectedElement) {+ return;+ }++ const mediaAsset = (() => {+ const { element } = selectedElement;+ if (!hasMediaId(element)) {+ return null;+ }++ return (+ editor.media.getAssets().find((asset) => asset.id === element.mediaId) ??+ null+ );+ })();+ if (+ !canToggleSourceAudio({+ element: selectedElement.element,+ mediaAsset,+ })+ ) {+ return;+ }++ editor.timeline.toggleSourceAudioSeparation({+ trackId: selectedElement.track.id,+ elementId: selectedElement.element.id,+ });+ },+ undefined,+ );+useActionHandler("select-all",() => {diff --git a/apps/web/src/lib/actions/definitions.ts b/apps/web/src/lib/actions/definitions.tsindex e48126d0..3059a2e3 100644--- a/apps/web/src/lib/actions/definitions.ts+++ b/apps/web/src/lib/actions/definitions.ts@@ -101,6 +101,10 @@ export const ACTIONS = {description: "Toggle ripple editing",category: "editing",},+ "toggle-source-audio": {+ description: "Extract or recover source audio",+ category: "editing",+ },"select-all": {description: "Select all elements",category: "selection",diff --git a/apps/web/src/lib/commands/timeline/element/index.ts b/apps/web/src/lib/commands/timeline/element/index.tsindex a7e513fe..5ce48d55 100644--- a/apps/web/src/lib/commands/timeline/element/index.ts+++ b/apps/web/src/lib/commands/timeline/element/index.ts@@ -8,6 +8,7 @@ export { SplitElementsCommand } from "./split-elements";export { UpdateElementCommand } from "./update-element";export { ToggleElementsVisibilityCommand } from "./toggle-elements-visibility";export { ToggleElementsMutedCommand } from "./toggle-elements-muted";+export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation";export { MoveElementCommand } from "./move-elements";export * from "./keyframes";export * from "./effects";diff --git a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.tsnew file mode 100644index 00000000..d0907c5b--- /dev/null+++ b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts@@ -0,0 +1,143 @@+import { EditorCore } from "@/core";+import { Command } from "@/lib/commands/base-command";+import {+ buildSeparatedAudioElement,+ canExtractSourceAudio,+ canRecoverSourceAudio,+} from "@/lib/timeline/audio-separation";+import {+ applyPlacement,+ resolveTrackPlacement,+} from "@/lib/timeline/placement";+import { updateElementInTracks } from "@/lib/timeline/track-element-update";+import type { TimelineTrack, VideoElement } from "@/lib/timeline/types";+import { generateUUID } from "@/utils/id";++export class ToggleSourceAudioSeparationCommand extends Command {+ private savedState: TimelineTrack[] | null = null;++ constructor(+ private readonly params: {+ trackId: string;+ elementId: string;+ },+ ) {+ super();+ }++ execute(): void {+ const editor = EditorCore.getInstance();+ this.savedState = editor.timeline.getTracks();++ const trackIndex = this.savedState.findIndex(+ (track) => track.id === this.params.trackId,+ );+ if (trackIndex < 0) {+ return;+ }++ const sourceTrack = this.savedState[trackIndex];+ const sourceElement = sourceTrack.elements.find(+ (element) => element.id === this.params.elementId,+ );+ if (!sourceElement) {+ return;+ }++ if (canRecoverSourceAudio({ element: sourceElement })) {+ editor.timeline.updateTracks(+ updateSourceAudioEnabled({+ tracks: this.savedState,+ trackId: this.params.trackId,+ elementId: this.params.elementId,+ isSourceAudioEnabled: true,+ }),+ );+ return;+ }++ const mediaAsset = editor+ .media+ .getAssets()+ .find((asset) =>+ sourceElement.type === "video" ? asset.id === sourceElement.mediaId : false,+ );+ if (!canExtractSourceAudio({ element: sourceElement, mediaAsset })) {+ return;+ }+ if (sourceElement.duration <= 0) {+ return;+ }++ const separatedAudioElement = {+ ...buildSeparatedAudioElement({+ sourceElement,+ }),+ id: generateUUID(),+ };+ const placementResult = resolveTrackPlacement({+ tracks: this.savedState,+ trackType: "audio",+ timeSpans: [+ {+ startTime: separatedAudioElement.startTime,+ duration: separatedAudioElement.duration,+ },+ ],+ strategy: { type: "aboveSource", sourceTrackIndex: trackIndex },+ });+ if (!placementResult) {+ return;+ }++ const appliedPlacement = applyPlacement({+ tracks: this.savedState,+ placementResult,+ elements: [separatedAudioElement],+ });+ if (!appliedPlacement) {+ return;+ }++ editor.timeline.updateTracks(+ updateSourceAudioEnabled({+ tracks: appliedPlacement.updatedTracks,+ trackId: this.params.trackId,+ elementId: this.params.elementId,+ isSourceAudioEnabled: false,+ }),+ );+ }++ undo(): void {+ if (!this.savedState) {+ return;+ }++ const editor = EditorCore.getInstance();+ editor.timeline.updateTracks(this.savedState);+ }+}++function updateSourceAudioEnabled({+ tracks,+ trackId,+ elementId,+ isSourceAudioEnabled,+}: {+ tracks: TimelineTrack[];+ trackId: string;+ elementId: string;+ isSourceAudioEnabled: boolean;+}): TimelineTrack[] {+ return updateElementInTracks({+ tracks,+ trackId,+ elementId,+ elementPredicate: (element): element is VideoElement => element.type === "video",+ update: (element) => ({+ ...element,+ isSourceAudioEnabled,+ }),+ });+}diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.tsindex 41d3b38e..0afc42fb 100644--- a/apps/web/src/lib/media/audio.ts+++ b/apps/web/src/lib/media/audio.ts@@ -12,7 +12,10 @@ import {hasAnimatedVolume,resolveEffectiveAudioGain,} from "@/lib/timeline/audio-state";-import { canElementHaveAudio } from "@/lib/timeline/element-utils";+import {+ doesElementHaveEnabledAudio,+} from "@/lib/timeline/audio-separation";+import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";import { canTracktHaveAudio } from "@/lib/timeline";import { mediaSupportsAudio } from "@/lib/media/media-utils";import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime";@@ -100,6 +103,11 @@ export async function collectAudioElements({if (!canElementHaveAudio(element)) continue;if (element.duration <= 0) continue;+ const mediaAsset = hasMediaId(element)+ ? (mediaMap.get(element.mediaId) ?? null)+ : null;+ if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;+const isTrackMuted = canTracktHaveAudio(track) && track.muted;if (element.type === "audio") {@@ -132,7 +140,6 @@ export async function collectAudioElements({}if (element.type === "video") {- const mediaAsset = mediaMap.get(element.mediaId);if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;pendingElements.push(@@ -451,6 +458,10 @@ export async function collectAudioMixSources({for (const element of track.elements) {if (!canElementHaveAudio(element)) continue;if (element.muted === true) continue;+ const mediaAsset = hasMediaId(element)+ ? (mediaMap.get(element.mediaId) ?? null)+ : null;+ if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;const volume = resolveEffectiveAudioGain({element,localTime: 0,@@ -473,10 +484,7 @@ export async function collectAudioMixSources({}if (element.type === "video") {- const mediaAsset = mediaMap.get(element.mediaId);- if (!mediaAsset) continue;-- if (mediaSupportsAudio({ media: mediaAsset })) {+ if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) {audioMixSources.push(collectMediaAudioSource({ element, mediaAsset, volume }),);@@ -512,6 +520,11 @@ export async function collectAudioClips({for (const element of track.elements) {if (!canElementHaveAudio(element)) continue;+ const mediaAsset = hasMediaId(element)+ ? (mediaMap.get(element.mediaId) ?? null)+ : null;+ if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;+const isElementMuted ="muted" in element ? (element.muted ?? false) : false;const muted = isTrackMuted || isElementMuted;@@ -543,10 +556,7 @@ export async function collectAudioClips({}if (element.type === "video") {- const mediaAsset = mediaMap.get(element.mediaId);- if (!mediaAsset) continue;-- if (mediaSupportsAudio({ media: mediaAsset })) {+ if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) {clips.push(collectMediaAudioClip({element,diff --git a/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.tsnew file mode 100644index 00000000..173b6402--- /dev/null+++ b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts@@ -0,0 +1,147 @@+import { describe, expect, test } from "bun:test";+import type { AudioElement, VideoElement } from "@/lib/timeline";+import {+ buildSeparatedAudioElement,+ doesElementHaveEnabledAudio,+ isSourceAudioEnabled,+ isSourceAudioSeparated,+} from "..";++describe("audio separation", () => {+ test("treats missing source audio state as enabled", () => {+ const element = buildVideoElement({});++ expect(isSourceAudioEnabled({ element })).toBe(true);+ expect(isSourceAudioSeparated({ element })).toBe(false);+ });++ test("builds a detached audio element from the source clip audio state", () => {+ const element = buildVideoElement({+ duration: 8,+ startTime: 3,+ trimStart: 1.5,+ trimEnd: 0.5,+ sourceDuration: 12,+ volume: -6,+ muted: true,+ retime: { rate: 1.25, maintainPitch: true },+ animations: {+ channels: {+ volume: {+ valueKind: "number",+ keyframes: [+ {+ id: "volume-keyframe",+ time: 2,+ value: -12,+ interpolation: "linear",+ },+ ],+ },+ opacity: {+ valueKind: "number",+ keyframes: [+ {+ id: "opacity-keyframe",+ time: 1,+ value: 0.5,+ interpolation: "linear",+ },+ ],+ },+ },+ },+ });++ const separatedAudioElement = buildSeparatedAudioElement({+ sourceElement: element,+ });++ expect(separatedAudioElement).toMatchObject({+ type: "audio",+ sourceType: "upload",+ mediaId: element.mediaId,+ name: element.name,+ duration: 8,+ startTime: 3,+ trimStart: 1.5,+ trimEnd: 0.5,+ sourceDuration: 12,+ volume: -6,+ muted: true,+ retime: { rate: 1.25, maintainPitch: true },+ });+ expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([+ "volume",+ ]);+ expect(+ separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id,+ ).not.toBe("volume-keyframe");+ });++ test("skips source audio collection when the source clip is separated", () => {+ const mediaAsset = {+ id: "media-1",+ type: "video",+ name: "Clip",+ size: 1,+ lastModified: 1,+ file: new File(["video"], "clip.mp4", { type: "video/mp4" }),+ url: "blob:clip",+ hasAudio: true,+ };+ const videoElement = buildVideoElement({+ isSourceAudioEnabled: false,+ });+ const audioElement = {+ id: "audio-1",+ type: "audio",+ sourceType: "upload",+ mediaId: "audio-media-1",+ name: "Detached audio",+ duration: 5,+ startTime: 0,+ trimStart: 0,+ trimEnd: 0,+ volume: 0,+ } as AudioElement;++ expect(+ doesElementHaveEnabledAudio({+ element: videoElement,+ mediaAsset,+ }),+ ).toBe(false);+ expect(+ doesElementHaveEnabledAudio({+ element: audioElement,+ }),+ ).toBe(true);+ });+});++function buildVideoElement(+ overrides: Partial<VideoElement>,+): VideoElement {+ return {+ id: "video-1",+ type: "video",+ mediaId: "media-1",+ name: "Clip",+ duration: 5,+ startTime: 0,+ trimStart: 0,+ trimEnd: 0,+ volume: 0,+ muted: false,+ isSourceAudioEnabled: true,+ transform: {+ scaleX: 1,+ scaleY: 1,+ position: { x: 0, y: 0 },+ rotate: 0,+ },+ opacity: 1,+ ...overrides,+ };+}diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.tsnew file mode 100644index 00000000..b92cb79b--- /dev/null+++ b/apps/web/src/lib/timeline/audio-separation/index.ts@@ -0,0 +1,133 @@+import { cloneAnimations, getChannel } from "@/lib/animation";+import type { ElementAnimations } from "@/lib/animation/types";+import type { MediaAsset } from "@/lib/media/types";+import { DEFAULTS } from "@/lib/timeline/defaults";+import type {+ CreateUploadAudioElement,+ TimelineElement,+ AudioElement,+ VideoElement,+} from "../types";++export function isSourceAudioEnabled({+ element,+}: {+ element: VideoElement;+}): boolean {+ return element.isSourceAudioEnabled !== false;+}++export function isSourceAudioSeparated({+ element,+}: {+ element: VideoElement;+}): boolean {+ return !isSourceAudioEnabled({ element });+}++export function canExtractSourceAudio({+ element,+ mediaAsset,+}: {+ element: TimelineElement;+ mediaAsset: MediaAsset | null | undefined;+}): element is VideoElement {+ return (+ element.type === "video" &&+ isSourceAudioEnabled({ element }) &&+ !!mediaAsset &&+ mediaAsset.hasAudio !== false+ );+}++export function canRecoverSourceAudio({+ element,+}: {+ element: TimelineElement;+}): element is VideoElement {+ return element.type === "video" && isSourceAudioSeparated({ element });+}++export function canToggleSourceAudio({+ element,+ mediaAsset,+}: {+ element: TimelineElement;+ mediaAsset: MediaAsset | null | undefined;+}): element is VideoElement {+ return (+ canRecoverSourceAudio({ element }) ||+ canExtractSourceAudio({ element, mediaAsset })+ );+}++export function doesElementHaveEnabledAudio({+ element,+ mediaAsset,+}: {+ element: AudioElement | VideoElement;+ mediaAsset?: MediaAsset | null;+}): boolean {+ if (element.type === "audio") {+ return true;+ }++ return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element });+}++export function buildSeparatedAudioElement({+ sourceElement,+}: {+ sourceElement: VideoElement;+}): CreateUploadAudioElement {+ return {+ type: "audio",+ sourceType: "upload",+ mediaId: sourceElement.mediaId,+ name: sourceElement.name,+ duration: sourceElement.duration,+ startTime: sourceElement.startTime,+ trimStart: sourceElement.trimStart,+ trimEnd: sourceElement.trimEnd,+ sourceDuration: sourceElement.sourceDuration,+ volume: sourceElement.volume ?? DEFAULTS.element.volume,+ muted: sourceElement.muted ?? false,+ retime: sourceElement.retime+ ? {+ rate: sourceElement.retime.rate,+ maintainPitch: sourceElement.retime.maintainPitch,+ }+ : undefined,+ animations: cloneVolumeAnimations({+ animations: sourceElement.animations,+ }),+ };+}++export function getSourceAudioActionLabel({+ element,+}: {+ element: VideoElement;+}): "Extract audio" | "Recover audio" {+ return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio";+}++function cloneVolumeAnimations({+ animations,+}: {+ animations: ElementAnimations | undefined;+}): ElementAnimations | undefined {+ const volumeChannel = getChannel({ animations, propertyPath: "volume" });+ if (!volumeChannel) {+ return undefined;+ }++ return cloneAnimations({+ animations: {+ channels: {+ volume: volumeChannel,+ },+ },+ shouldRegenerateKeyframeIds: true,+ });+}diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.tsindex ed7bb174..5820b96d 100644--- a/apps/web/src/lib/timeline/element-utils.ts+++ b/apps/web/src/lib/timeline/element-utils.ts@@ -242,6 +242,7 @@ function buildVideoElement({trimEnd: 0,sourceDuration: duration,muted: false,+ isSourceAudioEnabled: true,hidden: false,transform: {...DEFAULTS.element.transform,diff --git a/apps/web/src/lib/timeline/index.ts b/apps/web/src/lib/timeline/index.tsindex d4337833..83a6a0f0 100644--- a/apps/web/src/lib/timeline/index.ts+++ b/apps/web/src/lib/timeline/index.ts@@ -5,6 +5,7 @@ export * from "./drag";export * from "./track-utils";export * from "./track-element-update";export * from "./element-utils";+export * from "./audio-separation";export * from "./zoom-utils";export * from "./ruler-utils";export * from "./ripple-utils";diff --git a/apps/web/src/lib/timeline/types.ts b/apps/web/src/lib/timeline/types.tsindex 69d363c7..ebdc4c23 100644--- a/apps/web/src/lib/timeline/types.ts+++ b/apps/web/src/lib/timeline/types.ts@@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement {mediaId: string;volume?: number;muted?: boolean;+ isSourceAudioEnabled?: boolean;hidden?: boolean;retime?: RetimeConfig;transform: Transform;diff --git a/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.tsnew file mode 100644index 00000000..0e0cfbbe--- /dev/null+++ b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts@@ -0,0 +1,141 @@+import { describe, expect, test } from "bun:test";+import { transformProjectV19ToV20 } from "../transformers/v19-to-v20";++describe("V19 to V20 Migration", () => {+ test("backfills source audio enabled state on video elements", () => {+ const result = transformProjectV19ToV20({+ project: {+ id: "project-v19-source-audio",+ version: 19,+ metadata: {+ id: "project-v19-source-audio",+ name: "Project",+ createdAt: "2024-01-01T00:00:00.000Z",+ updatedAt: "2024-01-01T00:00:00.000Z",+ },+ settings: {+ fps: 30,+ canvasSize: { width: 1920, height: 1080 },+ background: { type: "color", color: "#000000" },+ },+ currentSceneId: "scene-main",+ scenes: [+ {+ id: "scene-main",+ name: "Main",+ isMain: true,+ bookmarks: [],+ createdAt: "2024-01-01T00:00:00.000Z",+ updatedAt: "2024-01-01T00:00:00.000Z",+ tracks: [+ {+ id: "track-video",+ type: "video",+ name: "Video",+ isMain: true,+ muted: false,+ hidden: false,+ elements: [+ {+ id: "video-1",+ type: "video",+ name: "Clip",+ mediaId: "media-1",+ duration: 5,+ startTime: 0,+ trimStart: 0,+ trimEnd: 0,+ transform: {+ position: { x: 0, y: 0 },+ scale: { x: 1, y: 1 },+ rotation: 0,+ },+ opacity: 1,+ },+ ],+ },+ {+ id: "track-audio",+ type: "audio",+ name: "Audio",+ muted: false,+ elements: [+ {+ id: "audio-1",+ type: "audio",+ sourceType: "upload",+ mediaId: "media-audio-1",+ name: "Audio",+ duration: 5,+ startTime: 0,+ trimStart: 0,+ trimEnd: 0,+ volume: 0,+ },+ ],+ },+ ],+ },+ ],+ },+ });++ expect(result.skipped).toBe(false);+ expect(result.project.version).toBe(20);+ expect(+ ((result.project.scenes as Array<Record<string, unknown>>)[0]+ .tracks as Array<Record<string, unknown>>)[0].elements,+ ).toEqual([+ expect.objectContaining({+ id: "video-1",+ isSourceAudioEnabled: true,+ }),+ ]);+ expect(+ (((result.project.scenes as Array<Record<string, unknown>>)[0]+ .tracks as Array<Record<string, unknown>>)[1]+ .elements as Array<Record<string, unknown>>)[0].isSourceAudioEnabled,+ ).toBeUndefined();+ });++ test("preserves existing explicit source audio state", () => {+ const result = transformProjectV19ToV20({+ project: {+ id: "project-v19-existing-state",+ version: 19,+ scenes: [+ {+ tracks: [+ {+ elements: [+ {+ type: "video",+ isSourceAudioEnabled: false,+ },+ ],+ },+ ],+ },+ ],+ },+ });++ expect(+ (((result.project.scenes as Array<Record<string, unknown>>)[0]+ .tracks as Array<Record<string, unknown>>)[0]+ .elements as Array<Record<string, unknown>>)[0].isSourceAudioEnabled,+ ).toBe(false);+ });++ test("skips projects already on v20", () => {+ const result = transformProjectV19ToV20({+ project: {+ id: "project-v20",+ version: 20,+ },+ });++ expect(result.skipped).toBe(true);+ expect(result.reason).toBe("already v20");+ });+});diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.tsindex e87d1cb9..d278263b 100644--- a/apps/web/src/services/storage/migrations/index.ts+++ b/apps/web/src/services/storage/migrations/index.ts@@ -18,10 +18,11 @@ import { V15toV16Migration } from "./v15-to-v16";import { V16toV17Migration } from "./v16-to-v17";import { V17toV18Migration } from "./v17-to-v18";import { V18toV19Migration } from "./v18-to-v19";+import { V19toV20Migration } from "./v19-to-v20";export { runStorageMigrations } from "./runner";export type { MigrationProgress } from "./runner";-export const CURRENT_PROJECT_VERSION = 19;+export const CURRENT_PROJECT_VERSION = 20;export const migrations = [new V0toV1Migration(),@@ -43,4 +44,5 @@ export const migrations = [new V16toV17Migration(),new V17toV18Migration(),new V18toV19Migration(),+ new V19toV20Migration(),];diff --git a/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.tsnew file mode 100644index 00000000..f72812a4--- /dev/null+++ b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts@@ -0,0 +1,87 @@+import type { MigrationResult, ProjectRecord } from "./types";+import { getProjectId, isRecord } from "./utils";++export function transformProjectV19ToV20({+ project,+}: {+ project: ProjectRecord;+}): MigrationResult<ProjectRecord> {+ if (!getProjectId({ project })) {+ return { project, skipped: true, reason: "no project id" };+ }++ if (typeof project.version === "number" && project.version >= 20) {+ return { project, skipped: true, reason: "already v20" };+ }++ return {+ project: {+ ...migrateVideoSourceAudioState({ project }),+ version: 20,+ },+ skipped: false,+ };+}++function migrateVideoSourceAudioState({+ project,+}: {+ project: ProjectRecord;+}): ProjectRecord {+ if (!Array.isArray(project.scenes)) {+ return project;+ }++ return {+ ...project,+ scenes: project.scenes.map((scene) => migrateScene({ scene })),+ };+}++function migrateScene({+ scene,+}: {+ scene: unknown;+}): unknown {+ if (!isRecord(scene) || !Array.isArray(scene.tracks)) {+ return scene;+ }++ return {+ ...scene,+ tracks: scene.tracks.map((track) => migrateTrack({ track })),+ };+}++function migrateTrack({+ track,+}: {+ track: unknown;+}): unknown {+ if (!isRecord(track) || !Array.isArray(track.elements)) {+ return track;+ }++ return {+ ...track,+ elements: track.elements.map((element) => migrateElement({ element })),+ };+}++function migrateElement({+ element,+}: {+ element: unknown;+}): unknown {+ if (!isRecord(element) || element.type !== "video") {+ return element;+ }++ return {+ ...element,+ isSourceAudioEnabled:+ typeof element.isSourceAudioEnabled === "boolean"+ ? element.isSourceAudioEnabled+ : true,+ };+}diff --git a/apps/web/src/services/storage/migrations/v19-to-v20.ts b/apps/web/src/services/storage/migrations/v19-to-v20.tsnew file mode 100644index 00000000..c6809067--- /dev/null+++ b/apps/web/src/services/storage/migrations/v19-to-v20.ts@@ -0,0 +1,16 @@+import { StorageMigration } from "./base";+import type { ProjectRecord } from "./transformers/types";+import { transformProjectV19ToV20 } from "./transformers/v19-to-v20";++export class V19toV20Migration extends StorageMigration {+ from = 19;+ to = 20;++ async transform(project: ProjectRecord): Promise<{+ project: ProjectRecord;+ skipped: boolean;+ reason?: string;+ }> {+ return transformProjectV19ToV20({ project });+ }+}