Skip to content

🎥 Make videos programmatically with React

@remotion/canvas: Add live timeline controller

JonnyBurgera6fb71f5c82a
diff --git a/bun.lock b/bun.lock
index eb6fdd40a5..c57a6274a9 100644
--- a/bun.lock
+++ b/bun.lock
@@ -195,12 +195,21 @@
"name": "@remotion/canvas",
"version": "4.0.516",
"dependencies": {
+ "@remotion/player": "workspace:*",
"remotion": "workspace:*",
},
"devDependencies": {
+ "@happy-dom/global-registrator": "14.5.1",
"@remotion/eslint-config-internal": "workspace:*",
+ "@testing-library/react": "16.1.0",
"@typescript/native-preview": "catalog:",
"eslint": "catalog:",
+ "react": "catalog:",
+ "react-dom": "catalog:",
+ },
+ "peerDependencies": {
+ "react": ">=16.8.0",
+ "react-dom": ">=16.8.0",
},
},
"packages/canvas-capture-extension": {
@@ -10461,6 +10470,10 @@
"@remotion/bundler/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],
+ "@remotion/canvas/react": ["[email protected]", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="],
+
+ "@remotion/canvas/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],
+
"@remotion/canvas-capture-extension/react": ["[email protected]", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="],
"@remotion/canvas-capture-extension/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],
diff --git a/packages/canvas/README.md b/packages/canvas/README.md
index 6f43f4d533..5b42d519d8 100644
--- a/packages/canvas/README.md
+++ b/packages/canvas/README.md
@@ -2,5 +2,27 @@
Headless primitives for Remotion authoring interfaces.
+`<Canvas>` renders an `@remotion/player` and publishes its live timeline through
+a controller:
+
+```tsx
+import {Canvas, createCanvasController} from '@remotion/canvas';
+
+const controller = createCanvasController();
+
+controller.timeline.subscribe(() => {
+ console.log(controller.timeline.getSnapshot());
+});
+
+<Canvas
+ controller={controller}
+ component={MyComposition}
+ durationInFrames={300}
+ compositionWidth={1920}
+ compositionHeight={1080}
+ fps={30}
+/>;
+```
+
This package is an internal dependency of Remotion Studio. Its API is not yet
ready for general use.
diff --git a/packages/canvas/bundle.ts b/packages/canvas/bundle.ts
index 4b515570bb..aecd71a704 100644
--- a/packages/canvas/bundle.ts
+++ b/packages/canvas/bundle.ts
@@ -5,7 +5,7 @@ if (process.env.NODE_ENV !== 'production') {
throw new Error('This script must be run using NODE_ENV=production');
}
-const external = ['remotion'];
+const external = ['@remotion/player', 'react', 'react-dom', 'remotion'];
console.time('Generated.');
const esmOutput = await build({
diff --git a/packages/canvas/bunfig.toml b/packages/canvas/bunfig.toml
new file mode 100644
index 0000000000..d20eff30d4
--- /dev/null
+++ b/packages/canvas/bunfig.toml
@@ -0,0 +1,2 @@
+[test]
+preload = "./happydom.ts"
diff --git a/packages/canvas/eslint.config.mjs b/packages/canvas/eslint.config.mjs
index 04fd4fac11..ab29acaba0 100644
--- a/packages/canvas/eslint.config.mjs
+++ b/packages/canvas/eslint.config.mjs
@@ -1,6 +1,6 @@
import {remotionFlatConfig} from '@remotion/eslint-config-internal';
-const config = remotionFlatConfig({react: false});
+const config = remotionFlatConfig({react: true});
export default {
...config,
diff --git a/packages/canvas/happydom.ts b/packages/canvas/happydom.ts
new file mode 100644
index 0000000000..7d4076c191
--- /dev/null
+++ b/packages/canvas/happydom.ts
@@ -0,0 +1,3 @@
+import {GlobalRegistrator} from '@happy-dom/global-registrator';
+
+GlobalRegistrator.register();
diff --git a/packages/canvas/package.json b/packages/canvas/package.json
index 50d1485c6e..917d751210 100644
--- a/packages/canvas/package.json
+++ b/packages/canvas/package.json
@@ -24,8 +24,13 @@
"url": "https://github.com/remotion-dev/remotion/issues"
},
"dependencies": {
+ "@remotion/player": "workspace:*",
"remotion": "workspace:*"
},
+ "peerDependencies": {
+ "react": ">=16.8.0",
+ "react-dom": ">=16.8.0"
+ },
"exports": {
"./package.json": "./package.json",
".": {
@@ -36,9 +41,13 @@
}
},
"devDependencies": {
+ "@happy-dom/global-registrator": "14.5.1",
"@remotion/eslint-config-internal": "workspace:*",
+ "@testing-library/react": "16.1.0",
"@typescript/native-preview": "catalog:",
- "eslint": "catalog:"
+ "eslint": "catalog:",
+ "react": "catalog:",
+ "react-dom": "catalog:"
},
"keywords": [
"remotion",
diff --git a/packages/canvas/src/canvas-controller.ts b/packages/canvas/src/canvas-controller.ts
new file mode 100644
index 0000000000..daba37e089
--- /dev/null
+++ b/packages/canvas/src/canvas-controller.ts
@@ -0,0 +1,73 @@
+import {useState} from 'react';
+import type {TSequence} from 'remotion';
+import {calculateTimeline} from './calculate-timeline';
+import type {TimelineTrackData} from './get-timeline-sequence-sort-key';
+
+export type CanvasController = {
+ readonly timeline: {
+ readonly getSnapshot: () => readonly TimelineTrackData[];
+ readonly subscribe: (listener: () => void) => () => void;
+ };
+};
+
+type CanvasControllerInternals = {
+ readonly setSequences: (sequences: TSequence[]) => void;
+ readonly clear: () => void;
+};
+
+const controllerInternals = new WeakMap<
+ CanvasController,
+ CanvasControllerInternals
+>();
+
+export const createCanvasController = (): CanvasController => {
+ let snapshot: readonly TimelineTrackData[] = [];
+ const listeners = new Set<() => void>();
+
+ const updateSnapshot = (nextSnapshot: readonly TimelineTrackData[]) => {
+ snapshot = nextSnapshot;
+ for (const listener of listeners) {
+ listener();
+ }
+ };
+
+ const controller: CanvasController = {
+ timeline: {
+ getSnapshot: () => snapshot,
+ subscribe: (listener) => {
+ listeners.add(listener);
+ return () => listeners.delete(listener);
+ },
+ },
+ };
+
+ controllerInternals.set(controller, {
+ setSequences: (sequences) => {
+ updateSnapshot(
+ calculateTimeline({
+ sequences,
+ overrideIdsToNodePaths: {},
+ }),
+ );
+ },
+ clear: () => updateSnapshot([]),
+ });
+
+ return controller;
+};
+
+export const useCanvasController = (): CanvasController => {
+ const [controller] = useState(createCanvasController);
+ return controller;
+};
+
+export const getCanvasControllerInternals = (
+ controller: CanvasController,
+): CanvasControllerInternals => {
+ const internals = controllerInternals.get(controller);
+ if (!internals) {
+ throw new Error('Invalid Canvas controller');
+ }
+
+ return internals;
+};
diff --git a/packages/canvas/src/canvas.tsx b/packages/canvas/src/canvas.tsx
new file mode 100644
index 0000000000..3a40889663
--- /dev/null
+++ b/packages/canvas/src/canvas.tsx
@@ -0,0 +1,50 @@
+import type {PlayerProps, PlayerRef} from '@remotion/player';
+import {Player, PlayerInternals} from '@remotion/player';
+import type {RefObject} from 'react';
+import React, {forwardRef, useCallback, useEffect} from 'react';
+import type {AnyZodObject, TSequence} from 'remotion';
+import type {CanvasController} from './canvas-controller';
+import {getCanvasControllerInternals} from './canvas-controller';
+
+export type CanvasProps<
+ Schema extends AnyZodObject,
+ Props extends Record<string, unknown>,
+> = PlayerProps<Schema, Props> & {
+ readonly controller: CanvasController;
+};
+
+const CanvasFn = <
+ Schema extends AnyZodObject,
+ Props extends Record<string, unknown>,
+>(
+ {controller, ...playerProps}: CanvasProps<Schema, Props>,
+ ref: RefObject<PlayerRef>,
+) => {
+ const internals = getCanvasControllerInternals(controller);
+ const onTimelineSequenceChange = useCallback(
+ (sequences: TSequence[]) => internals.setSequences(sequences),
+ [internals],
+ );
+
+ useEffect(() => {
+ return () => internals.clear();
+ }, [internals]);
+
+ return React.createElement(
+ PlayerInternals.TimelineSequenceObserverContext.Provider,
+ {value: onTimelineSequenceChange},
+ React.createElement(
+ Player as React.ComponentType,
+ {
+ ...(playerProps as Record<string, unknown>),
+ ref,
+ } as Record<string, unknown>,
+ ),
+ );
+};
+
+const forward = forwardRef as <T, P = {}>(
+ render: (props: P, ref: React.RefObject<T>) => React.ReactElement | null,
+) => (props: P & React.RefAttributes<T>) => React.ReactElement | null;
+
+export const Canvas = forward(CanvasFn);
diff --git a/packages/canvas/src/index.ts b/packages/canvas/src/index.ts
index 0bf302a9bd..1c4ba36102 100644
--- a/packages/canvas/src/index.ts
+++ b/packages/canvas/src/index.ts
@@ -1,3 +1,7 @@
+export {Canvas} from './canvas';
+export type {CanvasProps} from './canvas';
+export {createCanvasController, useCanvasController} from './canvas-controller';
+export type {CanvasController} from './canvas-controller';
export {calculateTimeline} from './calculate-timeline';
export type {
SequenceNodePathInfo,
diff --git a/packages/canvas/src/test/canvas.test.tsx b/packages/canvas/src/test/canvas.test.tsx
new file mode 100644
index 0000000000..cea67423a5
--- /dev/null
+++ b/packages/canvas/src/test/canvas.test.tsx
@@ -0,0 +1,85 @@
+import {afterEach, expect, test} from 'bun:test';
+import {cleanup, render, waitFor} from '@testing-library/react';
+import React from 'react';
+import {Sequence} from 'remotion';
+import {Canvas, createCanvasController} from '../index';
+
+afterEach(() => {
+ cleanup();
+});
+
+const Composition: React.FC<{
+ readonly showNestedSequence: boolean;
+}> = ({showNestedSequence}) => {
+ return (
+ <Sequence name="Parent" from={0} durationInFrames={80} layout="none">
+ {showNestedSequence ? (
+ <Sequence name="Child" from={5} durationInFrames={40} layout="none">
+ <div />
+ </Sequence>
+ ) : null}
+ </Sequence>
+ );
+};
+
+test('publishes the live timeline from the Player composition', async () => {
+ const controller = createCanvasController();
+ let updates = 0;
+ const unsubscribe = controller.timeline.subscribe(() => {
+ updates++;
+ });
+
+ const view = render(
+ <Canvas
+ controller={controller}
+ component={Composition}
+ inputProps={{showNestedSequence: false}}
+ durationInFrames={100}
+ compositionWidth={1920}
+ compositionHeight={1080}
+ fps={30}
+ initialFrame={10}
+ numberOfSharedAudioTags={0}
+ initiallyMuted
+ acknowledgeRemotionLicense
+ />,
+ );
+
+ await waitFor(() => {
+ expect(
+ controller.timeline
+ .getSnapshot()
+ .map((track) => track.sequence.displayName),
+ ).toEqual(['Parent']);
+ });
+
+ view.rerender(
+ <Canvas
+ controller={controller}
+ component={Composition}
+ inputProps={{showNestedSequence: true}}
+ durationInFrames={100}
+ compositionWidth={1920}
+ compositionHeight={1080}
+ fps={30}
+ initialFrame={10}
+ numberOfSharedAudioTags={0}
+ initiallyMuted
+ acknowledgeRemotionLicense
+ />,
+ );
+
+ await waitFor(() => {
+ const tracks = controller.timeline.getSnapshot();
+ expect(tracks.map((track) => track.sequence.displayName)).toEqual([
+ 'Parent',
+ 'Child',
+ ]);
+ expect(tracks.map((track) => track.depth)).toEqual([0, 1]);
+ });
+
+ expect(updates).toBeGreaterThan(0);
+ unsubscribe();
+ view.unmount();
+ expect(controller.timeline.getSnapshot()).toEqual([]);
+});
diff --git a/packages/canvas/tsconfig.json b/packages/canvas/tsconfig.json
index 143a0d2e89..7034960cb3 100644
--- a/packages/canvas/tsconfig.json
+++ b/packages/canvas/tsconfig.json
@@ -11,5 +11,5 @@
"emitDeclarationOnly": true
},
"include": ["src"],
- "references": [{"path": "../core"}]
+ "references": [{"path": "../core"}, {"path": "../player"}]
}
diff --git a/packages/core/src/Sequence.tsx b/packages/core/src/Sequence.tsx
index 07da4163e1..24720ab5e6 100644
--- a/packages/core/src/Sequence.tsx
+++ b/packages/core/src/Sequence.tsx
@@ -30,7 +30,10 @@ import {
} from './sequence-crop.js';
import type {SequenceContextType} from './SequenceContext.js';
import {SequenceContext} from './SequenceContext.js';
-import {SequenceManager} from './SequenceManager.js';
+import {
+ SequenceManager,
+ SequenceRegistrationContext,
+} from './SequenceManager.js';
import {IsInsideSeriesContext} from './series/is-inside-series.js';
import {useTimelinePosition} from './timeline-position-state.js';
import type {BasicMediaInTimelineReturnType} from './use-media-in-timeline.js';
@@ -293,6 +296,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<
Math.min(videoConfig.durationInFrames - from, parentSequenceDuration),
);
const {registerSequence, unregisterSequence} = useContext(SequenceManager);
+ const sequenceRegistrationEnabled = useContext(SequenceRegistrationContext);
const wrapperRefForOutline = useRef<HTMLDivElement | null>(null);
const refForOutline =
other.layout === 'none'
@@ -457,7 +461,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<
]);
useEffect(() => {
- if (!env.isStudio) {
+ if (!env.isStudio && !sequenceRegistrationEnabled) {
return;
}
@@ -570,6 +574,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<
premountDisplay,
postmountDisplay,
env.isStudio,
+ sequenceRegistrationEnabled,
registrationControls,
_remotionInternalEffects,
effectRuntimeValues,
diff --git a/packages/core/src/SequenceManager.tsx b/packages/core/src/SequenceManager.tsx
index 7569ba03ad..c08373ece7 100644
--- a/packages/core/src/SequenceManager.tsx
+++ b/packages/core/src/SequenceManager.tsx
@@ -37,6 +37,8 @@ export const SequenceManagerRefContext =
current: [],
});
+export const SequenceRegistrationContext = React.createContext(false);
+
export type VisualModePropStatuses = {
propStatuses: PropStatuses;
};
diff --git a/packages/core/src/internals.ts b/packages/core/src/internals.ts
index f1a8ad9bf5..37f86d6f90 100644
--- a/packages/core/src/internals.ts
+++ b/packages/core/src/internals.ts
@@ -174,7 +174,9 @@ import type {CannotUpdateSequenceReason} from './SequenceManager.js';
import {
makeSequencePropsSubscriptionKey,
SequenceManager,
+ SequenceManagerProvider,
SequenceManagerRefContext,
+ SequenceRegistrationContext,
VisualModeDragOverridesContext,
VisualModePropStatusesContext,
VisualModePropStatusesRefContext,
@@ -325,7 +327,9 @@ export const Internals = {
VisualModeDragOverridesContext,
VisualModeSettersContext,
SequenceManager,
+ SequenceManagerProvider,
SequenceManagerRefContext,
+ SequenceRegistrationContext,
SequenceStackTracesUpdateContext,
baseSchema,
sequenceSchema,
diff --git a/packages/core/src/use-media-in-timeline.ts b/packages/core/src/use-media-in-timeline.ts
index 3976e1d13d..77caf400c1 100644
--- a/packages/core/src/use-media-in-timeline.ts
+++ b/packages/core/src/use-media-in-timeline.ts
@@ -5,7 +5,10 @@ import {getAssetDisplayName} from './get-asset-file-name.js';
import {getTimelineDuration} from './get-timeline-duration.js';
import {useNonce} from './nonce.js';
import {SequenceContext} from './SequenceContext.js';
-import {SequenceManager} from './SequenceManager.js';
+import {
+ SequenceManager,
+ SequenceRegistrationContext,
+} from './SequenceManager.js';
import {useRemotionEnvironment} from './use-remotion-environment.js';
import {useVideoConfig} from './use-video-config.js';
import type {VolumeProp} from './volume-prop.js';
@@ -171,6 +174,7 @@ export const useMediaInTimeline = ({
const parentSequence = useContext(SequenceContext);
const startsAt = useMediaStartsAt();
const {registerSequence, unregisterSequence} = useContext(SequenceManager);
+ const sequenceRegistrationEnabled = useContext(SequenceRegistrationContext);
const {durationInFrames} = useVideoConfig();
const mediaStartsAt = useMediaStartsAt();
@@ -197,7 +201,11 @@ export const useMediaInTimeline = ({
throw new Error('No src passed');
}
- if (!isStudio && window.process?.env?.NODE_ENV !== 'test') {
+ if (
+ !isStudio &&
+ !sequenceRegistrationEnabled &&
+ window.process?.env?.NODE_ENV !== 'test'
+ ) {
return;
}
@@ -260,6 +268,7 @@ export const useMediaInTimeline = ({
documentationLink,
finalDisplayName,
isStudio,
+ sequenceRegistrationEnabled,
refForOutline,
muted,
]);
diff --git a/packages/player/src/Player.tsx b/packages/player/src/Player.tsx
index b5d13761ff..606885ba3a 100644
--- a/packages/player/src/Player.tsx
+++ b/packages/player/src/Player.tsx
@@ -31,6 +31,8 @@ import type {PosterFillMode, RenderLoading, RenderPoster} from './PlayerUI.js';
import PlayerUI from './PlayerUI.js';
import type {RenderVolumeSlider} from './render-volume-slider.js';
import {PLAYER_COMP_ID, SharedPlayerContexts} from './SharedPlayerContext.js';
+import type {TimelineSequenceObserver} from './timeline-sequence-observer-context.js';
+import {TimelineSequenceObserverContext} from './timeline-sequence-observer-context.js';
import {acknowledgeRemotionLicenseMessage} from './use-remotion-license-acknowledge.js';
import type {PropsIfHasProps} from './utils/props-if-has-props.js';
import {validateInOutFrames} from './utils/validate-in-out-frame.js';
@@ -117,6 +119,18 @@ export const componentOrNullIfLazy = <Props,>(
return null;
};
+const TimelineSequenceObserverComponent: React.FC<{
+ readonly onTimelineSequenceChange: TimelineSequenceObserver;
+}> = ({onTimelineSequenceChange}) => {
+ const {sequences} = React.useContext(Internals.SequenceManager);
+
+ useEffect(() => {
+ onTimelineSequenceChange(sequences);
+ }, [onTimelineSequenceChange, sequences]);
+
+ return null;
+};
+
const PlayerFn = <
Schema extends AnyZodObject,
Props extends Record<string, unknown>,
@@ -182,6 +196,10 @@ const PlayerFn = <
window.remotion_isPlayer = true;
}
+ const onTimelineSequenceChange = React.useContext(
+ TimelineSequenceObserverContext,
+ );
+
// @ts-expect-error
if (componentProps.defaultProps !== undefined) {
throw new Error(
@@ -436,7 +454,7 @@ const PlayerFn = <
);
}, [passedBrowserMediaControlsBehavior]);
- return (
+ const player = (
<Internals.IsPlayerContextProvider>
<SharedPlayerContexts
timelineContext={timelineContextValue}
@@ -515,6 +533,21 @@ const PlayerFn = <
</SharedPlayerContexts>
</Internals.IsPlayerContextProvider>
);
+
+ if (!onTimelineSequenceChange) {
+ return player;
+ }
+
+ return (
+ <Internals.SequenceRegistrationContext.Provider value>
+ <Internals.SequenceManagerProvider>
+ <TimelineSequenceObserverComponent
+ onTimelineSequenceChange={onTimelineSequenceChange}
+ />
+ {player}
+ </Internals.SequenceManagerProvider>
+ </Internals.SequenceRegistrationContext.Provider>
+ );
};
const forward = forwardRef as <T, P = {}>(
diff --git a/packages/player/src/index.ts b/packages/player/src/index.ts
index eef9f64194..4a380d0e10 100644
--- a/packages/player/src/index.ts
+++ b/packages/player/src/index.ts
@@ -5,6 +5,7 @@ import {PlayerEventEmitterContext} from './emitter-context.js';
import {PlayerEmitterProvider} from './EmitterProvider.js';
import type {CallbackListener, PlayerEventTypes} from './event-emitter.js';
import {PlayerEmitter} from './event-emitter.js';
+import {TimelineSequenceObserverContext} from './timeline-sequence-observer-context.js';
import {useHoverState} from './use-hover-state.js';
import {usePlayback} from './use-playback.js';
import {usePlayerMethods} from './use-player-methods.js';
@@ -45,4 +46,5 @@ export const PlayerInternals = {
updateAllElementsSizes,
PlayerEmitterProvider,
BufferingIndicator,
+ TimelineSequenceObserverContext,
};
diff --git a/packages/player/src/timeline-sequence-observer-context.ts b/packages/player/src/timeline-sequence-observer-context.ts
new file mode 100644
index 0000000000..ac22eb19e2
--- /dev/null
+++ b/packages/player/src/timeline-sequence-observer-context.ts
@@ -0,0 +1,7 @@
+import React from 'react';
+import type {TSequence} from 'remotion';
+
+export type TimelineSequenceObserver = (sequences: TSequence[]) => void;
+
+export const TimelineSequenceObserverContext =
+ React.createContext<TimelineSequenceObserver | null>(null);