🎥 Make videos programmatically with React
@remotion/canvas: Add live timeline controller
JonnyBurgera6fb71f5c82a
diff --git a/bun.lock b/bun.lockindex eb6fdd40a5..c57a6274a9 100644--- a/bun.lock+++ b/bun.lock@@ -195,12 +195,21 @@"name": "@remotion/canvas","version": "4.0.516","dependencies": {+ "@remotion/player": "workspace:*","remotion": "workspace:*",},"devDependencies": {+ "@happy-dom/global-registrator": "14.5.1","@remotion/eslint-config-internal": "workspace:*",+ "@testing-library/react": "16.1.0","@typescript/native-preview": "catalog:","eslint": "catalog:",+ "react": "catalog:",+ "react-dom": "catalog:",+ },+ "peerDependencies": {+ "react": ">=16.8.0",+ "react-dom": ">=16.8.0",},},"packages/canvas-capture-extension": {@@ -10461,6 +10470,10 @@"@remotion/bundler/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],+ "@remotion/canvas/react": ["[email protected]", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="],++ "@remotion/canvas/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],+"@remotion/canvas-capture-extension/react": ["[email protected]", "", {}, "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA=="],"@remotion/canvas-capture-extension/react-dom": ["[email protected]", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.3" } }, "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg=="],diff --git a/packages/canvas/README.md b/packages/canvas/README.mdindex 6f43f4d533..5b42d519d8 100644--- a/packages/canvas/README.md+++ b/packages/canvas/README.md@@ -2,5 +2,27 @@Headless primitives for Remotion authoring interfaces.+`<Canvas>` renders an `@remotion/player` and publishes its live timeline through+a controller:++```tsx+import {Canvas, createCanvasController} from '@remotion/canvas';++const controller = createCanvasController();++controller.timeline.subscribe(() => {+ console.log(controller.timeline.getSnapshot());+});++<Canvas+ controller={controller}+ component={MyComposition}+ durationInFrames={300}+ compositionWidth={1920}+ compositionHeight={1080}+ fps={30}+/>;+```+This package is an internal dependency of Remotion Studio. Its API is not yetready for general use.diff --git a/packages/canvas/bundle.ts b/packages/canvas/bundle.tsindex 4b515570bb..aecd71a704 100644--- a/packages/canvas/bundle.ts+++ b/packages/canvas/bundle.ts@@ -5,7 +5,7 @@ if (process.env.NODE_ENV !== 'production') {throw new Error('This script must be run using NODE_ENV=production');}-const external = ['remotion'];+const external = ['@remotion/player', 'react', 'react-dom', 'remotion'];console.time('Generated.');const esmOutput = await build({diff --git a/packages/canvas/bunfig.toml b/packages/canvas/bunfig.tomlnew file mode 100644index 0000000000..d20eff30d4--- /dev/null+++ b/packages/canvas/bunfig.toml@@ -0,0 +1,2 @@+[test]+preload = "./happydom.ts"diff --git a/packages/canvas/eslint.config.mjs b/packages/canvas/eslint.config.mjsindex 04fd4fac11..ab29acaba0 100644--- a/packages/canvas/eslint.config.mjs+++ b/packages/canvas/eslint.config.mjs@@ -1,6 +1,6 @@import {remotionFlatConfig} from '@remotion/eslint-config-internal';-const config = remotionFlatConfig({react: false});+const config = remotionFlatConfig({react: true});export default {...config,diff --git a/packages/canvas/happydom.ts b/packages/canvas/happydom.tsnew file mode 100644index 0000000000..7d4076c191--- /dev/null+++ b/packages/canvas/happydom.ts@@ -0,0 +1,3 @@+import {GlobalRegistrator} from '@happy-dom/global-registrator';++GlobalRegistrator.register();diff --git a/packages/canvas/package.json b/packages/canvas/package.jsonindex 50d1485c6e..917d751210 100644--- a/packages/canvas/package.json+++ b/packages/canvas/package.json@@ -24,8 +24,13 @@"url": "https://github.com/remotion-dev/remotion/issues"},"dependencies": {+ "@remotion/player": "workspace:*","remotion": "workspace:*"},+ "peerDependencies": {+ "react": ">=16.8.0",+ "react-dom": ">=16.8.0"+ },"exports": {"./package.json": "./package.json",".": {@@ -36,9 +41,13 @@}},"devDependencies": {+ "@happy-dom/global-registrator": "14.5.1","@remotion/eslint-config-internal": "workspace:*",+ "@testing-library/react": "16.1.0","@typescript/native-preview": "catalog:",- "eslint": "catalog:"+ "eslint": "catalog:",+ "react": "catalog:",+ "react-dom": "catalog:"},"keywords": ["remotion",diff --git a/packages/canvas/src/canvas-controller.ts b/packages/canvas/src/canvas-controller.tsnew file mode 100644index 0000000000..daba37e089--- /dev/null+++ b/packages/canvas/src/canvas-controller.ts@@ -0,0 +1,73 @@+import {useState} from 'react';+import type {TSequence} from 'remotion';+import {calculateTimeline} from './calculate-timeline';+import type {TimelineTrackData} from './get-timeline-sequence-sort-key';++export type CanvasController = {+ readonly timeline: {+ readonly getSnapshot: () => readonly TimelineTrackData[];+ readonly subscribe: (listener: () => void) => () => void;+ };+};++type CanvasControllerInternals = {+ readonly setSequences: (sequences: TSequence[]) => void;+ readonly clear: () => void;+};++const controllerInternals = new WeakMap<+ CanvasController,+ CanvasControllerInternals+>();++export const createCanvasController = (): CanvasController => {+ let snapshot: readonly TimelineTrackData[] = [];+ const listeners = new Set<() => void>();++ const updateSnapshot = (nextSnapshot: readonly TimelineTrackData[]) => {+ snapshot = nextSnapshot;+ for (const listener of listeners) {+ listener();+ }+ };++ const controller: CanvasController = {+ timeline: {+ getSnapshot: () => snapshot,+ subscribe: (listener) => {+ listeners.add(listener);+ return () => listeners.delete(listener);+ },+ },+ };++ controllerInternals.set(controller, {+ setSequences: (sequences) => {+ updateSnapshot(+ calculateTimeline({+ sequences,+ overrideIdsToNodePaths: {},+ }),+ );+ },+ clear: () => updateSnapshot([]),+ });++ return controller;+};++export const useCanvasController = (): CanvasController => {+ const [controller] = useState(createCanvasController);+ return controller;+};++export const getCanvasControllerInternals = (+ controller: CanvasController,+): CanvasControllerInternals => {+ const internals = controllerInternals.get(controller);+ if (!internals) {+ throw new Error('Invalid Canvas controller');+ }++ return internals;+};diff --git a/packages/canvas/src/canvas.tsx b/packages/canvas/src/canvas.tsxnew file mode 100644index 0000000000..3a40889663--- /dev/null+++ b/packages/canvas/src/canvas.tsx@@ -0,0 +1,50 @@+import type {PlayerProps, PlayerRef} from '@remotion/player';+import {Player, PlayerInternals} from '@remotion/player';+import type {RefObject} from 'react';+import React, {forwardRef, useCallback, useEffect} from 'react';+import type {AnyZodObject, TSequence} from 'remotion';+import type {CanvasController} from './canvas-controller';+import {getCanvasControllerInternals} from './canvas-controller';++export type CanvasProps<+ Schema extends AnyZodObject,+ Props extends Record<string, unknown>,+> = PlayerProps<Schema, Props> & {+ readonly controller: CanvasController;+};++const CanvasFn = <+ Schema extends AnyZodObject,+ Props extends Record<string, unknown>,+>(+ {controller, ...playerProps}: CanvasProps<Schema, Props>,+ ref: RefObject<PlayerRef>,+) => {+ const internals = getCanvasControllerInternals(controller);+ const onTimelineSequenceChange = useCallback(+ (sequences: TSequence[]) => internals.setSequences(sequences),+ [internals],+ );++ useEffect(() => {+ return () => internals.clear();+ }, [internals]);++ return React.createElement(+ PlayerInternals.TimelineSequenceObserverContext.Provider,+ {value: onTimelineSequenceChange},+ React.createElement(+ Player as React.ComponentType,+ {+ ...(playerProps as Record<string, unknown>),+ ref,+ } as Record<string, unknown>,+ ),+ );+};++const forward = forwardRef as <T, P = {}>(+ render: (props: P, ref: React.RefObject<T>) => React.ReactElement | null,+) => (props: P & React.RefAttributes<T>) => React.ReactElement | null;++export const Canvas = forward(CanvasFn);diff --git a/packages/canvas/src/index.ts b/packages/canvas/src/index.tsindex 0bf302a9bd..1c4ba36102 100644--- a/packages/canvas/src/index.ts+++ b/packages/canvas/src/index.ts@@ -1,3 +1,7 @@+export {Canvas} from './canvas';+export type {CanvasProps} from './canvas';+export {createCanvasController, useCanvasController} from './canvas-controller';+export type {CanvasController} from './canvas-controller';export {calculateTimeline} from './calculate-timeline';export type {SequenceNodePathInfo,diff --git a/packages/canvas/src/test/canvas.test.tsx b/packages/canvas/src/test/canvas.test.tsxnew file mode 100644index 0000000000..cea67423a5--- /dev/null+++ b/packages/canvas/src/test/canvas.test.tsx@@ -0,0 +1,85 @@+import {afterEach, expect, test} from 'bun:test';+import {cleanup, render, waitFor} from '@testing-library/react';+import React from 'react';+import {Sequence} from 'remotion';+import {Canvas, createCanvasController} from '../index';++afterEach(() => {+ cleanup();+});++const Composition: React.FC<{+ readonly showNestedSequence: boolean;+}> = ({showNestedSequence}) => {+ return (+ <Sequence name="Parent" from={0} durationInFrames={80} layout="none">+ {showNestedSequence ? (+ <Sequence name="Child" from={5} durationInFrames={40} layout="none">+ <div />+ </Sequence>+ ) : null}+ </Sequence>+ );+};++test('publishes the live timeline from the Player composition', async () => {+ const controller = createCanvasController();+ let updates = 0;+ const unsubscribe = controller.timeline.subscribe(() => {+ updates++;+ });++ const view = render(+ <Canvas+ controller={controller}+ component={Composition}+ inputProps={{showNestedSequence: false}}+ durationInFrames={100}+ compositionWidth={1920}+ compositionHeight={1080}+ fps={30}+ initialFrame={10}+ numberOfSharedAudioTags={0}+ initiallyMuted+ acknowledgeRemotionLicense+ />,+ );++ await waitFor(() => {+ expect(+ controller.timeline+ .getSnapshot()+ .map((track) => track.sequence.displayName),+ ).toEqual(['Parent']);+ });++ view.rerender(+ <Canvas+ controller={controller}+ component={Composition}+ inputProps={{showNestedSequence: true}}+ durationInFrames={100}+ compositionWidth={1920}+ compositionHeight={1080}+ fps={30}+ initialFrame={10}+ numberOfSharedAudioTags={0}+ initiallyMuted+ acknowledgeRemotionLicense+ />,+ );++ await waitFor(() => {+ const tracks = controller.timeline.getSnapshot();+ expect(tracks.map((track) => track.sequence.displayName)).toEqual([+ 'Parent',+ 'Child',+ ]);+ expect(tracks.map((track) => track.depth)).toEqual([0, 1]);+ });++ expect(updates).toBeGreaterThan(0);+ unsubscribe();+ view.unmount();+ expect(controller.timeline.getSnapshot()).toEqual([]);+});diff --git a/packages/canvas/tsconfig.json b/packages/canvas/tsconfig.jsonindex 143a0d2e89..7034960cb3 100644--- a/packages/canvas/tsconfig.json+++ b/packages/canvas/tsconfig.json@@ -11,5 +11,5 @@"emitDeclarationOnly": true},"include": ["src"],- "references": [{"path": "../core"}]+ "references": [{"path": "../core"}, {"path": "../player"}]}diff --git a/packages/core/src/Sequence.tsx b/packages/core/src/Sequence.tsxindex 07da4163e1..24720ab5e6 100644--- a/packages/core/src/Sequence.tsx+++ b/packages/core/src/Sequence.tsx@@ -30,7 +30,10 @@ import {} from './sequence-crop.js';import type {SequenceContextType} from './SequenceContext.js';import {SequenceContext} from './SequenceContext.js';-import {SequenceManager} from './SequenceManager.js';+import {+ SequenceManager,+ SequenceRegistrationContext,+} from './SequenceManager.js';import {IsInsideSeriesContext} from './series/is-inside-series.js';import {useTimelinePosition} from './timeline-position-state.js';import type {BasicMediaInTimelineReturnType} from './use-media-in-timeline.js';@@ -293,6 +296,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<Math.min(videoConfig.durationInFrames - from, parentSequenceDuration),);const {registerSequence, unregisterSequence} = useContext(SequenceManager);+ const sequenceRegistrationEnabled = useContext(SequenceRegistrationContext);const wrapperRefForOutline = useRef<HTMLDivElement | null>(null);const refForOutline =other.layout === 'none'@@ -457,7 +461,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<]);useEffect(() => {- if (!env.isStudio) {+ if (!env.isStudio && !sequenceRegistrationEnabled) {return;}@@ -570,6 +574,7 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<premountDisplay,postmountDisplay,env.isStudio,+ sequenceRegistrationEnabled,registrationControls,_remotionInternalEffects,effectRuntimeValues,diff --git a/packages/core/src/SequenceManager.tsx b/packages/core/src/SequenceManager.tsxindex 7569ba03ad..c08373ece7 100644--- a/packages/core/src/SequenceManager.tsx+++ b/packages/core/src/SequenceManager.tsx@@ -37,6 +37,8 @@ export const SequenceManagerRefContext =current: [],});+export const SequenceRegistrationContext = React.createContext(false);+export type VisualModePropStatuses = {propStatuses: PropStatuses;};diff --git a/packages/core/src/internals.ts b/packages/core/src/internals.tsindex f1a8ad9bf5..37f86d6f90 100644--- a/packages/core/src/internals.ts+++ b/packages/core/src/internals.ts@@ -174,7 +174,9 @@ import type {CannotUpdateSequenceReason} from './SequenceManager.js';import {makeSequencePropsSubscriptionKey,SequenceManager,+ SequenceManagerProvider,SequenceManagerRefContext,+ SequenceRegistrationContext,VisualModeDragOverridesContext,VisualModePropStatusesContext,VisualModePropStatusesRefContext,@@ -325,7 +327,9 @@ export const Internals = {VisualModeDragOverridesContext,VisualModeSettersContext,SequenceManager,+ SequenceManagerProvider,SequenceManagerRefContext,+ SequenceRegistrationContext,SequenceStackTracesUpdateContext,baseSchema,sequenceSchema,diff --git a/packages/core/src/use-media-in-timeline.ts b/packages/core/src/use-media-in-timeline.tsindex 3976e1d13d..77caf400c1 100644--- a/packages/core/src/use-media-in-timeline.ts+++ b/packages/core/src/use-media-in-timeline.ts@@ -5,7 +5,10 @@ import {getAssetDisplayName} from './get-asset-file-name.js';import {getTimelineDuration} from './get-timeline-duration.js';import {useNonce} from './nonce.js';import {SequenceContext} from './SequenceContext.js';-import {SequenceManager} from './SequenceManager.js';+import {+ SequenceManager,+ SequenceRegistrationContext,+} from './SequenceManager.js';import {useRemotionEnvironment} from './use-remotion-environment.js';import {useVideoConfig} from './use-video-config.js';import type {VolumeProp} from './volume-prop.js';@@ -171,6 +174,7 @@ export const useMediaInTimeline = ({const parentSequence = useContext(SequenceContext);const startsAt = useMediaStartsAt();const {registerSequence, unregisterSequence} = useContext(SequenceManager);+ const sequenceRegistrationEnabled = useContext(SequenceRegistrationContext);const {durationInFrames} = useVideoConfig();const mediaStartsAt = useMediaStartsAt();@@ -197,7 +201,11 @@ export const useMediaInTimeline = ({throw new Error('No src passed');}- if (!isStudio && window.process?.env?.NODE_ENV !== 'test') {+ if (+ !isStudio &&+ !sequenceRegistrationEnabled &&+ window.process?.env?.NODE_ENV !== 'test'+ ) {return;}@@ -260,6 +268,7 @@ export const useMediaInTimeline = ({documentationLink,finalDisplayName,isStudio,+ sequenceRegistrationEnabled,refForOutline,muted,]);diff --git a/packages/player/src/Player.tsx b/packages/player/src/Player.tsxindex b5d13761ff..606885ba3a 100644--- a/packages/player/src/Player.tsx+++ b/packages/player/src/Player.tsx@@ -31,6 +31,8 @@ import type {PosterFillMode, RenderLoading, RenderPoster} from './PlayerUI.js';import PlayerUI from './PlayerUI.js';import type {RenderVolumeSlider} from './render-volume-slider.js';import {PLAYER_COMP_ID, SharedPlayerContexts} from './SharedPlayerContext.js';+import type {TimelineSequenceObserver} from './timeline-sequence-observer-context.js';+import {TimelineSequenceObserverContext} from './timeline-sequence-observer-context.js';import {acknowledgeRemotionLicenseMessage} from './use-remotion-license-acknowledge.js';import type {PropsIfHasProps} from './utils/props-if-has-props.js';import {validateInOutFrames} from './utils/validate-in-out-frame.js';@@ -117,6 +119,18 @@ export const componentOrNullIfLazy = <Props,>(return null;};+const TimelineSequenceObserverComponent: React.FC<{+ readonly onTimelineSequenceChange: TimelineSequenceObserver;+}> = ({onTimelineSequenceChange}) => {+ const {sequences} = React.useContext(Internals.SequenceManager);++ useEffect(() => {+ onTimelineSequenceChange(sequences);+ }, [onTimelineSequenceChange, sequences]);++ return null;+};+const PlayerFn = <Schema extends AnyZodObject,Props extends Record<string, unknown>,@@ -182,6 +196,10 @@ const PlayerFn = <window.remotion_isPlayer = true;}+ const onTimelineSequenceChange = React.useContext(+ TimelineSequenceObserverContext,+ );+// @ts-expect-errorif (componentProps.defaultProps !== undefined) {throw new Error(@@ -436,7 +454,7 @@ const PlayerFn = <);}, [passedBrowserMediaControlsBehavior]);- return (+ const player = (<Internals.IsPlayerContextProvider><SharedPlayerContextstimelineContext={timelineContextValue}@@ -515,6 +533,21 @@ const PlayerFn = <</SharedPlayerContexts></Internals.IsPlayerContextProvider>);++ if (!onTimelineSequenceChange) {+ return player;+ }++ return (+ <Internals.SequenceRegistrationContext.Provider value>+ <Internals.SequenceManagerProvider>+ <TimelineSequenceObserverComponent+ onTimelineSequenceChange={onTimelineSequenceChange}+ />+ {player}+ </Internals.SequenceManagerProvider>+ </Internals.SequenceRegistrationContext.Provider>+ );};const forward = forwardRef as <T, P = {}>(diff --git a/packages/player/src/index.ts b/packages/player/src/index.tsindex eef9f64194..4a380d0e10 100644--- a/packages/player/src/index.ts+++ b/packages/player/src/index.ts@@ -5,6 +5,7 @@ import {PlayerEventEmitterContext} from './emitter-context.js';import {PlayerEmitterProvider} from './EmitterProvider.js';import type {CallbackListener, PlayerEventTypes} from './event-emitter.js';import {PlayerEmitter} from './event-emitter.js';+import {TimelineSequenceObserverContext} from './timeline-sequence-observer-context.js';import {useHoverState} from './use-hover-state.js';import {usePlayback} from './use-playback.js';import {usePlayerMethods} from './use-player-methods.js';@@ -45,4 +46,5 @@ export const PlayerInternals = {updateAllElementsSizes,PlayerEmitterProvider,BufferingIndicator,+ TimelineSequenceObserverContext,};diff --git a/packages/player/src/timeline-sequence-observer-context.ts b/packages/player/src/timeline-sequence-observer-context.tsnew file mode 100644index 0000000000..ac22eb19e2--- /dev/null+++ b/packages/player/src/timeline-sequence-observer-context.ts@@ -0,0 +1,7 @@+import React from 'react';+import type {TSequence} from 'remotion';++export type TimelineSequenceObserver = (sequences: TSequence[]) => void;++export const TimelineSequenceObserverContext =+ React.createContext<TimelineSequenceObserver | null>(null);