🎥 Make videos programmatically with React
`@remotion/studio-server`: Support affine frame clocks in Visual Mode (#10632)
Jonny Burgerf49317a7b048
diff --git a/packages/example/e2e/affine-frame-clock.test.mts b/packages/example/e2e/affine-frame-clock.test.mtsnew file mode 100644index 0000000000..d7ff43277b--- /dev/null+++ b/packages/example/e2e/affine-frame-clock.test.mts@@ -0,0 +1,105 @@+import fs from 'fs';+import path from 'path';+import {expect, test} from '@playwright/test';+import {EXPANDED_SIDEBAR_STATE, STUDIO_URL, exampleDir} from './constants.mts';+import {startStudio, stopStudio} from './studio-server.mts';++test.use({storageState: EXPANDED_SIDEBAR_STATE});++const affineFrameClockFile = path.join(+ exampleDir,+ 'src',+ 'VisualModeTests',+ 'AffineFrameClock.tsx',+);++const readRotationKeyframes = () => {+ const source = fs.readFileSync(affineFrameClockFile, 'utf-8');+ const match = /rotate:\s*interpolate\(\s*captureFrame,\s*\[([^\]]*)\]/.exec(+ source,+ );+ if (match === null) {+ throw new Error('Could not read affine frame clock keyframes');+ }++ return match[1]+ .split(',')+ .map((frame) => Number(frame.trim()))+ .filter((frame) => !Number.isNaN(frame));+};++test.describe('affine frame clock keyframes', () => {+ let sourceBefore: string;++ test.beforeEach(async () => {+ sourceBefore = fs.readFileSync(affineFrameClockFile, 'utf-8');+ await startStudio();+ });++ test.afterEach(async () => {+ await stopStudio();+ fs.writeFileSync(affineFrameClockFile, sourceBefore);+ });++ test('edits a keyframe through a useCurrentFrame alias', async ({page}) => {+ await page.goto(`${STUDIO_URL}/affine-frame-clock`);+ await expect(page).toHaveURL(/affine-frame-clock/, {timeout: 15_000});+ await page.waitForFunction(+ () => !document.body.innerText.includes('Loading...'),+ {timeout: 30_000},+ );++ const affineFrameClock = page.locator(+ '[data-timeline-marquee-item][title="Affine frame clock"]',+ );+ const rotation = page.getByTitle('Rotation', {exact: true}).first();+ await expect(async () => {+ await affineFrameClock.click();+ await expect(rotation).toBeVisible({timeout: 1_000});+ }).toPass({timeout: 30_000});+ await rotation.click();+ const firstKeyframe = page+ .getByTitle('Keyframe at frame 0', {exact: true})+ .first();+ await expect(firstKeyframe).toBeVisible({timeout: 15_000});+ await firstKeyframe.click();++ const rotationSurface = page.locator(+ '[data-remotion-studio-canvas-rotation]',+ );+ await expect(rotationSurface).toBeVisible({timeout: 15_000});++ await page.keyboard.press('g');+ const currentFrameInput = page.locator('input:focus');+ await expect(currentFrameInput).toBeVisible();+ await currentFrameInput.fill('10');+ await currentFrameInput.press('Enter');++ const point = await rotationSurface.evaluate((surface) => {+ const box = surface.getBoundingClientRect();+ for (let yIndex = 1; yIndex < 10; yIndex++) {+ for (let xIndex = 1; xIndex < 10; xIndex++) {+ const candidate = {+ x: box.x + (box.width * xIndex) / 10,+ y: box.y + (box.height * yIndex) / 10,+ };+ if (+ document+ .elementFromPoint(candidate.x, candidate.y)+ ?.closest('[data-remotion-studio-canvas-rotation]') === surface+ ) {+ return candidate;+ }+ }+ }++ throw new Error('Rotation surface should have an interactive point');+ });+ await page.mouse.move(point.x, point.y);+ await page.mouse.down();+ await page.mouse.move(point.x + 40, point.y + 30, {steps: 5});+ await page.mouse.up();++ await expect.poll(readRotationKeyframes).toEqual([30, 40, 60]);+ });+});diff --git a/packages/example/src/E2eTestRoot.tsx b/packages/example/src/E2eTestRoot.tsxindex 13583705d7..800918a8d2 100644--- a/packages/example/src/E2eTestRoot.tsx+++ b/packages/example/src/E2eTestRoot.tsx@@ -18,6 +18,7 @@ import {SchemaTest, schemaTestSchema} from './SchemaTest';import {TimelineVirtualizationTestbed} from './TimelineVirtualizationTestbed';import {VisualControls} from './VisualControls';import {VisualMode3D} from './VisualMode3D';+import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';import {SequenceShiftRepro} from './VisualModeTests/SequenceShiftRepro';const UseCurrentScaleOnLoad: React.FC = () => {@@ -182,6 +183,14 @@ export const E2eTestRoot: React.FC = () => {fps={30}durationInFrames={90}/>+ <Composition+ id="affine-frame-clock"+ component={AffineFrameClock}+ width={1280}+ height={720}+ fps={30}+ durationInFrames={60}+ /><Compositionid="sequence-shift-repro"component={SequenceShiftRepro}diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex ec99465287..de3a507c53 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -303,6 +303,7 @@ import {VideoTestingPlayback} from './VideoTesting/playback';import {VideoTestingTrim} from './VideoTesting/trim';import {RemotionMediaVideoTexture} from './VideoTexture';import {VisualControls} from './VisualControls';+import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';import {FastUpdates} from './VisualModeTests/FastUpdates';import {FastUpdatesNested} from './VisualModeTests/FastUpdatesNested';import {@@ -2875,6 +2876,14 @@ export const Index: React.FC = () => {durationInFrames={180}/><Folder name="VisualModeTests">+ <Composition+ id="affine-frame-clock"+ component={AffineFrameClock}+ width={1280}+ height={720}+ fps={30}+ durationInFrames={60}+ /><Compositionid="sequence-shift-repro"component={SequenceShiftRepro}diff --git a/packages/example/src/VisualModeTests/AffineFrameClock.tsx b/packages/example/src/VisualModeTests/AffineFrameClock.tsxnew file mode 100644index 0000000000..7f2eeba4e9--- /dev/null+++ b/packages/example/src/VisualModeTests/AffineFrameClock.tsx@@ -0,0 +1,23 @@+import React from 'react';+import {AbsoluteFill, interpolate, Sequence, useCurrentFrame} from 'remotion';++export const AffineFrameClock: React.FC = () => {+ const frame = useCurrentFrame();+ const captureFrame = frame + 30;++ return (+ <AbsoluteFill style={{backgroundColor: '#111'}} from={-30}>+ <Sequence+ name="Affine frame clock"+ durationInFrames={60}+ style={{+ height: 400,+ rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg']),+ width: 400,+ }}+ >+ <AbsoluteFill style={{backgroundColor: '#0b84ff'}} />+ </Sequence>+ </AbsoluteFill>+ );+};diff --git a/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts b/packages/studio-server/src/preview-server/routes/can-update-sequence-props.tsindex 8d6432148a..5858ec7d3e 100644--- a/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts+++ b/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts@@ -904,26 +904,69 @@ const getDefaultFrameDisplayOffsetAdjustment = ({return result?.adjustment ?? null;};-const getCurrentFrameDisplayOffsetAdjustment = ({+type ResolvedCurrentFrameExpression = {+ readonly bindingScopePath: recast.types.NodePath;+ readonly offset: number;+};++const resolveCurrentFrameExpression = ({node,ast,- videoConfigValues,+ seenDeclarations,}: {node: Expression;ast: File;- videoConfigValues: VideoConfigIdentifierValues;-}): {- readonly adjustment: number;- readonly hasEnclosingElement: boolean;-} | null => {+ seenDeclarations: Set<object>;+}): ResolvedCurrentFrameExpression | null => {if (node.type === 'TSAsExpression') {- return getCurrentFrameDisplayOffsetAdjustment({+ return resolveCurrentFrameExpression({node: node.expression as Expression,ast,- videoConfigValues,+ seenDeclarations,});}+ if (+ node.type === 'BinaryExpression' &&+ (node.operator === '+' || node.operator === '-')+ ) {+ const rightValue = getNumericValue(node.right as Expression);+ if (rightValue !== null && Number.isFinite(rightValue)) {+ const resolvedLeft = resolveCurrentFrameExpression({+ node: node.left as Expression,+ ast,+ seenDeclarations,+ });+ if (resolvedLeft !== null) {+ return {+ ...resolvedLeft,+ offset:+ resolvedLeft.offset ++ (node.operator === '+' ? rightValue : -rightValue),+ };+ }+ }++ if (node.operator === '+') {+ const leftValue = getNumericValue(node.left as Expression);+ if (leftValue !== null && Number.isFinite(leftValue)) {+ const resolvedRight = resolveCurrentFrameExpression({+ node: node.right as Expression,+ ast,+ seenDeclarations,+ });+ if (resolvedRight !== null) {+ return {+ ...resolvedRight,+ offset: resolvedRight.offset + leftValue,+ };+ }+ }+ }++ return null;+ }+if (node.type !== 'Identifier') {return null;}@@ -935,7 +978,9 @@ const getCurrentFrameDisplayOffsetAdjustment = ({}let matchingDeclarations = 0;- let isCurrentFrameBinding = false;+ let matchingDeclaration: object | null = null;+ let initializer: Expression | null = null;+ let isConstDeclaration = false;recast.types.visit(bindingScope.path, {visitVariableDeclarator(p) {const {id, init} = p.node;@@ -948,23 +993,90 @@ const getCurrentFrameDisplayOffsetAdjustment = ({}matchingDeclarations++;- isCurrentFrameBinding = Boolean(- init && isUseCurrentFrameCall(init as Expression, ast),- );+ const declaration = p.parentPath.node;+ if (init) {+ matchingDeclaration = p.node;+ initializer = init as Expression;+ isConstDeclaration =+ declaration.type === 'VariableDeclaration' &&+ declaration.kind === 'const';+ }return false;},});- if (matchingDeclarations !== 1 || !isCurrentFrameBinding) {+ if (+ matchingDeclarations !== 1 ||+ matchingDeclaration === null ||+ initializer === null ||+ seenDeclarations.has(matchingDeclaration)+ ) {+ return null;+ }++ if (isUseCurrentFrameCall(initializer, ast)) {+ return {bindingScopePath: bindingScope.path, offset: 0};+ }++ if (!isConstDeclaration) {+ return null;+ }++ const nextSeenDeclarations = new Set(seenDeclarations);+ nextSeenDeclarations.add(matchingDeclaration);+ return resolveCurrentFrameExpression({+ node: initializer,+ ast,+ seenDeclarations: nextSeenDeclarations,+ });+};++const getCurrentFrameDisplayOffsetAdjustment = ({+ node,+ ast,+ videoConfigValues,+}: {+ node: Expression;+ ast: File;+ videoConfigValues: VideoConfigIdentifierValues;+}): {+ readonly adjustment: number;+ readonly hasEnclosingElement: boolean;+} | null => {+ if (node.type === 'TSAsExpression') {+ return getCurrentFrameDisplayOffsetAdjustment({+ node: node.expression as Expression,+ ast,+ videoConfigValues,+ });+ }++ const nodePath = findNodePath(ast, node);+ const resolved = resolveCurrentFrameExpression({+ node,+ ast,+ seenDeclarations: new Set(),+ });+ if (!nodePath || resolved === null) {return null;}- return getFrameDisplayOffsetAdjustmentBetweenPaths({+ const frameDisplayOffset = getFrameDisplayOffsetAdjustmentBetweenPaths({startPath: nodePath,- endPath: bindingScope.path,+ endPath: resolved.bindingScopePath,videoConfigValues,});+ if (frameDisplayOffset === null) {+ return null;+ }++ return {+ ...frameDisplayOffset,+ // interpolate(frame + offset, [keyframe], ...) reaches the keyframe at+ // composition frame `keyframe - offset`.+ adjustment: frameDisplayOffset.adjustment - resolved.offset,+ };};export const getComputedStatus = (diff --git a/packages/studio-server/src/test/compute-sequence-props-status.test.ts b/packages/studio-server/src/test/compute-sequence-props-status.test.tsindex 325d5d5f0a..d1995559d0 100644--- a/packages/studio-server/src/test/compute-sequence-props-status.test.ts+++ b/packages/studio-server/src/test/compute-sequence-props-status.test.ts@@ -1176,6 +1176,60 @@ export const Example: React.FC = () => {});});+test('computeSequencePropsStatus resolves affine aliases of useCurrentFrame', () => {+ const input = `import React from 'react';+import {AbsoluteFill, Sequence, interpolate, useCurrentFrame} from 'remotion';++export const Example: React.FC = () => {+ const frame = useCurrentFrame();+ const captureFrame = frame + 30;+ let mutableCaptureFrame = frame + 30;+ return (+ <AbsoluteFill from={-30}>+ <Sequence+ style={{+ rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg']),+ opacity: interpolate(captureFrame - 10, [20, 50], [0, 1]),+ scale: interpolate(frame * 2, [0, 60], [1, 2]),+ translate: interpolate(mutableCaptureFrame, [30, 60], ['0px', '10px']),+ }}+ />+ </AbsoluteFill>+ );+};+`;+ const result = computeSequencePropsStatusFromContent({+ videoConfigValues: null,+ fileContents: input,+ nodePath: getNodePathFromContent(input, 10),+ componentIdentity: null,+ keys: ['style.rotate', 'style.opacity', 'style.scale', 'style.translate'],+ effects: [],+ });++ expect(result.canUpdate).toBe(true);+ if (!result.canUpdate) throw new Error('Expected canUpdate to be true');++ expect(result.props['style.rotate']).toMatchObject({+ status: 'keyframed',+ keyframeDisplayOffsetAdjustment: 0,+ keyframes: [+ {frame: 30, value: '0deg'},+ {frame: 60, value: '30deg'},+ ],+ });+ expect(result.props['style.opacity']).toMatchObject({+ status: 'keyframed',+ keyframeDisplayOffsetAdjustment: 10,+ keyframes: [+ {frame: 20, value: 0},+ {frame: 50, value: 1},+ ],+ });+ expect(result.props['style.scale']).toEqual({status: 'computed'});+ expect(result.props['style.translate']).toEqual({status: 'computed'});+});+test('computeSequencePropsStatus reports computed when a timing component offset is uncertain', () => {const input = `import React from 'react';import {AbsoluteFill, interpolate, useCurrentFrame} from 'remotion';diff --git a/packages/studio-server/src/test/update-keyframes.test.ts b/packages/studio-server/src/test/update-keyframes.test.tsindex 9594533458..23245ee379 100644--- a/packages/studio-server/src/test/update-keyframes.test.ts+++ b/packages/studio-server/src/test/update-keyframes.test.ts@@ -29,6 +29,54 @@ const videoConfigValues = {width: 1920,};+test('updateSequenceKeyframes preserves an affine frame alias', async () => {+ const input = `import React from 'react';+import {AbsoluteFill, Sequence, interpolate, useCurrentFrame} from 'remotion';++export const Example: React.FC = () => {+ const frame = useCurrentFrame();+ const captureFrame = frame + 30;+ return (+ <AbsoluteFill from={-30}>+ <Sequence style={{rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg'])}} />+ </AbsoluteFill>+ );+};+`;+ const {output, updatedNodePath} = await updateSequenceKeyframes({+ input,+ nodePath: lineColumnToNodePath(input, 9),+ updates: [+ {+ key: 'style.rotate',+ operation: {type: 'add', frame: 45, value: '15deg'},+ },+ ],+ videoConfigValues,+ });++ expect(output).toMatch(+ /interpolate\(\s*captureFrame,\s*\[30, 45, 60\],\s*\['0deg', '15deg', '30deg'\]/,+ );+ const status = computeSequencePropsStatusFromContent({+ fileContents: output,+ nodePath: updatedNodePath,+ componentIdentity: null,+ keys: ['style.rotate'],+ effects: [],+ videoConfigValues,+ });+ expect(status.props['style.rotate']).toMatchObject({+ status: 'keyframed',+ keyframeDisplayOffsetAdjustment: 0,+ keyframes: [+ {frame: 30, value: '0deg'},+ {frame: 45, value: '15deg'},+ {frame: 60, value: '30deg'},+ ],+ });+});+test('updateSequenceKeyframes preserves representable video config frame expressions', async () => {const input = `import React from 'react';import {Sequence, interpolate, useCurrentFrame, useVideoConfig} from 'remotion';