Skip to content

🎥 Make videos programmatically with React

`@remotion/studio-server`: Support affine frame clocks in Visual Mode (#10632)

Jonny Burgerf49317a7b048
diff --git a/packages/example/e2e/affine-frame-clock.test.mts b/packages/example/e2e/affine-frame-clock.test.mts
new file mode 100644
index 0000000000..d7ff43277b
--- /dev/null
+++ b/packages/example/e2e/affine-frame-clock.test.mts
@@ -0,0 +1,105 @@
+import fs from 'fs';
+import path from 'path';
+import {expect, test} from '@playwright/test';
+import {EXPANDED_SIDEBAR_STATE, STUDIO_URL, exampleDir} from './constants.mts';
+import {startStudio, stopStudio} from './studio-server.mts';
+
+test.use({storageState: EXPANDED_SIDEBAR_STATE});
+
+const affineFrameClockFile = path.join(
+ exampleDir,
+ 'src',
+ 'VisualModeTests',
+ 'AffineFrameClock.tsx',
+);
+
+const readRotationKeyframes = () => {
+ const source = fs.readFileSync(affineFrameClockFile, 'utf-8');
+ const match = /rotate:\s*interpolate\(\s*captureFrame,\s*\[([^\]]*)\]/.exec(
+ source,
+ );
+ if (match === null) {
+ throw new Error('Could not read affine frame clock keyframes');
+ }
+
+ return match[1]
+ .split(',')
+ .map((frame) => Number(frame.trim()))
+ .filter((frame) => !Number.isNaN(frame));
+};
+
+test.describe('affine frame clock keyframes', () => {
+ let sourceBefore: string;
+
+ test.beforeEach(async () => {
+ sourceBefore = fs.readFileSync(affineFrameClockFile, 'utf-8');
+ await startStudio();
+ });
+
+ test.afterEach(async () => {
+ await stopStudio();
+ fs.writeFileSync(affineFrameClockFile, sourceBefore);
+ });
+
+ test('edits a keyframe through a useCurrentFrame alias', async ({page}) => {
+ await page.goto(`${STUDIO_URL}/affine-frame-clock`);
+ await expect(page).toHaveURL(/affine-frame-clock/, {timeout: 15_000});
+ await page.waitForFunction(
+ () => !document.body.innerText.includes('Loading...'),
+ {timeout: 30_000},
+ );
+
+ const affineFrameClock = page.locator(
+ '[data-timeline-marquee-item][title="Affine frame clock"]',
+ );
+ const rotation = page.getByTitle('Rotation', {exact: true}).first();
+ await expect(async () => {
+ await affineFrameClock.click();
+ await expect(rotation).toBeVisible({timeout: 1_000});
+ }).toPass({timeout: 30_000});
+ await rotation.click();
+ const firstKeyframe = page
+ .getByTitle('Keyframe at frame 0', {exact: true})
+ .first();
+ await expect(firstKeyframe).toBeVisible({timeout: 15_000});
+ await firstKeyframe.click();
+
+ const rotationSurface = page.locator(
+ '[data-remotion-studio-canvas-rotation]',
+ );
+ await expect(rotationSurface).toBeVisible({timeout: 15_000});
+
+ await page.keyboard.press('g');
+ const currentFrameInput = page.locator('input:focus');
+ await expect(currentFrameInput).toBeVisible();
+ await currentFrameInput.fill('10');
+ await currentFrameInput.press('Enter');
+
+ const point = await rotationSurface.evaluate((surface) => {
+ const box = surface.getBoundingClientRect();
+ for (let yIndex = 1; yIndex < 10; yIndex++) {
+ for (let xIndex = 1; xIndex < 10; xIndex++) {
+ const candidate = {
+ x: box.x + (box.width * xIndex) / 10,
+ y: box.y + (box.height * yIndex) / 10,
+ };
+ if (
+ document
+ .elementFromPoint(candidate.x, candidate.y)
+ ?.closest('[data-remotion-studio-canvas-rotation]') === surface
+ ) {
+ return candidate;
+ }
+ }
+ }
+
+ throw new Error('Rotation surface should have an interactive point');
+ });
+ await page.mouse.move(point.x, point.y);
+ await page.mouse.down();
+ await page.mouse.move(point.x + 40, point.y + 30, {steps: 5});
+ await page.mouse.up();
+
+ await expect.poll(readRotationKeyframes).toEqual([30, 40, 60]);
+ });
+});
diff --git a/packages/example/src/E2eTestRoot.tsx b/packages/example/src/E2eTestRoot.tsx
index 13583705d7..800918a8d2 100644
--- a/packages/example/src/E2eTestRoot.tsx
+++ b/packages/example/src/E2eTestRoot.tsx
@@ -18,6 +18,7 @@ import {SchemaTest, schemaTestSchema} from './SchemaTest';
import {TimelineVirtualizationTestbed} from './TimelineVirtualizationTestbed';
import {VisualControls} from './VisualControls';
import {VisualMode3D} from './VisualMode3D';
+import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';
import {SequenceShiftRepro} from './VisualModeTests/SequenceShiftRepro';
const UseCurrentScaleOnLoad: React.FC = () => {
@@ -182,6 +183,14 @@ export const E2eTestRoot: React.FC = () => {
fps={30}
durationInFrames={90}
/>
+ <Composition
+ id="affine-frame-clock"
+ component={AffineFrameClock}
+ width={1280}
+ height={720}
+ fps={30}
+ durationInFrames={60}
+ />
<Composition
id="sequence-shift-repro"
component={SequenceShiftRepro}
diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsx
index ec99465287..de3a507c53 100644
--- a/packages/example/src/Root.tsx
+++ b/packages/example/src/Root.tsx
@@ -303,6 +303,7 @@ import {VideoTestingPlayback} from './VideoTesting/playback';
import {VideoTestingTrim} from './VideoTesting/trim';
import {RemotionMediaVideoTexture} from './VideoTexture';
import {VisualControls} from './VisualControls';
+import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';
import {FastUpdates} from './VisualModeTests/FastUpdates';
import {FastUpdatesNested} from './VisualModeTests/FastUpdatesNested';
import {
@@ -2875,6 +2876,14 @@ export const Index: React.FC = () => {
durationInFrames={180}
/>
<Folder name="VisualModeTests">
+ <Composition
+ id="affine-frame-clock"
+ component={AffineFrameClock}
+ width={1280}
+ height={720}
+ fps={30}
+ durationInFrames={60}
+ />
<Composition
id="sequence-shift-repro"
component={SequenceShiftRepro}
diff --git a/packages/example/src/VisualModeTests/AffineFrameClock.tsx b/packages/example/src/VisualModeTests/AffineFrameClock.tsx
new file mode 100644
index 0000000000..7f2eeba4e9
--- /dev/null
+++ b/packages/example/src/VisualModeTests/AffineFrameClock.tsx
@@ -0,0 +1,23 @@
+import React from 'react';
+import {AbsoluteFill, interpolate, Sequence, useCurrentFrame} from 'remotion';
+
+export const AffineFrameClock: React.FC = () => {
+ const frame = useCurrentFrame();
+ const captureFrame = frame + 30;
+
+ return (
+ <AbsoluteFill style={{backgroundColor: '#111'}} from={-30}>
+ <Sequence
+ name="Affine frame clock"
+ durationInFrames={60}
+ style={{
+ height: 400,
+ rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg']),
+ width: 400,
+ }}
+ >
+ <AbsoluteFill style={{backgroundColor: '#0b84ff'}} />
+ </Sequence>
+ </AbsoluteFill>
+ );
+};
diff --git a/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts b/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts
index 8d6432148a..5858ec7d3e 100644
--- a/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts
+++ b/packages/studio-server/src/preview-server/routes/can-update-sequence-props.ts
@@ -904,26 +904,69 @@ const getDefaultFrameDisplayOffsetAdjustment = ({
return result?.adjustment ?? null;
};
-const getCurrentFrameDisplayOffsetAdjustment = ({
+type ResolvedCurrentFrameExpression = {
+ readonly bindingScopePath: recast.types.NodePath;
+ readonly offset: number;
+};
+
+const resolveCurrentFrameExpression = ({
node,
ast,
- videoConfigValues,
+ seenDeclarations,
}: {
node: Expression;
ast: File;
- videoConfigValues: VideoConfigIdentifierValues;
-}): {
- readonly adjustment: number;
- readonly hasEnclosingElement: boolean;
-} | null => {
+ seenDeclarations: Set<object>;
+}): ResolvedCurrentFrameExpression | null => {
if (node.type === 'TSAsExpression') {
- return getCurrentFrameDisplayOffsetAdjustment({
+ return resolveCurrentFrameExpression({
node: node.expression as Expression,
ast,
- videoConfigValues,
+ seenDeclarations,
});
}
+ if (
+ node.type === 'BinaryExpression' &&
+ (node.operator === '+' || node.operator === '-')
+ ) {
+ const rightValue = getNumericValue(node.right as Expression);
+ if (rightValue !== null && Number.isFinite(rightValue)) {
+ const resolvedLeft = resolveCurrentFrameExpression({
+ node: node.left as Expression,
+ ast,
+ seenDeclarations,
+ });
+ if (resolvedLeft !== null) {
+ return {
+ ...resolvedLeft,
+ offset:
+ resolvedLeft.offset +
+ (node.operator === '+' ? rightValue : -rightValue),
+ };
+ }
+ }
+
+ if (node.operator === '+') {
+ const leftValue = getNumericValue(node.left as Expression);
+ if (leftValue !== null && Number.isFinite(leftValue)) {
+ const resolvedRight = resolveCurrentFrameExpression({
+ node: node.right as Expression,
+ ast,
+ seenDeclarations,
+ });
+ if (resolvedRight !== null) {
+ return {
+ ...resolvedRight,
+ offset: resolvedRight.offset + leftValue,
+ };
+ }
+ }
+ }
+
+ return null;
+ }
+
if (node.type !== 'Identifier') {
return null;
}
@@ -935,7 +978,9 @@ const getCurrentFrameDisplayOffsetAdjustment = ({
}
let matchingDeclarations = 0;
- let isCurrentFrameBinding = false;
+ let matchingDeclaration: object | null = null;
+ let initializer: Expression | null = null;
+ let isConstDeclaration = false;
recast.types.visit(bindingScope.path, {
visitVariableDeclarator(p) {
const {id, init} = p.node;
@@ -948,23 +993,90 @@ const getCurrentFrameDisplayOffsetAdjustment = ({
}
matchingDeclarations++;
- isCurrentFrameBinding = Boolean(
- init && isUseCurrentFrameCall(init as Expression, ast),
- );
+ const declaration = p.parentPath.node;
+ if (init) {
+ matchingDeclaration = p.node;
+ initializer = init as Expression;
+ isConstDeclaration =
+ declaration.type === 'VariableDeclaration' &&
+ declaration.kind === 'const';
+ }
return false;
},
});
- if (matchingDeclarations !== 1 || !isCurrentFrameBinding) {
+ if (
+ matchingDeclarations !== 1 ||
+ matchingDeclaration === null ||
+ initializer === null ||
+ seenDeclarations.has(matchingDeclaration)
+ ) {
+ return null;
+ }
+
+ if (isUseCurrentFrameCall(initializer, ast)) {
+ return {bindingScopePath: bindingScope.path, offset: 0};
+ }
+
+ if (!isConstDeclaration) {
+ return null;
+ }
+
+ const nextSeenDeclarations = new Set(seenDeclarations);
+ nextSeenDeclarations.add(matchingDeclaration);
+ return resolveCurrentFrameExpression({
+ node: initializer,
+ ast,
+ seenDeclarations: nextSeenDeclarations,
+ });
+};
+
+const getCurrentFrameDisplayOffsetAdjustment = ({
+ node,
+ ast,
+ videoConfigValues,
+}: {
+ node: Expression;
+ ast: File;
+ videoConfigValues: VideoConfigIdentifierValues;
+}): {
+ readonly adjustment: number;
+ readonly hasEnclosingElement: boolean;
+} | null => {
+ if (node.type === 'TSAsExpression') {
+ return getCurrentFrameDisplayOffsetAdjustment({
+ node: node.expression as Expression,
+ ast,
+ videoConfigValues,
+ });
+ }
+
+ const nodePath = findNodePath(ast, node);
+ const resolved = resolveCurrentFrameExpression({
+ node,
+ ast,
+ seenDeclarations: new Set(),
+ });
+ if (!nodePath || resolved === null) {
return null;
}
- return getFrameDisplayOffsetAdjustmentBetweenPaths({
+ const frameDisplayOffset = getFrameDisplayOffsetAdjustmentBetweenPaths({
startPath: nodePath,
- endPath: bindingScope.path,
+ endPath: resolved.bindingScopePath,
videoConfigValues,
});
+ if (frameDisplayOffset === null) {
+ return null;
+ }
+
+ return {
+ ...frameDisplayOffset,
+ // interpolate(frame + offset, [keyframe], ...) reaches the keyframe at
+ // composition frame `keyframe - offset`.
+ adjustment: frameDisplayOffset.adjustment - resolved.offset,
+ };
};
export const getComputedStatus = (
diff --git a/packages/studio-server/src/test/compute-sequence-props-status.test.ts b/packages/studio-server/src/test/compute-sequence-props-status.test.ts
index 325d5d5f0a..d1995559d0 100644
--- a/packages/studio-server/src/test/compute-sequence-props-status.test.ts
+++ b/packages/studio-server/src/test/compute-sequence-props-status.test.ts
@@ -1176,6 +1176,60 @@ export const Example: React.FC = () => {
});
});
+test('computeSequencePropsStatus resolves affine aliases of useCurrentFrame', () => {
+ const input = `import React from 'react';
+import {AbsoluteFill, Sequence, interpolate, useCurrentFrame} from 'remotion';
+
+export const Example: React.FC = () => {
+ const frame = useCurrentFrame();
+ const captureFrame = frame + 30;
+ let mutableCaptureFrame = frame + 30;
+ return (
+ <AbsoluteFill from={-30}>
+ <Sequence
+ style={{
+ rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg']),
+ opacity: interpolate(captureFrame - 10, [20, 50], [0, 1]),
+ scale: interpolate(frame * 2, [0, 60], [1, 2]),
+ translate: interpolate(mutableCaptureFrame, [30, 60], ['0px', '10px']),
+ }}
+ />
+ </AbsoluteFill>
+ );
+};
+`;
+ const result = computeSequencePropsStatusFromContent({
+ videoConfigValues: null,
+ fileContents: input,
+ nodePath: getNodePathFromContent(input, 10),
+ componentIdentity: null,
+ keys: ['style.rotate', 'style.opacity', 'style.scale', 'style.translate'],
+ effects: [],
+ });
+
+ expect(result.canUpdate).toBe(true);
+ if (!result.canUpdate) throw new Error('Expected canUpdate to be true');
+
+ expect(result.props['style.rotate']).toMatchObject({
+ status: 'keyframed',
+ keyframeDisplayOffsetAdjustment: 0,
+ keyframes: [
+ {frame: 30, value: '0deg'},
+ {frame: 60, value: '30deg'},
+ ],
+ });
+ expect(result.props['style.opacity']).toMatchObject({
+ status: 'keyframed',
+ keyframeDisplayOffsetAdjustment: 10,
+ keyframes: [
+ {frame: 20, value: 0},
+ {frame: 50, value: 1},
+ ],
+ });
+ expect(result.props['style.scale']).toEqual({status: 'computed'});
+ expect(result.props['style.translate']).toEqual({status: 'computed'});
+});
+
test('computeSequencePropsStatus reports computed when a timing component offset is uncertain', () => {
const input = `import React from 'react';
import {AbsoluteFill, interpolate, useCurrentFrame} from 'remotion';
diff --git a/packages/studio-server/src/test/update-keyframes.test.ts b/packages/studio-server/src/test/update-keyframes.test.ts
index 9594533458..23245ee379 100644
--- a/packages/studio-server/src/test/update-keyframes.test.ts
+++ b/packages/studio-server/src/test/update-keyframes.test.ts
@@ -29,6 +29,54 @@ const videoConfigValues = {
width: 1920,
};
+test('updateSequenceKeyframes preserves an affine frame alias', async () => {
+ const input = `import React from 'react';
+import {AbsoluteFill, Sequence, interpolate, useCurrentFrame} from 'remotion';
+
+export const Example: React.FC = () => {
+ const frame = useCurrentFrame();
+ const captureFrame = frame + 30;
+ return (
+ <AbsoluteFill from={-30}>
+ <Sequence style={{rotate: interpolate(captureFrame, [30, 60], ['0deg', '30deg'])}} />
+ </AbsoluteFill>
+ );
+};
+`;
+ const {output, updatedNodePath} = await updateSequenceKeyframes({
+ input,
+ nodePath: lineColumnToNodePath(input, 9),
+ updates: [
+ {
+ key: 'style.rotate',
+ operation: {type: 'add', frame: 45, value: '15deg'},
+ },
+ ],
+ videoConfigValues,
+ });
+
+ expect(output).toMatch(
+ /interpolate\(\s*captureFrame,\s*\[30, 45, 60\],\s*\['0deg', '15deg', '30deg'\]/,
+ );
+ const status = computeSequencePropsStatusFromContent({
+ fileContents: output,
+ nodePath: updatedNodePath,
+ componentIdentity: null,
+ keys: ['style.rotate'],
+ effects: [],
+ videoConfigValues,
+ });
+ expect(status.props['style.rotate']).toMatchObject({
+ status: 'keyframed',
+ keyframeDisplayOffsetAdjustment: 0,
+ keyframes: [
+ {frame: 30, value: '0deg'},
+ {frame: 45, value: '15deg'},
+ {frame: 60, value: '30deg'},
+ ],
+ });
+});
+
test('updateSequenceKeyframes preserves representable video config frame expressions', async () => {
const input = `import React from 'react';
import {Sequence, interpolate, useCurrentFrame, useVideoConfig} from 'remotion';