Skip to content

🎥 Make videos programmatically with React

Update announcement scene and transcription docs

Jonny Burger7a84ad05bad8
diff --git a/bun.lock b/bun.lock
index 76294b99de..b53274aac0 100644
--- a/bun.lock
+++ b/bun.lock
@@ -107,7 +107,10 @@
"name": "@remotion/brand",
"version": "4.0.527",
"dependencies": {
+ "@huggingface/transformers": "4.2.0",
"@mediabunny/prores": "catalog:",
+ "@remotion/video-matting": "workspace:*",
+ "@remotion/whisper-webgpu": "4.0.527",
},
"devDependencies": {
"@code-hike/lighter": "1.0.3",
diff --git a/packages/brand/package.json b/packages/brand/package.json
index 713c8d298e..0a71cf9718 100644
--- a/packages/brand/package.json
+++ b/packages/brand/package.json
@@ -17,7 +17,10 @@
"render-homepage-assets": "./render-homepage-assets.sh"
},
"dependencies": {
- "@mediabunny/prores": "catalog:"
+ "@huggingface/transformers": "4.2.0",
+ "@mediabunny/prores": "catalog:",
+ "@remotion/video-matting": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*"
},
"devDependencies": {
"@code-hike/lighter": "1.0.3",
diff --git a/packages/brand/public/whats1-base-base.webm b/packages/brand/public/whats1-base-base.webm
new file mode 100644
index 0000000000..65903da67c
Binary files /dev/null and b/packages/brand/public/whats1-base-base.webm differ
diff --git a/packages/brand/public/whats1-base-foreground.webm b/packages/brand/public/whats1-base-foreground.webm
new file mode 100644
index 0000000000..a9b8c00de1
Binary files /dev/null and b/packages/brand/public/whats1-base-foreground.webm differ
diff --git a/packages/brand/public/whats1-base.webm b/packages/brand/public/whats1-base.webm
new file mode 100644
index 0000000000..fb295ec247
Binary files /dev/null and b/packages/brand/public/whats1-base.webm differ
diff --git a/packages/brand/public/whats1-foreground.webm b/packages/brand/public/whats1-foreground.webm
new file mode 100644
index 0000000000..3e7605d906
Binary files /dev/null and b/packages/brand/public/whats1-foreground.webm differ
diff --git a/packages/brand/src/announcements/whats-new-in-remotion/Scene1.tsx b/packages/brand/src/announcements/whats-new-in-remotion/Scene1.tsx
index f862960157..65f197c011 100644
--- a/packages/brand/src/announcements/whats-new-in-remotion/Scene1.tsx
+++ b/packages/brand/src/announcements/whats-new-in-remotion/Scene1.tsx
@@ -1,5 +1,9 @@
+import {BasicCaptions} from './basic-captions.element';
+import {halftone} from '@remotion/effects/halftone';
+import {grayscale} from '@remotion/effects/grayscale';
+import {blur} from '@remotion/effects/blur';
import {Video} from '@remotion/media';
-import {AbsoluteFill, Sequence, useVideoConfig} from 'remotion';
+import {AbsoluteFill, Sequence, useVideoConfig, staticFile} from 'remotion';
import {assetUrl} from './assets';
import {SILENCES} from './Composition';
import {LowerThird} from './LowerThird';
@@ -15,10 +19,99 @@ export const Scene1: React.FC = () => {
return (
<AbsoluteFill>
<Video
- src={assetUrl(FILE)}
+ src={staticFile('whats1-base-base.webm')}
trimBefore={trimBefore}
trimAfter={trimAfter}
+ effects={[blur({
+ radius: 13
+ }), grayscale({})]}
/>
+ <BasicCaptions captions={[
+ {
+ "text": "Here",
+ "startMs": 520,
+ "endMs": 700,
+ "timestampMs": 610,
+ "confidence": null
+ },
+ {
+ "text": " are",
+ "startMs": 700,
+ "endMs": 880,
+ "timestampMs": 790,
+ "confidence": null
+ },
+ {
+ "text": " some",
+ "startMs": 880,
+ "endMs": 1020,
+ "timestampMs": 950,
+ "confidence": null
+ },
+ {
+ "text": " of",
+ "startMs": 1020,
+ "endMs": 1140,
+ "timestampMs": 1080,
+ "confidence": null
+ },
+ {
+ "text": " the",
+ "startMs": 1140,
+ "endMs": 1380,
+ "timestampMs": 1260,
+ "confidence": null
+ },
+ {
+ "text": " things",
+ "startMs": 1380,
+ "endMs": 1560,
+ "timestampMs": 1470,
+ "confidence": null
+ },
+ {
+ "text": " that",
+ "startMs": 1560,
+ "endMs": 1720,
+ "timestampMs": 1640,
+ "confidence": null
+ },
+ {
+ "text": " we",
+ "startMs": 1720,
+ "endMs": 2080,
+ "timestampMs": 1900,
+ "confidence": null
+ },
+ {
+ "text": " recently",
+ "startMs": 2080,
+ "endMs": 2640,
+ "timestampMs": 2360,
+ "confidence": null
+ },
+ {
+ "text": " improved",
+ "startMs": 2640,
+ "endMs": 2960,
+ "timestampMs": 2800,
+ "confidence": null
+ },
+ {
+ "text": " in",
+ "startMs": 2960,
+ "endMs": 3140,
+ "timestampMs": 3050,
+ "confidence": null
+ },
+ {
+ "text": " Remotion.",
+ "startMs": 3140,
+ "endMs": 4000,
+ "timestampMs": 3570,
+ "confidence": null
+ }
+ ]} trimBefore={trimBefore} durationInFrames={114} />
<Sequence from={15} layout="none">
<LowerThird
name="Jonny Burger"
diff --git a/packages/brand/src/announcements/whats-new-in-remotion/basic-captions.element.tsx b/packages/brand/src/announcements/whats-new-in-remotion/basic-captions.element.tsx
new file mode 100644
index 0000000000..f936df1927
--- /dev/null
+++ b/packages/brand/src/announcements/whats-new-in-remotion/basic-captions.element.tsx
@@ -0,0 +1,174 @@
+import type {Caption} from '@remotion/captions';
+import {createTikTokStyleCaptions} from '@remotion/captions';
+import React, {forwardRef, useImperativeHandle, useMemo, useRef} from 'react';
+import {
+ Interactive,
+ Sequence,
+ useCurrentFrame,
+ useVideoConfig,
+ type InteractiveBaseProps,
+ type InteractiveTransformProps,
+ type InteractivitySchema,
+ type SequenceControls,
+ type SequenceProps,
+} from 'remotion';
+
+type BasicCaptionsProps = InteractiveBaseProps &
+ InteractiveTransformProps &
+ Pick<SequenceProps, 'from' | 'durationInFrames' | 'trimBefore' | 'width' | 'height'> & {
+ readonly captions: Caption[];
+ readonly playbackRate?: number;
+ readonly combineTokensWithinMilliseconds?: number;
+ };
+
+const BasicCaptionsContent: React.FC<{
+ readonly captions: Caption[];
+ readonly playbackRate: number;
+ readonly trimBefore: number;
+ readonly combineTokensWithinMilliseconds: number;
+}> = ({captions, playbackRate, trimBefore, combineTokensWithinMilliseconds}) => {
+ const frame = useCurrentFrame();
+ const {fps} = useVideoConfig();
+ const pages = useMemo(
+ () =>
+ createTikTokStyleCaptions({
+ captions,
+ combineTokensWithinMilliseconds,
+ }).pages,
+ [captions, combineTokensWithinMilliseconds],
+ );
+ // The Sequence frame already includes trimBefore; only elapsed frames speed up.
+ const currentTimeMs =
+ ((trimBefore + (frame - trimBefore) * playbackRate) / fps) * 1000;
+ const page = pages.find(
+ (candidate) =>
+ currentTimeMs >= candidate.startMs &&
+ currentTimeMs < candidate.startMs + candidate.durationMs,
+ );
+
+ if (!page) {
+ return null;
+ }
+
+ return (
+ <div
+ style={{
+ backgroundColor: 'rgba(64, 64, 64, 0.75)',
+ color: '#ffffff',
+ display: '-webkit-box',
+ fontFamily: 'Arial, Helvetica, sans-serif',
+ fontSize: 64,
+ fontWeight: 400,
+ lineHeight: 1.2,
+ overflow: 'hidden',
+ padding: '14px 22px',
+ textAlign: 'center',
+ textWrap: 'balance',
+ WebkitBoxOrient: 'vertical',
+ WebkitLineClamp: 2,
+ whiteSpace: 'pre-wrap',
+ }}
+ >
+ {page.text.trim()}
+ </div>
+ );
+};
+
+const basicCaptionsSchema = {
+ ...Interactive.baseSchema,
+ ...Interactive.captionsSchema,
+ width: {
+ type: 'number',
+ min: 1,
+ step: 1,
+ default: undefined,
+ description: 'Caption area width',
+ hiddenFromList: false,
+ },
+ height: {
+ type: 'number',
+ min: 1,
+ step: 1,
+ default: undefined,
+ description: 'Caption area height',
+ hiddenFromList: false,
+ },
+ combineTokensWithinMilliseconds: {
+ type: 'number',
+ min: 0,
+ step: 50,
+ default: 2000,
+ description: 'Time between caption pages',
+ hiddenFromList: false,
+ },
+ ...Interactive.transformSchema,
+} as const satisfies InteractivitySchema;
+
+const BasicCaptionsInner = forwardRef<
+ HTMLDivElement,
+ BasicCaptionsProps & {readonly controls: SequenceControls | undefined}
+>(
+ (
+ {
+ captions,
+ combineTokensWithinMilliseconds = 2000,
+ controls,
+ durationInFrames,
+ from,
+ height = 220,
+ name,
+ playbackRate = 1,
+ style,
+ trimBefore,
+ width = 900,
+ ...interactiveProps
+ },
+ ref,
+ ) => {
+ const outlineRef = useRef<HTMLDivElement>(null);
+ useImperativeHandle(ref, () => outlineRef.current as HTMLDivElement, []);
+
+ return (
+ <Sequence
+ layout="none"
+ {...interactiveProps}
+ controls={controls}
+ name={name ?? 'Basic captions'}
+ from={from}
+ durationInFrames={durationInFrames}
+ trimBefore={trimBefore}
+ outlineRef={outlineRef}
+ >
+ <div
+ ref={outlineRef}
+ style={{
+ alignItems: 'center',
+ display: 'flex',
+ justifyContent: 'center',
+ position: 'absolute',
+ bottom: 120,
+ left: '50%',
+ transform: 'translateX(-50%)',
+ width,
+ height,
+ ...style,
+ }}
+ >
+ <BasicCaptionsContent
+ captions={captions}
+ combineTokensWithinMilliseconds={combineTokensWithinMilliseconds}
+ playbackRate={playbackRate}
+ trimBefore={trimBefore ?? 0}
+ />
+ </div>
+ </Sequence>
+ );
+ },
+);
+
+export const BasicCaptions = Interactive.withSchema({
+ Component: BasicCaptionsInner,
+ componentName: '<BasicCaptions>',
+ schema: basicCaptionsSchema,
+ supportsEffects: false,
+}) as React.FC<BasicCaptionsProps>;
diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdx
index 6e76f0145a..56f06323e0 100644
--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx
+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx
@@ -81,7 +81,10 @@ Called while a previously downloaded model is initialized. Call [`downloadWhispe
### signal?<AvailableFrom v="4.0.528" />
-An [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) for canceling transcription. Default: no cancellation signal. Calling `abort()` on its controller stops token generation and rejects the promise with the signal's reason. A GPU operation already in progress must finish first. No partial transcript is returned.
+An [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) for canceling transcription.
+Calling `abort()` on its controller stops token generation and rejects the promise with the signal's reason.
+A GPU operation already in progress must finish first.
+No partial transcript is returned.
```ts twoslash title="cancel-transcription.ts"
import {transcribe} from '@remotion/whisper-webgpu';