The VS Code of video editing, built for agents.
TypeScript
Enhance audio downsampling logic and update captions documentation
konstantin-paulus07c47fae94c9
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 725ae9e2..781244e8 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -431,7 +431,7 @@ async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {const dir = opts.output ?? tmpdir();mkdirSync(dir, { recursive: true });try {- const shots = await editor.node.capture.query({ id: eid, frames, timestamp: opts.timestamp });+ const shots = await editor.node.capture.query({ id: eid, frames, timestamp: opts.timestamp }, GENERATE);for (const [i, { base64 }] of shots.entries()) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));diff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.tsindex 98d1953d..ba136f16 100644--- a/apps/web/src/components/engine/utils/transcode.ts+++ b/apps/web/src/components/engine/utils/transcode.ts@@ -462,8 +462,13 @@ type SamplerOptions = {async function downsampleAudio(track: InputAudioTrack, options: SamplerOptions) {const sink = new AudioSampleSink(track);const iterator = options.range ? sink.samples(...options.range) : sink.samples();+ const framesPerPeak = track.sampleRate / options.peaksPerSecond;++ const peaks: number[] = [];+ let frameCursor = 0; // frames consumed since the range start+ let currentPeak = 0; // index of the peak currently being accumulated+ let peakMax = 0;- const peaks: Uint8ClampedArray<ArrayBuffer>[] = [];for await (const sample of iterator) {try {const size = sample.allocationSize({ format: 'f32', planeIndex: 0 });@@ -472,44 +477,26 @@ async function downsampleAudio(track: InputAudioTrack, options: SamplerOptions)const numChannels = sample.numberOfChannels;const numFrames = floats.length / numChannels;- const duration = sample.duration;-- // Downsample to peaksPerSecond- const targetPeaks = Math.ceil(duration * options.peaksPerSecond);- const downsampledBytes = new Uint8ClampedArray(targetPeaks);-- // Downsample by calculating peaks directly for target positions- const ratio = numFrames / targetPeaks;-- for (let i = 0; i < targetPeaks; i++) {- const start = Math.floor(i * ratio);- const end = Math.floor((i + 1) * ratio);- let max = 0;- for (let frameIdx = start; frameIdx < end && frameIdx < numFrames; frameIdx++) {- for (let ch = 0; ch < numChannels; ch++) {- const floatIdx = frameIdx * numChannels + ch;- max = Math.max(max, Math.sqrt(Math.abs(floats[floatIdx])));- }+ for (let frameIdx = 0; frameIdx < numFrames; frameIdx++) {+ while (currentPeak < Math.floor(frameCursor / framesPerPeak)) {+ peaks.push(Math.floor(255 * peakMax));+ peakMax = 0;+ currentPeak++;}- downsampledBytes[i] = Math.floor(255 * max);+ for (let ch = 0; ch < numChannels; ch++) {+ peakMax = Math.max(peakMax, Math.sqrt(Math.abs(floats[frameIdx * numChannels + ch])));+ }+ frameCursor++;}-- peaks.push(downsampledBytes);} finally {sample.close();}}- const length = peaks.reduce((acc, peak) => acc + peak.length, 0);- const mergedPeaks = new Uint8ClampedArray(length);-- let offset = 0;- for (const peak of peaks) {- mergedPeaks.set(peak, offset);- offset += peak.length;- }+ // Flush the final in-progress peak.+ if (frameCursor > 0) peaks.push(Math.floor(255 * peakMax));- return mergedPeaks;+ return Uint8ClampedArray.from(peaks);}diff --git a/reference/jsx/README.md b/reference/jsx/README.mdindex ac277a0c..a9cae658 100644--- a/reference/jsx/README.md+++ b/reference/jsx/README.md@@ -65,24 +65,6 @@ const TITLES = [{ text: "Chapter One", start: 6, end: 9 },];-function Title(props: { text: string; start: Time; end: Time }) {- return (- <text- textAlign="center"- textBaseline="middle"- fill="#FFFFFF"- fontSize={128}- fontWeight="bold"- height={1080}- width={1920}- start={props.start}- end={props.end}- >- {props.text}- </text>- );-}-export default function Project() {return (<scene key="my-first-scene" name="MyFirstScene" fill="black" width={1920} height={1080}>@@ -93,16 +75,12 @@ export default function Project() {<image src={hero} x={40} y={40} width={200} height={112} />- <For each={TITLES}>{(t) => <Title {...t} />}</For>-<audiosrc="https://my.videoarchive.com/audio/video-xyz.wav"sourceIn={6}sourceOut={16}volume={-6}/>-- <captions /></scene>);}diff --git a/reference/jsx/captions.md b/reference/jsx/captions.mdindex 19d35635..43812ac0 100644--- a/reference/jsx/captions.md+++ b/reference/jsx/captions.md@@ -18,10 +18,19 @@ Transcripts are **cached**: every transcript asset records a fingerprint of the| `colors` | `string[]` | preset defaults | Fills the preset's color slots in order; any CSS color, alpha ignored. Ignored by presets without slots. || `verticalAlign` | `"top" \| "center" \| "bottom"` | preset default | Vertical placement of the caption block: anchored to the top or bottom safe margin (100 px), or centered. Horizontal placement stays with the preset. || `offsetX`, `offsetY` | `Animatable<number>` | `0` | Render-time nudge in px on top of the preset placement; subpixel values are kept. A slide animation drives the same channel and wins while it plays. |-| `start`, `end`, `sourceIn`, `sourceOut` | `Time` | full transcript | Trim which stretch of the transcript is captioned, using the same [timing](./timing.md) semantics as media nodes. See below. |+| `start`, `end`, `sourceIn`, `sourceOut` | `Time` | full transcript | Trim which stretch of the transcript is captioned, using the same [timing](./timing.md) semantics as media nodes. Set the source window (`sourceIn`/`sourceOut`) and timeline placement (`start`/`end`) together — see [Trimming](#trimming). |The preset positions the caption block; `verticalAlign` overrides only its vertical anchor (`whisper` and `cascade` default to `bottom`, all other presets to `center`), and `offsetX`/`offsetY` nudge the drawn result from there.+## Trimming++Captions carry the same [timing](./timing.md) props as media nodes, and the transcript is source content that must stay aligned to the audio — so advance `start` and `sourceIn` together (and `end`/`sourceOut`), never `start` alone. To show only from 15 s onward, set both `sourceIn={15}` and `start={15}`. One node can't skip a gap, so to blank captions out for a middle stretch — e.g. under an overlay — use two nodes that meet at the gap:++```tsx+<captions start={0} end={15} sourceIn={0} sourceOut={15} /> // before the overlay+<captions start={20} sourceIn={20} /> // after the overlay+```+## Presets`preset` selects the caption style: the same presets as the editor's caption inspector. Some presets expose **color slots**, filled in order by the `colors` prop; a missing or omitted entry falls back to the slot's default.diff --git a/reference/jsx/html-paint.md b/reference/jsx/html-paint.mdindex cc4db7bc..846af7d5 100644--- a/reference/jsx/html-paint.md+++ b/reference/jsx/html-paint.md@@ -1,6 +1,6 @@# `<html>`-An element whose children are **real HTML**: the browser lays them out at the element's box size and the result is drawn into the box via the [html-in-canvas](https://github.com/WICG/html-in-canvas) API. Use it for content that is painful to build from `<rect>` and `<text>`: styled cards, tables, code blocks, flex/grid layouts.+An element whose children are **real HTML**: the browser lays them out at the element's box size and the result is drawn into the box via the [html-in-canvas](https://github.com/WICG/html-in-canvas) API. `dapi` always drives the Diffusion Studio Electron app, which ships with this API enabled, so `<html>` is always available — reach for it liberally. It is the recommended way to build motion graphics, overlays, and any UI-heavy content: styled cards, tables, code blocks, flex/grid layouts — anything painful to assemble from `<rect>` and `<text>`.```tsx<html x={40} y={40} width={800} height={120} cornerRadius={24}>@@ -51,7 +51,7 @@ The compiled module is persisted with the document: on reload, export, and `dapi## Requirements and limitations-- Requires Chromium's html-in-canvas API, currently behind `chrome://flags/#canvas-draw-element`. Mounting `<html>` or `<htmlPaint>` fails with an explicit error when the API is unavailable.+- Relies on Chromium's html-in-canvas API. The Diffusion Studio Electron app that `dapi` drives ships with this API enabled, so it is always available in this environment — do not avoid `<html>` out of availability concerns.- Markup renders with the page's fonts and full CSS. Event handlers are dropped: the content is painted, not interactive.- CSS animations in the markup play on the wall clock, not the composition playhead. Animate the element's props with [keyframes](./keyframes.md) for frame-accurate motion.- Cross-origin subresources (e.g. remote images) are excluded from the painted output by the browser's read-back rules; use local assets.diff --git a/reference/jsx/timing.md b/reference/jsx/timing.mdindex 83cdef9f..d08f55bb 100644--- a/reference/jsx/timing.md+++ b/reference/jsx/timing.md@@ -12,6 +12,7 @@ Timing splits the two independent questions a clip answers: **where it sits on t## Semantics- `start` / `end` place the clip on the parent timeline. `sourceIn` / `sourceOut` select which part of the source plays. On-timeline duration always equals the played source length, so `end - start == sourceOut - sourceIn`.+- **Trimming the source is not the same as moving the clip.** To drop part of the source you must move `sourceIn` / `sourceOut`; changing `start` alone only slides the clip along the timeline while the full source keeps playing. If the source is meant to stay aligned with something else in the scene (synced audio, a transcript, another track), advance `start` and `sourceIn` **together** — moving one without the other offsets the content instead of trimming it.- **`end` and `sourceOut` are two spellings of the same out edge** — the clip's end in timeline time (`end`) versus source time (`sourceOut`). Set one; the last one set wins.- Sourceless nodes (`<rect>`, `<group>`, `<text>`) have no footage to trim, so you place them with `start` / `end` alone.- Instead of setting `start`, a media node can derive its placement from another node's audio with `syncTo` (see [audio-sync.md](./audio-sync.md)).