The VS Code of video editing, built for agents.
TypeScript
Refactor media output structures and enhance waveform functionality
konstantin-paulus25b418769157
diff --git a/apps/cli/reference/media/filmstrip.md b/apps/cli/reference/media/filmstrip.mdindex beb35ce3..1855871b 100644--- a/apps/cli/reference/media/filmstrip.md+++ b/apps/cli/reference/media/filmstrip.md@@ -14,13 +14,11 @@ Tick labels use `HH:MM:SS:FF` timecode (hours, minutes, seconds, frame within th## Output-One JSON object, the absolute path to the written PNG plus the grid geometry:+One JSON object, the absolute path to the written PNG:```ts{path: string, // e.g. "/tmp/3f2c1a8e-....png", or the --output path- column: { width: number, count: number, duration: number },- row: { height: number, count: number, duration: number },}```diff --git a/apps/cli/reference/media/waveform.md b/apps/cli/reference/media/waveform.mdindex 9ad30a50..d4e3b9d8 100644--- a/apps/cli/reference/media/waveform.md+++ b/apps/cli/reference/media/waveform.md@@ -14,13 +14,12 @@ Tick labels use `HH:MM:SS:FF` timecode (hours, minutes, seconds, frame within th## Output-One JSON object, the absolute path to the written PNG plus the grid geometry:+One JSON object, the absolute path to the written PNG plus the silent stretches (the red spans on the waveform) as `[start, end]` second ranges, in absolute seconds (offset by `--start` when a window is used):```ts{path: string, // e.g. "/tmp/3f2c1a8e-....png", or the --output path- column: { width: number, count: number, duration: number },- row: { height: number, count: number, duration: number },+ silences: Array<{ start: number, end: number }>,}```diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex f1556f47..03be64b7 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -130,14 +130,13 @@ export type TranscriptSegment = { text: string; words: TranscriptWord[] };export type MediaTranscribeResult = { segments: TranscriptSegment[] };export type MediaFilmstripRequest = AssetRef & { start?: number; end?: number; scale?: number };-export type MediaFilmstripResult = {- base64: string;-} & Record<string, unknown>;+export type MediaFilmstripResult = { base64: string };export type MediaWaveformRequest = AssetRef & { start?: number; end?: number; scale?: number };export type MediaWaveformResult = {base64: string;-} & Record<string, unknown>;+ silences: Array<{ start: number; end: number }>;+};export type MediaListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };export type MediaListenResult = { result?: string; start?: number; end?: number };diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex d3f51c22..b542f5cf 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -1132,7 +1132,7 @@ mediamedia.command("waveform").alias("wave")- .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`)+ .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`).argument("<id|path>", "video or audio asset id, or a local file to preview").option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)diff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.tsindex a9f6041c..429c7c30 100644--- a/apps/web/src/components/engine/utils/transcode.ts+++ b/apps/web/src/components/engine/utils/transcode.ts@@ -100,7 +100,11 @@ export function formatTimestamp(seconds: number, frameRate: number): string {export type RenderedPreview = {dataUrl: string;-} & Record<string, unknown>;+};++export type RenderedWaveform = RenderedPreview & {+ silences: Array<{ start: number; end: number }>;+};export type TimeWindow = { start?: number; end?: number };@@ -165,7 +169,6 @@ async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWindconst cells = rows * columns;const secondsPerColumn = Math.max(duration / cells, 1 / frameRate);- const secondsPerRow = secondsPerColumn * columns;const canvas = document.createElement('canvas');canvas.width = columnWidthInTokens * columns * PATCH_SIZE;@@ -213,24 +216,12 @@ async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWindstartOffset: windowStart,});- return {- dataUrl: canvas.toDataURL('image/png'),- column: {- width: thumbnailWidthInPixels,- count: columns,- duration: secondsPerColumn,- },- row: {- height: rowHeightInPixels,- count: rows,- duration: secondsPerRow,- },- };+ return { dataUrl: canvas.toDataURL('image/png') };}// Render the audio track of a video or audio file as a loudness-over-time// waveform with a timestamp ruler and silent stretches highlighted.-export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedPreview> {+export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedWaveform> {assert(asset.type === "VIDEO" || asset.type === "AUDIO", "The waveform needs a video or audio asset.");const blob = await getAssetFile(asset);const scale = resolveScale(options?.scale);@@ -242,7 +233,7 @@ export async function waveformAsset(asset: Asset, options?: PreviewOptions): Pro}}-async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedPreview> {+async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedWaveform> {const audioTrack = await input.getPrimaryAudioTrack();assert(audioTrack, "No audio track found, so no waveform can be rendered.");assert(@@ -267,7 +258,6 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, windconst cells = rows * columns;const secondsPerColumn = duration / cells;- const secondsPerRow = secondsPerColumn * columns;const columnWidthInPixels = columnWidthInTokens * PATCH_SIZE;const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;@@ -286,13 +276,18 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, windconst peaksPerSecond = Math.round(columnWidthInPixels / secondsPerColumn / WAVEFORM_SAMPLE_WIDTH);const audioPeaks = await downsampleAudio(audioTrack, { peaksPerSecond, range: [windowStart, windowStart + duration] });- drawWaveform(ctx, audioPeaks, {+ const silentSpans = findSilentSpans(audioPeaks, peaksPerSecond);+ const silences = silentSpans.map((span) => ({+ start: windowStart + span.start / peaksPerSecond,+ end: windowStart + span.end / peaksPerSecond,+ }));++ drawWaveform(ctx, audioPeaks, silentSpans, {rows,rowWidth,rowHeightInPixels,waveformOffset: rulerHeight,waveformHeight,- peaksPerSecond,});drawRuler(ctx, {@@ -306,19 +301,7 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, windstartOffset: windowStart,});- return {- dataUrl: canvas.toDataURL('image/png'),- column: {- width: columnWidthInPixels,- count: columns,- duration: secondsPerColumn,- },- row: {- height: rowHeightInPixels,- count: rows,- duration: secondsPerRow,- },- };+ return { dataUrl: canvas.toDataURL('image/png'), silences };}type WaveformLayout = {@@ -327,12 +310,31 @@ type WaveformLayout = {rowHeightInPixels: number;waveformOffset: number; // y offset within a row where the waveform beginswaveformHeight: number;- peaksPerSecond: number;};-// Draws a center-aligned waveform, then overlays the silent stretches in red.-function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArray, layout: WaveformLayout) {- const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight, peaksPerSecond } = layout;+// Scan the peaks for runs at/below the silence threshold that last at least+// SILENCE_MIN_SECONDS, returned as [start, end) peak-index spans.+function findSilentSpans(audioPeaks: Uint8ClampedArray, peaksPerSecond: number): Array<{ start: number; end: number }> {+ const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);+ const silentSpans: Array<{ start: number; end: number }> = [];+ let runStart = -1;+ for (let i = 0; i <= audioPeaks.length; i++) {+ const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;+ if (silent && runStart === -1) {+ runStart = i;+ } else if (!silent && runStart !== -1) {+ if (i - runStart >= minSilencePeaks) {+ silentSpans.push({ start: runStart, end: i });+ }+ runStart = -1;+ }+ }+ return silentSpans;+}++// Draws a center-aligned waveform, then overlays the given silent stretches in red.+function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArray, silentSpans: Array<{ start: number; end: number }>, layout: WaveformLayout) {+ const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight } = layout;const peaksPerRow = Math.floor(rowWidth / WAVEFORM_SAMPLE_WIDTH);const halfHeight = waveformHeight / 2;@@ -353,21 +355,6 @@ function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArr}}- const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);- const silentSpans: Array<{ start: number; end: number }> = [];- let runStart = -1;- for (let i = 0; i <= audioPeaks.length; i++) {- const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;- if (silent && runStart === -1) {- runStart = i;- } else if (!silent && runStart !== -1) {- if (i - runStart >= minSilencePeaks) {- silentSpans.push({ start: runStart, end: i });- }- runStart = -1;- }- }-for (const span of silentSpans) {let i = span.start;while (i < span.end) {