Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor media output structures and enhance waveform functionality

konstantin-paulus25b418769157
diff --git a/apps/cli/reference/media/filmstrip.md b/apps/cli/reference/media/filmstrip.md
index beb35ce3..1855871b 100644
--- a/apps/cli/reference/media/filmstrip.md
+++ b/apps/cli/reference/media/filmstrip.md
@@ -14,13 +14,11 @@ Tick labels use `HH:MM:SS:FF` timecode (hours, minutes, seconds, frame within th
## Output
-One JSON object, the absolute path to the written PNG plus the grid geometry:
+One JSON object, the absolute path to the written PNG:
```ts
{
path: string, // e.g. "/tmp/3f2c1a8e-....png", or the --output path
- column: { width: number, count: number, duration: number },
- row: { height: number, count: number, duration: number },
}
```
diff --git a/apps/cli/reference/media/waveform.md b/apps/cli/reference/media/waveform.md
index 9ad30a50..d4e3b9d8 100644
--- a/apps/cli/reference/media/waveform.md
+++ b/apps/cli/reference/media/waveform.md
@@ -14,13 +14,12 @@ Tick labels use `HH:MM:SS:FF` timecode (hours, minutes, seconds, frame within th
## Output
-One JSON object, the absolute path to the written PNG plus the grid geometry:
+One JSON object, the absolute path to the written PNG plus the silent stretches (the red spans on the waveform) as `[start, end]` second ranges, in absolute seconds (offset by `--start` when a window is used):
```ts
{
path: string, // e.g. "/tmp/3f2c1a8e-....png", or the --output path
- column: { width: number, count: number, duration: number },
- row: { height: number, count: number, duration: number },
+ silences: Array<{ start: number, end: number }>,
}
```
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index f1556f47..03be64b7 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -130,14 +130,13 @@ export type TranscriptSegment = { text: string; words: TranscriptWord[] };
export type MediaTranscribeResult = { segments: TranscriptSegment[] };
export type MediaFilmstripRequest = AssetRef & { start?: number; end?: number; scale?: number };
-export type MediaFilmstripResult = {
- base64: string;
-} & Record<string, unknown>;
+export type MediaFilmstripResult = { base64: string };
export type MediaWaveformRequest = AssetRef & { start?: number; end?: number; scale?: number };
export type MediaWaveformResult = {
base64: string;
-} & Record<string, unknown>;
+ silences: Array<{ start: number; end: number }>;
+};
export type MediaListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean };
export type MediaListenResult = { result?: string; start?: number; end?: number };
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index d3f51c22..b542f5cf 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -1132,7 +1132,7 @@ media
media
.command("waveform")
.alias("wave")
- .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`)
+ .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`)
.argument("<id|path>", "video or audio asset id, or a local file to preview")
.option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`)
.option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
diff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.ts
index a9f6041c..429c7c30 100644
--- a/apps/web/src/components/engine/utils/transcode.ts
+++ b/apps/web/src/components/engine/utils/transcode.ts
@@ -100,7 +100,11 @@ export function formatTimestamp(seconds: number, frameRate: number): string {
export type RenderedPreview = {
dataUrl: string;
-} & Record<string, unknown>;
+};
+
+export type RenderedWaveform = RenderedPreview & {
+ silences: Array<{ start: number; end: number }>;
+};
export type TimeWindow = { start?: number; end?: number };
@@ -165,7 +169,6 @@ async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWind
const cells = rows * columns;
const secondsPerColumn = Math.max(duration / cells, 1 / frameRate);
- const secondsPerRow = secondsPerColumn * columns;
const canvas = document.createElement('canvas');
canvas.width = columnWidthInTokens * columns * PATCH_SIZE;
@@ -213,24 +216,12 @@ async function renderFilmstrip(asset: VideoAsset, input: Input, window: TimeWind
startOffset: windowStart,
});
- return {
- dataUrl: canvas.toDataURL('image/png'),
- column: {
- width: thumbnailWidthInPixels,
- count: columns,
- duration: secondsPerColumn,
- },
- row: {
- height: rowHeightInPixels,
- count: rows,
- duration: secondsPerRow,
- },
- };
+ return { dataUrl: canvas.toDataURL('image/png') };
}
// Render the audio track of a video or audio file as a loudness-over-time
// waveform with a timestamp ruler and silent stretches highlighted.
-export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedPreview> {
+export async function waveformAsset(asset: Asset, options?: PreviewOptions): Promise<RenderedWaveform> {
assert(asset.type === "VIDEO" || asset.type === "AUDIO", "The waveform needs a video or audio asset.");
const blob = await getAssetFile(asset);
const scale = resolveScale(options?.scale);
@@ -242,7 +233,7 @@ export async function waveformAsset(asset: Asset, options?: PreviewOptions): Pro
}
}
-async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedPreview> {
+async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, window: TimeWindow | undefined, scale: number): Promise<RenderedWaveform> {
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack, "No audio track found, so no waveform can be rendered.");
assert(
@@ -267,7 +258,6 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, wind
const cells = rows * columns;
const secondsPerColumn = duration / cells;
- const secondsPerRow = secondsPerColumn * columns;
const columnWidthInPixels = columnWidthInTokens * PATCH_SIZE;
const rowHeightInPixels = rowHeightInTokens * PATCH_SIZE;
@@ -286,13 +276,18 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, wind
const peaksPerSecond = Math.round(columnWidthInPixels / secondsPerColumn / WAVEFORM_SAMPLE_WIDTH);
const audioPeaks = await downsampleAudio(audioTrack, { peaksPerSecond, range: [windowStart, windowStart + duration] });
- drawWaveform(ctx, audioPeaks, {
+ const silentSpans = findSilentSpans(audioPeaks, peaksPerSecond);
+ const silences = silentSpans.map((span) => ({
+ start: windowStart + span.start / peaksPerSecond,
+ end: windowStart + span.end / peaksPerSecond,
+ }));
+
+ drawWaveform(ctx, audioPeaks, silentSpans, {
rows,
rowWidth,
rowHeightInPixels,
waveformOffset: rulerHeight,
waveformHeight,
- peaksPerSecond,
});
drawRuler(ctx, {
@@ -306,19 +301,7 @@ async function renderWaveform(asset: VideoAsset | AudioAsset, input: Input, wind
startOffset: windowStart,
});
- return {
- dataUrl: canvas.toDataURL('image/png'),
- column: {
- width: columnWidthInPixels,
- count: columns,
- duration: secondsPerColumn,
- },
- row: {
- height: rowHeightInPixels,
- count: rows,
- duration: secondsPerRow,
- },
- };
+ return { dataUrl: canvas.toDataURL('image/png'), silences };
}
type WaveformLayout = {
@@ -327,12 +310,31 @@ type WaveformLayout = {
rowHeightInPixels: number;
waveformOffset: number; // y offset within a row where the waveform begins
waveformHeight: number;
- peaksPerSecond: number;
};
-// Draws a center-aligned waveform, then overlays the silent stretches in red.
-function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArray, layout: WaveformLayout) {
- const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight, peaksPerSecond } = layout;
+// Scan the peaks for runs at/below the silence threshold that last at least
+// SILENCE_MIN_SECONDS, returned as [start, end) peak-index spans.
+function findSilentSpans(audioPeaks: Uint8ClampedArray, peaksPerSecond: number): Array<{ start: number; end: number }> {
+ const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);
+ const silentSpans: Array<{ start: number; end: number }> = [];
+ let runStart = -1;
+ for (let i = 0; i <= audioPeaks.length; i++) {
+ const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;
+ if (silent && runStart === -1) {
+ runStart = i;
+ } else if (!silent && runStart !== -1) {
+ if (i - runStart >= minSilencePeaks) {
+ silentSpans.push({ start: runStart, end: i });
+ }
+ runStart = -1;
+ }
+ }
+ return silentSpans;
+}
+
+// Draws a center-aligned waveform, then overlays the given silent stretches in red.
+function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArray, silentSpans: Array<{ start: number; end: number }>, layout: WaveformLayout) {
+ const { rows, rowWidth, rowHeightInPixels, waveformOffset, waveformHeight } = layout;
const peaksPerRow = Math.floor(rowWidth / WAVEFORM_SAMPLE_WIDTH);
const halfHeight = waveformHeight / 2;
@@ -353,21 +355,6 @@ function drawWaveform(ctx: CanvasRenderingContext2D, audioPeaks: Uint8ClampedArr
}
}
- const minSilencePeaks = Math.round(SILENCE_MIN_SECONDS * peaksPerSecond);
- const silentSpans: Array<{ start: number; end: number }> = [];
- let runStart = -1;
- for (let i = 0; i <= audioPeaks.length; i++) {
- const silent = i < audioPeaks.length && audioPeaks[i] <= SILENCE_THRESHOLD;
- if (silent && runStart === -1) {
- runStart = i;
- } else if (!silent && runStart !== -1) {
- if (i - runStart >= minSilencePeaks) {
- silentSpans.push({ start: runStart, end: i });
- }
- runStart = -1;
- }
- }
-
for (const span of silentSpans) {
let i = span.start;
while (i < span.end) {