Skip to content

The VS Code of video editing, built for agents.

TypeScript
4.8 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// The thumbnail of an image, video or mask file: scaled to a width, its
// aspect ratio kept, encoded as WebP. For a video it is the frame shortly
// after the start; for a mask, its middle tracked frame. The decoder does
// the scaling (mediabunny's CanvasSink for video, createImageBitmap for
// images), so the picture is encoded once, as it comes.

import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';

import { MASK_MIME_TYPE, MaskFile } from '../mask';
import { traceMask } from '../mask-outline';
import { getSvgDimensions } from '../probe';

/** The width the asset bar shows; what the cache stores without a variant. */
export const DEFAULT_THUMBNAIL_WIDTH = 300;

/** How far into a video the thumbnail frame is taken from, in seconds. */
const VIDEO_THUMBNAIL_OFFSET = 0.3;

const WEBP_QUALITY = 0.7;

/** A mask thumbnail's object and background: white on near-black, as a matte reads. */
const MASK_FOREGROUND = '#ffffff';
const MASK_BACKGROUND = '#1a1a1a';

/** A WebP thumbnail of `file`, `width` wide, or null when it is no image, video with a picture, or mask. */
export async function deriveThumbnail(file: Blob, mimeType: string, width = DEFAULT_THUMBNAIL_WIDTH): Promise<Blob | null> {
	if (mimeType === 'image/svg+xml') return svgThumbnail(file, width);
	if (mimeType.startsWith('image/')) return imageThumbnail(file, width);
	if (mimeType.startsWith('video/')) return videoThumbnail(file, width);
	if (mimeType === MASK_MIME_TYPE) return maskThumbnail(file, width);
	return null;
}

async function imageThumbnail(file: Blob, width: number): Promise<Blob | null> {
	const bitmap = await createImageBitmap(file, { resizeWidth: width, resizeQuality: 'high' });
	try {
		const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
		const ctx = canvas.getContext('bitmaprenderer');
		if (!ctx) throw new Error('Could not create a bitmaprenderer context');
		ctx.transferFromImageBitmap(bitmap);
		return await encode(canvas);
	} finally {
		bitmap.close();
	}
}

/**
 * createImageBitmap takes no SVG blob, so an SVG goes through an `<img>`
 * and is drawn at the thumbnail's size: rasterized there, not scaled down.
 * Its size comes from the file, as an SVG with only a viewBox reports none.
 */
async function svgThumbnail(file: Blob, width: number): Promise<Blob | null> {
	const size = await getSvgDimensions(file);
	const height = Math.max(1, Math.round((width * size.height) / size.width));
	const url = URL.createObjectURL(file);
	try {
		const img = new Image();
		img.src = url;
		await img.decode();
		const canvas = new OffscreenCanvas(width, height);
		const ctx = canvas.getContext('2d');
		if (!ctx) throw new Error('Could not create a 2d context');
		ctx.drawImage(img, 0, 0, width, height);
		return await encode(canvas);
	} finally {
		URL.revokeObjectURL(url);
	}
}

async function videoThumbnail(file: Blob, width: number): Promise<Blob | null> {
	const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
	try {
		const track = await input.getPrimaryVideoTrack();
		if (!track) return null;
		const sink = new CanvasSink(track, { width });
		const first = await track.getFirstTimestamp();
		const wrapped = (await sink.getCanvas(first + VIDEO_THUMBNAIL_OFFSET)) ?? (await sink.getCanvas(first));
		if (!wrapped) return null;
		return await encode(wrapped.canvas);
	} finally {
		input.dispose();
	}
}

async function maskThumbnail(file: Blob, width: number): Promise<Blob | null> {
	const mask = await MaskFile.read(file);
	const { gridWidth, gridHeight, width: frameWidth, height: frameHeight, frameCount } = mask.header;
	const tracked: number[] = [];
	for (let i = 0; i < frameCount; i++) if (mask.has(i)) tracked.push(i);
	const middle = tracked[Math.floor(tracked.length / 2)];
	if (middle === undefined || !gridWidth || !gridHeight) return null;

	const field = new Int8Array(gridWidth * gridHeight);
	mask.field(middle, field);
	const outline = new Path2D();
	traceMask(field, gridWidth, gridHeight, outline);

	const height = Math.max(1, Math.round((width * (frameHeight || gridHeight)) / (frameWidth || gridWidth)));
	const canvas = new OffscreenCanvas(width, height);
	const ctx = canvas.getContext('2d')!;
	ctx.fillStyle = MASK_BACKGROUND;
	ctx.fillRect(0, 0, width, height);
	ctx.scale(width / gridWidth, height / gridHeight);
	ctx.fillStyle = MASK_FOREGROUND;
	ctx.fill(outline);
	return encode(canvas);
}

function encode(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob | null> {
	if (canvas instanceof OffscreenCanvas) return canvas.convertToBlob({ type: 'image/webp', quality: WEBP_QUALITY });
	return new Promise((resolve) => canvas.toBlob(resolve, 'image/webp', WEBP_QUALITY));
}