The VS Code of video editing, built for agents.
TypeScript
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { BYTES_PER_FLOAT, HIDDEN_DIM } from './constants';
import { WORKGROUP_SIZE, preprocessShader, transposeShader } from './shaders';
import type { Sam2Geometry } from './constants';
/** Degrees clockwise a decoded frame is turned for display. */
export type Rotation = 0 | 90 | 180 | 270;
export function createStorageBuffer(device: GPUDevice, size: number, label: string): GPUBuffer {
return device.createBuffer({
label,
size,
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,
});
}
function createPipeline(device: GPUDevice, code: string, label: string): GPUComputePipeline {
return device.createComputePipeline({
label,
layout: 'auto',
compute: { module: device.createShaderModule({ label, code }), entryPoint: 'main' },
});
}
function dispatch(device: GPUDevice, pipeline: GPUComputePipeline, bindGroup: GPUBindGroup, x: number, y: number): void {
const encoder = device.createCommandEncoder();
const pass = encoder.beginComputePass();
pass.setPipeline(pipeline);
pass.setBindGroup(0, bindGroup);
pass.dispatchWorkgroups(x, y);
pass.end();
device.queue.submit([encoder.finish()]);
}
const PARAMS_BYTES = 48;
const MAX_TAPS = 8;
/** Uploads a frame and turns it into the vision encoder's `pixel_values`. */
export class FramePreprocessor {
readonly pixels: GPUBuffer;
private readonly pipeline: GPUComputePipeline;
private readonly sampler: GPUSampler;
private readonly params: GPUBuffer;
private texture: GPUTexture | null = null;
constructor(
private readonly device: GPUDevice,
private readonly imageSize: number,
private readonly mean: readonly number[],
private readonly std: readonly number[],
) {
this.pixels = createStorageBuffer(device, 3 * imageSize * imageSize * BYTES_PER_FLOAT, 'sam2 pixel_values');
this.pipeline = createPipeline(device, preprocessShader(imageSize), 'sam2 preprocess');
this.sampler = device.createSampler({ magFilter: 'linear', minFilter: 'linear' });
this.params = device.createBuffer({
label: 'sam2 preprocess params',
size: PARAMS_BYTES,
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
});
}
run(frame: VideoFrame, rotation: Rotation): void {
const width = frame.displayWidth;
const height = frame.displayHeight;
const texture = this.textureFor(width, height);
this.device.queue.copyExternalImageToTexture({ source: frame }, { texture }, [width, height]);
const params = new ArrayBuffer(PARAMS_BYTES);
const floats = new Float32Array(params);
const uints = new Uint32Array(params);
floats[0] = width;
floats[1] = height;
uints[2] = rotation / 90;
uints[3] = Math.min(MAX_TAPS, Math.max(1, Math.ceil(Math.max(width, height) / this.imageSize)));
floats.set(this.mean, 4);
floats.set(this.std, 8);
this.device.queue.writeBuffer(this.params, 0, params);
const bindGroup = this.device.createBindGroup({
layout: this.pipeline.getBindGroupLayout(0),
entries: [
{ binding: 0, resource: { buffer: this.params } },
{ binding: 1, resource: texture.createView() },
{ binding: 2, resource: this.sampler },
{ binding: 3, resource: { buffer: this.pixels } },
],
});
dispatch(this.device, this.pipeline, bindGroup, this.imageSize / WORKGROUP_SIZE, this.imageSize / WORKGROUP_SIZE);
}
private textureFor(width: number, height: number): GPUTexture {
if (this.texture?.width === width && this.texture.height === height) return this.texture;
this.texture?.destroy();
this.texture = this.device.createTexture({
label: 'sam2 frame',
size: [width, height],
format: 'rgba8unorm',
usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,
});
return this.texture;
}
dispose(): void {
this.texture?.destroy();
this.pixels.destroy();
this.params.destroy();
}
}
/** Rewrites a [1, C, H, W] feature map as the [H*W, 1, C] tokens memory attention reads. */
export class TokenTransposer {
private readonly pipeline: GPUComputePipeline;
constructor(
private readonly device: GPUDevice,
private readonly geometry: Sam2Geometry,
) {
this.pipeline = createPipeline(device, transposeShader(geometry.featTokens), 'sam2 transpose');
}
run(channels: GPUBuffer, tokens: GPUBuffer): void {
const size = this.geometry.featTokens * HIDDEN_DIM * BYTES_PER_FLOAT;
const bindGroup = this.device.createBindGroup({
layout: this.pipeline.getBindGroupLayout(0),
entries: [
{ binding: 0, resource: { buffer: channels, size } },
{ binding: 1, resource: { buffer: tokens, size } },
],
});
dispatch(this.device, this.pipeline, bindGroup, this.geometry.featTokens / WORKGROUP_SIZE, HIDDEN_DIM / WORKGROUP_SIZE);
}
}