Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance Support for Mask Assets Across Components

konstantin-paulus22d32d9c2260
diff --git a/apps/web/src/components/timeline/layers/node.tsx b/apps/web/src/components/timeline/layers/node.tsx
index 0a6db331..367e3504 100644
--- a/apps/web/src/components/timeline/layers/node.tsx
+++ b/apps/web/src/components/timeline/layers/node.tsx
@@ -364,6 +364,7 @@ function getLayerIcon(world: World, layer: TimelineNode) {
return "image-small";
case 'VIDEO':
case 'SEQUENCE':
+ case 'MASK':
return "video-small";
case 'AUDIO':
return "audio-small";
diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsx
index 34a39e6f..fdc8fea2 100644
--- a/apps/web/src/engine/insert-asset.tsx
+++ b/apps/web/src/engine/insert-asset.tsx
@@ -47,6 +47,7 @@ export function insertAsset(world: World, asset: Asset, options: InsertAssetOpti
switch (asset.type) {
case 'VIDEO':
case 'SEQUENCE':
+ case 'MASK':
return (
<Rect name={name} keepAspectRatio {...position} {...size} {...timing}>
<VideoPaint src={src} />
@@ -76,6 +77,7 @@ function sizeOf(asset: Asset): { width: number; height: number } | undefined {
case 'VIDEO':
case 'IMAGE':
case 'SEQUENCE':
+ case 'MASK':
return { width: Math.round(asset.width), height: Math.round(asset.height) };
case 'AUDIO':
return { ...AUDIO_SIZE };
diff --git a/apps/web/src/engine/timeline/media.ts b/apps/web/src/engine/timeline/media.ts
index 24f4ec6e..fbca991b 100644
--- a/apps/web/src/engine/timeline/media.ts
+++ b/apps/web/src/engine/timeline/media.ts
@@ -6,9 +6,10 @@
* The pictures the timeline draws inside clips, in the two kinds the two
* kinds of clip need.
*
- * A still — an image, or a frames directory shown by its first frame — is one
- * picture tiled along the clip, so it is decoded once per asset at whatever
- * size the row is currently drawn at, and kept until the row changes size.
+ * A still — an image, a frames directory shown by its first frame, or a mask
+ * shown by its first matte — is one picture tiled along the clip, so it is
+ * decoded once per asset at whatever size the row is currently drawn at, and
+ * kept until the row changes size.
*
* A video is a strip of its own frames, which is a decode per tile, so it
* comes in two layers like the peaks do. The asset layer decodes a handful of
@@ -24,6 +25,7 @@
import { CanvasSink } from 'mediabunny';
import { getAssetFile, getVideoTrack, secondsToFrames } from '@diffusionstudio/runtime';
+import { deriveThumbnail } from '@diffusionstudio/assets';
import { MAX_CLIP_HEIGHT } from './config';
@@ -119,12 +121,25 @@ export function resolveStill(asset: Asset, width: number): Still | null {
return existing ?? null;
}
+/**
+ * What a still is decoded from: the file itself, or for a mask — a file of
+ * logits, not a picture — its matte rendered as one.
+ */
+async function stillSource(asset: Asset, width: number): Promise<Blob> {
+ const file = await getAssetFile(asset);
+ if (asset.type !== 'MASK') return file;
+
+ const thumbnail = await deriveThumbnail(file, asset.mimeType, Math.ceil(width * window.devicePixelRatio));
+ if (!thumbnail) throw new Error('Could not render the mask');
+ return thumbnail;
+}
+
async function decodeStill(asset: Asset, width: number, hash: string): Promise<void> {
if (stillsDecoding.has(asset.id)) return;
stillsDecoding.add(asset.id);
try {
- const bitmap = await createImageBitmap(await getAssetFile(asset));
+ const bitmap = await createImageBitmap(await stillSource(asset, width));
try {
// Big enough for the widest tile and the tallest row it could be
diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.ts
index 50f5b871..6e9e2c02 100644
--- a/apps/web/src/engine/timeline/render/clip.ts
+++ b/apps/web/src/engine/timeline/render/clip.ts
@@ -197,6 +197,7 @@ function renderContent(
switch (asset?.type) {
case 'IMAGE':
case 'SEQUENCE':
+ case 'MASK':
renderStillThumbnails(world, scene, surface, entity, asset, row);
return;
diff --git a/apps/web/src/engine/timeline/style.ts b/apps/web/src/engine/timeline/style.ts
index b4fabc92..8e6c8639 100644
--- a/apps/web/src/engine/timeline/style.ts
+++ b/apps/web/src/engine/timeline/style.ts
@@ -41,6 +41,7 @@ export function getClipStyle(entity: Entity, asset: Asset | null, errored = fals
switch (asset?.type) {
case 'VIDEO':
case 'SEQUENCE':
+ case 'MASK':
return COLORS.clip.video;
case 'IMAGE':
return COLORS.clip.image;
@@ -76,6 +77,7 @@ export function getClipFallbackName(world: World, entity: Entity): string {
switch (getClipAsset(world, entity)?.type) {
case 'VIDEO':
case 'SEQUENCE':
+ case 'MASK':
return 'Video';
case 'IMAGE':
return 'Image';
diff --git a/apps/web/src/utils/common.ts b/apps/web/src/utils/common.ts
index c26b5aba..fdc9ec1a 100644
--- a/apps/web/src/utils/common.ts
+++ b/apps/web/src/utils/common.ts
@@ -76,7 +76,7 @@ export class AsyncMutex {
* @returns The formatted duration string, or null if the asset is not a video.
*/
export function formatAssetDuration(asset: Pick<Asset, 'type'> & { duration?: number } | null | undefined): string | null {
- if (asset?.type != "VIDEO" && asset?.type != "AUDIO" && asset?.type != "SEQUENCE") {
+ if (asset?.type != "VIDEO" && asset?.type != "AUDIO" && asset?.type != "SEQUENCE" && asset?.type != "MASK") {
return null;
}