Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add keyframe index management and enhance video buffer handling

konstantin-paulus092625434650
diff --git a/apps/web/src/components/engine/decoders/frame-cache.ts b/apps/web/src/components/engine/decoders/frame-cache.ts
index f47461a4..7630b459 100644
--- a/apps/web/src/components/engine/decoders/frame-cache.ts
+++ b/apps/web/src/components/engine/decoders/frame-cache.ts
@@ -131,13 +131,18 @@ export class FrameCache {
return this.tiles.some(t => t.frameIndex === frameIndex);
}
+ /**
+ * Frame index of the cached frame closest to `frameIndex`, or `undefined` when
+ * nothing is cached within `tolerance`. Ties resolve to the earlier frame, so a
+ * playhead sitting between two cached frames never shows one that hasn't played yet.
+ */
public findNearest(frameIndex: number, tolerance: number): number | undefined {
let best: number | undefined;
let bestDistance = tolerance + 1;
for (const tile of this.tiles) {
const distance = Math.abs(tile.frameIndex - frameIndex);
- if (distance < bestDistance) {
+ if (distance < bestDistance || (best !== undefined && distance === bestDistance && tile.frameIndex < best)) {
bestDistance = distance;
best = tile.frameIndex;
}
diff --git a/apps/web/src/components/engine/decoders/index.ts b/apps/web/src/components/engine/decoders/index.ts
index 85b09456..a21ec116 100644
--- a/apps/web/src/components/engine/decoders/index.ts
+++ b/apps/web/src/components/engine/decoders/index.ts
@@ -7,6 +7,7 @@ export * from './audio-peaks';
export * from './caption';
export * from './html';
export * from './image';
+export * from './keyframe-index';
export * from './sequence';
export * from './shader';
export * from './time-stretcher';
diff --git a/apps/web/src/components/engine/decoders/keyframe-index.ts b/apps/web/src/components/engine/decoders/keyframe-index.ts
new file mode 100644
index 00000000..d046fa5f
--- /dev/null
+++ b/apps/web/src/components/engine/decoders/keyframe-index.ts
@@ -0,0 +1,109 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { EncodedPacketSink } from 'mediabunny';
+
+import type { InputVideoTrack } from 'mediabunny';
+
+/**
+ * Key packets walked between two yields to the event loop. Keeps an all-intra
+ * track (every packet is a keyframe) from starving the render loop.
+ */
+const YIELD_INTERVAL = 512;
+
+const indexCache = new Map<string, KeyframeIndex>();
+
+export function clearKeyframeIndexCache() {
+ indexCache.clear();
+}
+
+/**
+ * Returns the shared keyframe index for an asset, starting the walk on first
+ * access. Cheap to call repeatedly — one index is built per asset, not per decoder.
+ */
+export function getKeyframeIndex(assetId: string, track: InputVideoTrack): KeyframeIndex {
+ let index = indexCache.get(assetId);
+
+ if (!index) {
+ index = new KeyframeIndex(track);
+ indexCache.set(assetId, index);
+ }
+
+ return index;
+}
+
+/**
+ * Ascending presentation timestamps of every key packet in a video track, walked
+ * ahead of time so seeking resolves its seed keyframe with a binary search
+ * instead of a container lookup.
+ */
+export class KeyframeIndex {
+ private timestamps: number[] = [];
+ private complete = false;
+
+ public constructor(track: InputVideoTrack) {
+ this.build(track);
+ }
+
+ private async build(track: InputVideoTrack) {
+ try {
+ const sink = new EncodedPacketSink(track);
+
+ let packet = await sink.getFirstKeyPacket({ metadataOnly: true });
+ let walked = 0;
+
+ while (packet) {
+ this.timestamps.push(packet.timestamp);
+
+ // `getNextKeyPacket` resolves without I/O once the sample table is parsed,
+ // so awaiting it only queues a microtask and never lets the browser paint.
+ // A timer is a macrotask, which does.
+ if (++walked % YIELD_INTERVAL === 0) {
+ await new Promise((resolve) => setTimeout(resolve));
+ }
+
+ packet = await sink.getNextKeyPacket(packet, { metadataOnly: true });
+ }
+
+ this.complete = true;
+ } catch (e) {
+ // A disposed input (project closed, permission revoked) lands here; the
+ // index simply stays incomplete and callers fall back to a container lookup.
+ console.error('Error building keyframe index', e);
+ }
+ }
+
+ /**
+ * Timestamp of the last keyframe at or before `seconds`, or `null` when the
+ * index cannot answer: `seconds` precedes the first keyframe, or the walk has
+ * not reached that far yet and a closer keyframe may still show up.
+ */
+ public floor(seconds: number): number | null {
+ const timestamps = this.timestamps;
+ const length = timestamps.length;
+
+ if (length === 0 || seconds < timestamps[0]) {
+ return null;
+ }
+ if (!this.complete && seconds > timestamps[length - 1]) {
+ return null;
+ }
+
+ let low = 0;
+ let high = length - 1;
+
+ while (low < high) {
+ const mid = (low + high + 1) >> 1;
+ if (timestamps[mid] <= seconds) {
+ low = mid;
+ } else {
+ high = mid - 1;
+ }
+ }
+
+ return timestamps[low];
+ }
+}
diff --git a/apps/web/src/components/engine/decoders/video.ts b/apps/web/src/components/engine/decoders/video.ts
index b19be583..d94fc096 100644
--- a/apps/web/src/components/engine/decoders/video.ts
+++ b/apps/web/src/components/engine/decoders/video.ts
@@ -7,7 +7,9 @@ import { addComponent } from '../api/events';
import { getAssetFile } from '../api/assets';
import { assert } from '@/utils/common';
import { FrameCache } from './frame-cache';
+import { getKeyframeIndex } from './keyframe-index';
+import type { KeyframeIndex } from './keyframe-index';
import type { EngineWorld } from '../api/world';
import type { VideoAsset } from '../db';
@@ -32,6 +34,12 @@ const FORWARD_BIAS_FRAMES = 24;
*/
const DRAIN_FRAMES = 8;
+/**
+ * How far the preview may drift from the requested frame. A neighbour this close
+ * beats holding the last drawn frame while the exact one is still decoding.
+ */
+const DISPLAY_TOLERANCE_FRAMES = 4;
+
/**
* Total pixel budget of the preview frame cache
*/
@@ -62,6 +70,7 @@ export class VideoBuffer {
private currentFrame: number = -1;
private isDirty: boolean = true;
+ private keyframes: KeyframeIndex | null = null;
private lastFrameIndex: number = 0;
private seekGeneration = 0;
@@ -86,6 +95,8 @@ export class VideoBuffer {
assert(videoTrack, 'Video track not found');
+ this.keyframes = getKeyframeIndex(this.asset.id, videoTrack);
+
await this.queue.init(videoTrack);
this.cache.rotation = videoTrack.rotation;
@@ -212,14 +223,14 @@ export class VideoBuffer {
const live = !!(this.iterator && this.queue.isAlive && cursor);
let reuse = live && cursor!.timestamp < untilSecs;
- let keyPacket: EncodedPacket | null = null;
+ let keyTimestamp = this.keyframes?.floor(fromSecs) ?? null;
// If the cursor lags far behind the range start, skip forward to the nearest
// keyframe rather than decoding through the whole gap.
const biasSecs = FORWARD_BIAS_FRAMES / this.asset.frameRate;
if (reuse && cursor!.timestamp < fromSecs - biasSecs) {
- keyPacket = (await this.packetSink?.getKeyPacket(fromSecs)) ?? null;
- if (keyPacket && keyPacket.timestamp - cursor!.timestamp > biasSecs) {
+ keyTimestamp ??= (await this.packetSink?.getKeyPacket(fromSecs))?.timestamp ?? null;
+ if (keyTimestamp !== null && keyTimestamp - cursor!.timestamp > biasSecs) {
reuse = false; // jumping to the keyframe
}
}
@@ -227,7 +238,7 @@ export class VideoBuffer {
// we cant reuse the iterator, so we need to reseed the decoder
// and get create a new iterator
if (!reuse) {
- keyPacket ??= (await this.packetSink?.getKeyPacket(fromSecs)) ?? null;
+ const keyPacket = (await this.packetSink?.getKeyPacket(keyTimestamp ?? fromSecs)) ?? null;
if (!keyPacket) return;
await this.iterator?.return();
this.iterator = this.packetSink?.packets(keyPacket) ?? null;
@@ -252,7 +263,9 @@ export class VideoBuffer {
public toBitmap() {
if (this.isDirty && this.currentFrame >= 0) {
- const tile = this.cache.findTile(this.currentFrame);
+
+ const nearest = this.cache.findNearest(this.currentFrame, DISPLAY_TOLERANCE_FRAMES);
+ const tile = nearest === undefined ? undefined : this.cache.findTile(nearest);
const ctx = this.ctx;
if (tile && (this.canvas.width !== tile.width || this.canvas.height !== tile.height)) {
diff --git a/apps/web/src/context/engine.tsx b/apps/web/src/context/engine.tsx
index 826398e5..68364c9c 100644
--- a/apps/web/src/context/engine.tsx
+++ b/apps/web/src/context/engine.tsx
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createEngine, invalidateAssets } from '@/components/engine';
-import { clearAudioPeaksCache, clearAudioTrackCache, clearVideoTrackCache } from '@/components/engine/decoders';
+import { clearAudioPeaksCache, clearAudioTrackCache, clearKeyframeIndexCache, clearVideoTrackCache } from '@/components/engine/decoders';
import { clearImageThumbnailCache } from '@/components/engine/timeline/render/image';
import { markProjectOpened, setProjectThumbnail } from '@/components/engine/db';
import { captureCanvasThumbnail } from '@/components/engine/thumbnail';
@@ -143,6 +143,7 @@ export function EngineProvider(props: EngineProviderProps) {
clearAudioPeaksCache();
clearAudioTrackCache();
clearVideoTrackCache();
+ clearKeyframeIndexCache();
clearImageThumbnailCache();
for (const eid of getAllEntities(engine.world)) {