The VS Code of video editing, built for agents.
TypeScript
Add keyframe index management and enhance video buffer handling
konstantin-paulus092625434650
diff --git a/apps/web/src/components/engine/decoders/frame-cache.ts b/apps/web/src/components/engine/decoders/frame-cache.tsindex f47461a4..7630b459 100644--- a/apps/web/src/components/engine/decoders/frame-cache.ts+++ b/apps/web/src/components/engine/decoders/frame-cache.ts@@ -131,13 +131,18 @@ export class FrameCache {return this.tiles.some(t => t.frameIndex === frameIndex);}+ /**+ * Frame index of the cached frame closest to `frameIndex`, or `undefined` when+ * nothing is cached within `tolerance`. Ties resolve to the earlier frame, so a+ * playhead sitting between two cached frames never shows one that hasn't played yet.+ */public findNearest(frameIndex: number, tolerance: number): number | undefined {let best: number | undefined;let bestDistance = tolerance + 1;for (const tile of this.tiles) {const distance = Math.abs(tile.frameIndex - frameIndex);- if (distance < bestDistance) {+ if (distance < bestDistance || (best !== undefined && distance === bestDistance && tile.frameIndex < best)) {bestDistance = distance;best = tile.frameIndex;}diff --git a/apps/web/src/components/engine/decoders/index.ts b/apps/web/src/components/engine/decoders/index.tsindex 85b09456..a21ec116 100644--- a/apps/web/src/components/engine/decoders/index.ts+++ b/apps/web/src/components/engine/decoders/index.ts@@ -7,6 +7,7 @@ export * from './audio-peaks';export * from './caption';export * from './html';export * from './image';+export * from './keyframe-index';export * from './sequence';export * from './shader';export * from './time-stretcher';diff --git a/apps/web/src/components/engine/decoders/keyframe-index.ts b/apps/web/src/components/engine/decoders/keyframe-index.tsnew file mode 100644index 00000000..d046fa5f--- /dev/null+++ b/apps/web/src/components/engine/decoders/keyframe-index.ts@@ -0,0 +1,109 @@+/**+ * This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.+ */++import { EncodedPacketSink } from 'mediabunny';++import type { InputVideoTrack } from 'mediabunny';++/**+ * Key packets walked between two yields to the event loop. Keeps an all-intra+ * track (every packet is a keyframe) from starving the render loop.+ */+const YIELD_INTERVAL = 512;++const indexCache = new Map<string, KeyframeIndex>();++export function clearKeyframeIndexCache() {+ indexCache.clear();+}++/**+ * Returns the shared keyframe index for an asset, starting the walk on first+ * access. Cheap to call repeatedly — one index is built per asset, not per decoder.+ */+export function getKeyframeIndex(assetId: string, track: InputVideoTrack): KeyframeIndex {+ let index = indexCache.get(assetId);++ if (!index) {+ index = new KeyframeIndex(track);+ indexCache.set(assetId, index);+ }++ return index;+}++/**+ * Ascending presentation timestamps of every key packet in a video track, walked+ * ahead of time so seeking resolves its seed keyframe with a binary search+ * instead of a container lookup.+ */+export class KeyframeIndex {+ private timestamps: number[] = [];+ private complete = false;++ public constructor(track: InputVideoTrack) {+ this.build(track);+ }++ private async build(track: InputVideoTrack) {+ try {+ const sink = new EncodedPacketSink(track);++ let packet = await sink.getFirstKeyPacket({ metadataOnly: true });+ let walked = 0;++ while (packet) {+ this.timestamps.push(packet.timestamp);++ // `getNextKeyPacket` resolves without I/O once the sample table is parsed,+ // so awaiting it only queues a microtask and never lets the browser paint.+ // A timer is a macrotask, which does.+ if (++walked % YIELD_INTERVAL === 0) {+ await new Promise((resolve) => setTimeout(resolve));+ }++ packet = await sink.getNextKeyPacket(packet, { metadataOnly: true });+ }++ this.complete = true;+ } catch (e) {+ // A disposed input (project closed, permission revoked) lands here; the+ // index simply stays incomplete and callers fall back to a container lookup.+ console.error('Error building keyframe index', e);+ }+ }++ /**+ * Timestamp of the last keyframe at or before `seconds`, or `null` when the+ * index cannot answer: `seconds` precedes the first keyframe, or the walk has+ * not reached that far yet and a closer keyframe may still show up.+ */+ public floor(seconds: number): number | null {+ const timestamps = this.timestamps;+ const length = timestamps.length;++ if (length === 0 || seconds < timestamps[0]) {+ return null;+ }+ if (!this.complete && seconds > timestamps[length - 1]) {+ return null;+ }++ let low = 0;+ let high = length - 1;++ while (low < high) {+ const mid = (low + high + 1) >> 1;+ if (timestamps[mid] <= seconds) {+ low = mid;+ } else {+ high = mid - 1;+ }+ }++ return timestamps[low];+ }+}diff --git a/apps/web/src/components/engine/decoders/video.ts b/apps/web/src/components/engine/decoders/video.tsindex b19be583..d94fc096 100644--- a/apps/web/src/components/engine/decoders/video.ts+++ b/apps/web/src/components/engine/decoders/video.ts@@ -7,7 +7,9 @@ import { addComponent } from '../api/events';import { getAssetFile } from '../api/assets';import { assert } from '@/utils/common';import { FrameCache } from './frame-cache';+import { getKeyframeIndex } from './keyframe-index';+import type { KeyframeIndex } from './keyframe-index';import type { EngineWorld } from '../api/world';import type { VideoAsset } from '../db';@@ -32,6 +34,12 @@ const FORWARD_BIAS_FRAMES = 24;*/const DRAIN_FRAMES = 8;+/**+ * How far the preview may drift from the requested frame. A neighbour this close+ * beats holding the last drawn frame while the exact one is still decoding.+ */+const DISPLAY_TOLERANCE_FRAMES = 4;+/*** Total pixel budget of the preview frame cache*/@@ -62,6 +70,7 @@ export class VideoBuffer {private currentFrame: number = -1;private isDirty: boolean = true;+ private keyframes: KeyframeIndex | null = null;private lastFrameIndex: number = 0;private seekGeneration = 0;@@ -86,6 +95,8 @@ export class VideoBuffer {assert(videoTrack, 'Video track not found');+ this.keyframes = getKeyframeIndex(this.asset.id, videoTrack);+await this.queue.init(videoTrack);this.cache.rotation = videoTrack.rotation;@@ -212,14 +223,14 @@ export class VideoBuffer {const live = !!(this.iterator && this.queue.isAlive && cursor);let reuse = live && cursor!.timestamp < untilSecs;- let keyPacket: EncodedPacket | null = null;+ let keyTimestamp = this.keyframes?.floor(fromSecs) ?? null;// If the cursor lags far behind the range start, skip forward to the nearest// keyframe rather than decoding through the whole gap.const biasSecs = FORWARD_BIAS_FRAMES / this.asset.frameRate;if (reuse && cursor!.timestamp < fromSecs - biasSecs) {- keyPacket = (await this.packetSink?.getKeyPacket(fromSecs)) ?? null;- if (keyPacket && keyPacket.timestamp - cursor!.timestamp > biasSecs) {+ keyTimestamp ??= (await this.packetSink?.getKeyPacket(fromSecs))?.timestamp ?? null;+ if (keyTimestamp !== null && keyTimestamp - cursor!.timestamp > biasSecs) {reuse = false; // jumping to the keyframe}}@@ -227,7 +238,7 @@ export class VideoBuffer {// we cant reuse the iterator, so we need to reseed the decoder// and get create a new iteratorif (!reuse) {- keyPacket ??= (await this.packetSink?.getKeyPacket(fromSecs)) ?? null;+ const keyPacket = (await this.packetSink?.getKeyPacket(keyTimestamp ?? fromSecs)) ?? null;if (!keyPacket) return;await this.iterator?.return();this.iterator = this.packetSink?.packets(keyPacket) ?? null;@@ -252,7 +263,9 @@ export class VideoBuffer {public toBitmap() {if (this.isDirty && this.currentFrame >= 0) {- const tile = this.cache.findTile(this.currentFrame);++ const nearest = this.cache.findNearest(this.currentFrame, DISPLAY_TOLERANCE_FRAMES);+ const tile = nearest === undefined ? undefined : this.cache.findTile(nearest);const ctx = this.ctx;if (tile && (this.canvas.width !== tile.width || this.canvas.height !== tile.height)) {diff --git a/apps/web/src/context/engine.tsx b/apps/web/src/context/engine.tsxindex 826398e5..68364c9c 100644--- a/apps/web/src/context/engine.tsx+++ b/apps/web/src/context/engine.tsx@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createEngine, invalidateAssets } from '@/components/engine';-import { clearAudioPeaksCache, clearAudioTrackCache, clearVideoTrackCache } from '@/components/engine/decoders';+import { clearAudioPeaksCache, clearAudioTrackCache, clearKeyframeIndexCache, clearVideoTrackCache } from '@/components/engine/decoders';import { clearImageThumbnailCache } from '@/components/engine/timeline/render/image';import { markProjectOpened, setProjectThumbnail } from '@/components/engine/db';import { captureCanvasThumbnail } from '@/components/engine/thumbnail';@@ -143,6 +143,7 @@ export function EngineProvider(props: EngineProviderProps) {clearAudioPeaksCache();clearAudioTrackCache();clearVideoTrackCache();+ clearKeyframeIndexCache();clearImageThumbnailCache();for (const eid of getAllEntities(engine.world)) {