Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance VideoBuffer scrubbing and seeking functionality

konstantin-paulus3e3a4bdd6315
diff --git a/apps/web/src/components/engine/decoders/video.ts b/apps/web/src/components/engine/decoders/video.ts
index 5f95e960..0db4d0b0 100644
--- a/apps/web/src/components/engine/decoders/video.ts
+++ b/apps/web/src/components/engine/decoders/video.ts
@@ -50,6 +50,16 @@ const CACHE_PIXEL_BUDGET = 768 * 432 * 81; // 81 tiles at 768x432
*/
const MAX_TILE_PIXELS = 1280 * 720;
+/**
+ * Two seeks arriving closer together than this are treated as one continuous drag.
+ */
+const SCRUB_EVENT_WINDOW_MS = 250;
+
+/**
+ * Quiet period after the last scrub seek before the exact frame is resolved.
+ */
+const SCRUB_SETTLE_MS = 120;
+
const MIN_CACHE_COUNT = 30;
const MAX_CACHE_COUNT = 81;
@@ -72,11 +82,26 @@ export class VideoBuffer {
private isDirty: boolean = true;
private keyframes: KeyframeIndex | null = null;
+ /**
+ * Frame the preview is aiming to show. Tracks `currentFrame` except while scrubbing,
+ * where it points at the covering keyframe instead of the requested frame.
+ */
+ private displayFrame: number = -1;
+
+ /**
+ * Keyframes a scrub is waiting on. The preview keeps showing the last one that landed
+ * until the next arrives, so a drag never blanks out for the length of a decode.
+ * Holds several at once: decoder output trails submission by a handful of packets.
+ */
+ private readonly pendingScrub = new Set<number>();
+
private lastFrameIndex: number = 0;
private seekGeneration = 0;
private seekLock: Promise<void> = Promise.resolve();
private iterator: AsyncGenerator<EncodedPacket, void, unknown> | null = null;
private idleTimer: ReturnType<typeof setTimeout> | null = null;
+ private settleTimer: ReturnType<typeof setTimeout> | null = null;
+ private lastSeekAt: number = -Infinity;
public constructor(asset: VideoAsset) {
this.asset = asset;
@@ -114,10 +139,16 @@ export class VideoBuffer {
const frameIndex = this.secondsToFrames(timestampSeconds);
this.cache.insert(frame, frameIndex);
this.isDirty = true;
+
+ // A scrub keyframe takes over the preview the moment it lands, and only then:
+ // matching on the index keeps frames from an abandoned fill out of the display.
+ if (this.pendingScrub.delete(frameIndex)) {
+ this.displayFrame = frameIndex;
+ }
}
public seekTo(frame: number, frameRate: number): undefined {
- let targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);
+ const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);
const isEmpty = !this.packetSink;
const isCurrentFrame = targetFrame === this.currentFrame;
@@ -130,36 +161,126 @@ export class VideoBuffer {
}
const previousFrame = this.currentFrame;
- const forward = this.isBlockedFrame(targetFrame)
- ? targetFrame >= previousFrame
- : true;
-
+ const consecutive = performance.now() - this.lastSeekAt < SCRUB_EVENT_WINDOW_MS;
+ this.lastSeekAt = performance.now();
this.currentFrame = targetFrame;
this.isDirty = true;
this.touch();
+ // A run of large jumps is a drag, not a playback step. Walking each GOP to the
+ // exact frame costs more than the gap between pointer events, so every walk gets
+ // cancelled by the next one and nothing ever paints.
+ const jumped = Math.abs(targetFrame - previousFrame) > FORWARD_BIAS_FRAMES;
+
+ if (consecutive && jumped && this.scrubTo(targetFrame)) {
+ return;
+ }
+
+ this.exactSeekTo(targetFrame, previousFrame);
+ }
+
+ /**
+ * Paints the keyframe covering `targetFrame` rather than the frame itself: one packet,
+ * no walk through the GOP. Returns false when the keyframe index cannot place the
+ * target yet, leaving the caller to fall back to an exact seek.
+ */
+ private scrubTo(targetFrame: number): boolean {
+ const keyTimestamp = this.keyframes?.floor(this.framesToSeconds(targetFrame)) ?? null;
+
+ if (keyTimestamp === null) {
+ return false;
+ }
+
+ const keyFrame = this.secondsToFrames(keyTimestamp);
+ this.scheduleSettle();
+
+ // Supersedes any fill still running for a position we have already left.
+ const generation = ++this.seekGeneration;
+
+ // Consecutive positions inside one GOP resolve to a keyframe we already hold.
+ if (this.cache.has(keyFrame)) {
+ this.displayFrame = keyFrame;
+ return true;
+ }
+
+ this.pendingScrub.add(keyFrame);
+
+ this.seekLock = this.seekLock
+ .then(() => this.decodeKeyframe(keyTimestamp, generation))
+ .catch(() => { });
+
+ return true;
+ }
+
+ private async decodeKeyframe(keyTimestamp: number, generation: number): Promise<void> {
+ if (generation !== this.seekGeneration) return;
+
+ const keyPacket = await this.packetSink?.getKeyPacket(keyTimestamp);
+ if (!keyPacket || generation !== this.seekGeneration) return;
+
+ // Seed the iterator here as well, so the settle pass continues forward from this
+ // keyframe instead of reseeding the decoder a second time.
+ await this.iterator?.return();
+ const iterator = this.packetSink?.packets(keyPacket) ?? null;
+ this.iterator = iterator;
+ if (!iterator) return;
+
+ if (!this.queue.isAlive) {
+ this.queue.reseed();
+ }
+
+ const { value: packet } = await iterator.next();
+ if (packet) {
+ await this.queue.decode(packet);
+ }
+ }
+
+ /**
+ * Re-runs the seek for real once the drag stops.
+ */
+ private scheduleSettle() {
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ }
+
+ this.settleTimer = setTimeout(() => {
+ this.settleTimer = null;
+ if (this.mode !== 'alive' || this.errored) return;
+ this.exactSeekTo(this.currentFrame, this.currentFrame);
+ }, SCRUB_SETTLE_MS);
+ }
+
+ private exactSeekTo(targetFrame: number, previousFrame: number) {
+ this.displayFrame = targetFrame;
+ this.pendingScrub.clear();
+
+ const forward = this.isBlockedFrame(targetFrame)
+ ? targetFrame >= previousFrame
+ : true;
+
const generation = ++this.seekGeneration;
const [left, right] = this.computeWindow(targetFrame, forward);
this.cache.leftFrameIndex = left;
this.cache.rightFrameIndex = right;
+ let seed = targetFrame;
let run: Promise<void>
if (forward) {
- while (this.isBlockedFrame(targetFrame) && targetFrame <= this.cache.rightFrameIndex) {
- targetFrame++;
+ while (this.isBlockedFrame(seed) && seed <= this.cache.rightFrameIndex) {
+ seed++;
}
- run = this.seekLock.then(() => this.fillCache([targetFrame, this.cache.rightFrameIndex], generation));
+ run = this.seekLock.then(() => this.fillCache([seed, this.cache.rightFrameIndex], generation));
} else {
- while (this.cache.has(targetFrame) && targetFrame >= this.cache.leftFrameIndex) {
- targetFrame--;
+ while (this.cache.has(seed) && seed >= this.cache.leftFrameIndex) {
+ seed--;
}
// TODO: Find a better solution for DRAIN_FRAMES
- run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, targetFrame + DRAIN_FRAMES], generation));
+ run = this.seekLock.then(() => this.fillCache([this.cache.leftFrameIndex, seed + DRAIN_FRAMES], generation));
}
// Run after the previous seek finishes — never concurrently.
@@ -264,9 +385,9 @@ export class VideoBuffer {
}
public toBitmap() {
- if (this.isDirty && this.currentFrame >= 0) {
+ if (this.isDirty && this.displayFrame >= 0) {
- const nearest = this.cache.findNearest(this.currentFrame, DISPLAY_TOLERANCE_FRAMES);
+ const nearest = this.cache.findNearest(this.displayFrame, DISPLAY_TOLERANCE_FRAMES);
const tile = nearest === undefined ? undefined : this.cache.findTile(nearest);
const ctx = this.ctx;
@@ -316,6 +437,12 @@ export class VideoBuffer {
this.idleTimer = null;
}
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ this.settleTimer = null;
+ }
+
+ this.pendingScrub.clear();
this.cache.dispose();
this.queue.dispose();
this.iterator?.return();
@@ -331,6 +458,12 @@ export class VideoBuffer {
this.idleTimer = null;
}
+ if (this.settleTimer !== null) {
+ clearTimeout(this.settleTimer);
+ this.settleTimer = null;
+ }
+
+ this.pendingScrub.clear();
this.cache.dispose();
this.queue.dispose();
this.iterator?.return();