Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Accept long video references with capped trims

Bittibf3ac9bd6252
diff --git a/src/lib/referenceVideo.import.test.ts b/src/lib/referenceVideo.import.test.ts
new file mode 100644
index 0000000..2d31995
--- /dev/null
+++ b/src/lib/referenceVideo.import.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from "vitest";
+import { ArrayBufferTarget, Muxer } from "mp4-muxer";
+import { readMediaFileBytes } from "./persistence";
+import { inspectReferenceVideo } from "./referenceVideo";
+
+vi.mock("./persistence", () => ({ readMediaFileBytes: vi.fn() }));
+
+describe("importing long MP4 references", () => {
+ it.each(["in-memory", "fragmented"] as const)("uses the first 15 seconds of a real %s MP4 container", async (fastStart) => {
+ const target = new ArrayBufferTarget();
+ const muxer = new Muxer({ target, video: { codec: "avc", width: 320, height: 180, frameRate: 30 }, fastStart });
+ const sps = [0x67, 0x42, 0x00, 0x1f, 0xda, 0x02, 0x80, 0xf6, 0x80];
+ const pps = [0x68, 0xce, 0x3c, 0x80];
+ const description = new Uint8Array([1, 0x42, 0x00, 0x1f, 0xff, 0xe1, 0, sps.length, ...sps, 1, 0, pps.length, ...pps]);
+ for (let i = 0; i < 900; i++) {
+ muxer.addVideoChunkRaw(new Uint8Array([0, 0, 0, 3, 0x65, i % 256, 0]), i % 30 === 0 ? "key" : "delta",
+ Math.round(i * 1_000_000 / 30), Math.round(1_000_000 / 30), i === 0 ? { decoderConfig: { codec: "avc1.42001f", codedWidth: 320, codedHeight: 180, description } } : undefined);
+ }
+ muxer.finalize();
+ vi.mocked(readMediaFileBytes).mockResolvedValue(target.buffer);
+ await expect(inspectReferenceVideo("C:/project", "C:/30-seconds.mp4")).resolves.toEqual({ durationSeconds: 15, includeAudio: false });
+ });
+});
diff --git a/src/lib/referenceVideo.test.ts b/src/lib/referenceVideo.test.ts
index c9f3833..3766958 100644
--- a/src/lib/referenceVideo.test.ts
+++ b/src/lib/referenceVideo.test.ts
@@ -1,7 +1,7 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { invoke } from "@tauri-apps/api/core";
import { demux, type DemuxedSource } from "./exportPipeline";
-import { prepareReferenceVideos, referenceSoundtrack, referenceVideoRange, uploadReferenceVideoFrames } from "./referenceVideo";
+import { inspectReferenceVideo, prepareReferenceVideos, referenceSoundtrack, referenceVideoRange, sourceVideoDuration, uploadReferenceVideoFrames } from "./referenceVideo";
import { createProjectConfig } from "./project";
vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
@@ -16,6 +16,16 @@ const source: DemuxedSource = { config: { codec: "avc1" }, durationSeconds: 4, h
timestampUs, durationUs: 500_000, key: timestampUs === 0, data: new Uint8Array(1),
})) };
+it.each([0, NaN, .5, 120, undefined])("imports long sources with header duration %s using a capped selection", async (durationSeconds) => {
+ const long = { ...source, durationSeconds, samples: [0, 59_000_000].map((timestampUs) => ({
+ timestampUs, durationUs: 1_000_000, key: true, data: new Uint8Array(1),
+ })) };
+ vi.mocked(demux).mockResolvedValue(long);
+ expect(sourceVideoDuration(long)).toBe(60);
+ await expect(inspectReferenceVideo("C:/project", "C:/long.mp4")).resolves.toEqual({ durationSeconds: 15, includeAudio: false });
+ expect(referenceVideoRange({ ...input, startSeconds: 45, durationSeconds: 15 }, sourceVideoDuration(long))).toEqual({ start: 45, duration: 15 });
+});
+
function decoderMocks() {
const frames: Array<{ close: ReturnType<typeof vi.fn> }> = [];
const make = (timestamp: number) => {
diff --git a/src/lib/referenceVideo.ts b/src/lib/referenceVideo.ts
index 33c9213..e2344da 100644
--- a/src/lib/referenceVideo.ts
+++ b/src/lib/referenceVideo.ts
@@ -3,6 +3,7 @@ import { demux, type DemuxedSource } from "./exportPipeline";
import { readMediaFileBytes } from "./persistence";
import type { ProjectConfig } from "./project";
import type { SlopfabGenerationRequest } from "./runtime";
+import { initialReferenceVideoTrim } from "./referenceVideoTrim";
type VideoInput = NonNullable<SlopfabGenerationRequest["referenceVideos"]>[number];
const cancelledError = () => new Error("Video reference preparation cancelled.");
@@ -22,14 +23,18 @@ export function referenceVideoRange(input: VideoInput, sourceDuration: number) {
export function sourceVideoDuration(source: DemuxedSource): number {
const first = source.samples.reduce((value, sample) => Math.min(value, sample.timestampUs), Infinity);
const last = source.samples.reduce((value, sample) => Math.max(value, sample.timestampUs + sample.durationUs), -Infinity);
- return source.durationSeconds ?? (last - first) / 1_000_000;
+ const sampledDuration = (last - first) / 1_000_000;
+ // Fragmented MP4s can have a zero/placeholder track-header duration even
+ // though their fragments contain a full video. Preparation already uses this
+ // same sample origin, so use the actual presentation span for trim bounds.
+ return Number.isFinite(sampledDuration) && sampledDuration > 0 ? sampledDuration : source.durationSeconds ?? NaN;
}
export async function inspectReferenceVideo(folderPath: string, sourcePath: string) {
const source = await demux(await readMediaFileBytes(folderPath, { sourcePath }), "Video reference");
const duration = sourceVideoDuration(source);
- referenceVideoRange({ name: "Video reference", startSeconds: 0, includeAudio: true }, duration);
- return { durationSeconds: Math.min(15, duration), includeAudio: Boolean(source.hasAudio) };
+ const { durationSeconds } = initialReferenceVideoTrim(duration);
+ return { durationSeconds, includeAudio: Boolean(source.hasAudio) };
}
async function referenceDecoderConfig(config: VideoDecoderConfig): Promise<VideoDecoderConfig> {
diff --git a/src/lib/referenceVideoTrim.test.ts b/src/lib/referenceVideoTrim.test.ts
new file mode 100644
index 0000000..928289d
--- /dev/null
+++ b/src/lib/referenceVideoTrim.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { editReferenceVideoTrim, initialReferenceVideoTrim, normalizeReferenceVideoTrim } from "./referenceVideoTrim";
+
+describe("video reference selection", () => {
+ it("starts with the first 15 seconds of a long source and the whole of a shorter source", () => {
+ expect(initialReferenceVideoTrim(7200)).toEqual({ startSeconds: 0, durationSeconds: 15 });
+ expect(initialReferenceVideoTrim(7.25)).toEqual({ startSeconds: 0, durationSeconds: 7.25 });
+ expect(() => initialReferenceVideoTrim(1.9)).toThrow("at least 2");
+ });
+ it("moves a selection without changing its length or exceeding the source", () => {
+ const original = initialReferenceVideoTrim(120);
+ expect(editReferenceVideoTrim(original, 120, "move", 60)).toEqual({ startSeconds: 60, durationSeconds: 15 });
+ expect(editReferenceVideoTrim(original, 120, "move", 119)).toEqual({ startSeconds: 105, durationSeconds: 15 });
+ expect(editReferenceVideoTrim(original, 120, "move", -1).startSeconds).toBe(0);
+ });
+ it("resizes either edge while retaining the opposite edge and the 2–15 second bounds", () => {
+ const original = { startSeconds: 30, durationSeconds: 10 };
+ expect(editReferenceVideoTrim(original, 120, "start", 35)).toEqual({ startSeconds: 35, durationSeconds: 5 });
+ expect(editReferenceVideoTrim(original, 120, "start", 0)).toEqual({ startSeconds: 25, durationSeconds: 15 });
+ expect(editReferenceVideoTrim(original, 120, "end", 31)).toEqual({ startSeconds: 30, durationSeconds: 2 });
+ expect(editReferenceVideoTrim(original, 120, "end", 90)).toEqual({ startSeconds: 30, durationSeconds: 15 });
+ expect(editReferenceVideoTrim({ startSeconds: 117, durationSeconds: 3 }, 120, "duration", 15).durationSeconds).toBe(3);
+ });
+ it("clamps older saved selections after a source is replaced by a shorter video", () => {
+ expect(normalizeReferenceVideoTrim({ startSeconds: 50, durationSeconds: 15 }, 8)).toEqual({ startSeconds: 0, durationSeconds: 8 });
+ });
+ it("keeps fractional edits within schema bounds", () => {
+ for (let i = 0; i < 200; i++) {
+ const trim = editReferenceVideoTrim({ startSeconds: i / 10, durationSeconds: 15 }, 60, "start", 0);
+ expect(trim.durationSeconds).toBeGreaterThanOrEqual(2);
+ expect(trim.durationSeconds).toBeLessThanOrEqual(15);
+ expect(trim.startSeconds + trim.durationSeconds).toBeLessThanOrEqual(60);
+ }
+ });
+});
diff --git a/src/lib/referenceVideoTrim.ts b/src/lib/referenceVideoTrim.ts
new file mode 100644
index 0000000..12bf1b9
--- /dev/null
+++ b/src/lib/referenceVideoTrim.ts
@@ -0,0 +1,38 @@
+export const MIN_REFERENCE_SECONDS = 2;
+export const MAX_REFERENCE_SECONDS = 15;
+export type ReferenceVideoTrim = { startSeconds: number; durationSeconds: number };
+export type TrimAction = "move" | "start" | "end" | "duration";
+const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value));
+
+export function initialReferenceVideoTrim(sourceDuration: number): ReferenceVideoTrim {
+ if (!Number.isFinite(sourceDuration) || sourceDuration < MIN_REFERENCE_SECONDS) {
+ throw new Error("Choose a source video lasting at least 2 seconds.");
+ }
+ return { startSeconds: 0, durationSeconds: Math.min(MAX_REFERENCE_SECONDS, sourceDuration) };
+}
+
+export function normalizeReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number): ReferenceVideoTrim {
+ const initial = initialReferenceVideoTrim(sourceDuration);
+ const durationSeconds = clamp(Number.isFinite(trim.durationSeconds) ? trim.durationSeconds : initial.durationSeconds, MIN_REFERENCE_SECONDS, initial.durationSeconds);
+ return { startSeconds: clamp(Number.isFinite(trim.startSeconds) ? trim.startSeconds : 0, 0, sourceDuration - durationSeconds), durationSeconds };
+}
+
+/** Moving preserves length; dragging an edge preserves the opposite edge. */
+export function editReferenceVideoTrim(trim: ReferenceVideoTrim, sourceDuration: number, action: TrimAction, value: number): ReferenceVideoTrim {
+ const current = normalizeReferenceVideoTrim(trim, sourceDuration);
+ if (!Number.isFinite(value)) return current;
+ const end = current.startSeconds + current.durationSeconds;
+ if (action === "move") return { ...current, startSeconds: clamp(value, 0, sourceDuration - current.durationSeconds) };
+ if (action === "start") {
+ const startSeconds = clamp(value, Math.max(0, end - MAX_REFERENCE_SECONDS), end - MIN_REFERENCE_SECONDS);
+ return { startSeconds, durationSeconds: clamp(end - startSeconds, MIN_REFERENCE_SECONDS, MAX_REFERENCE_SECONDS) };
+ }
+ const durationSeconds = clamp(action === "end" ? value - current.startSeconds : value,
+ MIN_REFERENCE_SECONDS, Math.min(MAX_REFERENCE_SECONDS, sourceDuration - current.startSeconds));
+ return { ...current, durationSeconds };
+}
+
+export function referenceTime(seconds: number): string {
+ const tenths = Math.round(seconds * 10);
+ return `${Math.floor(tenths / 600)}:${String(Math.floor(tenths / 10) % 60).padStart(2, "0")}.${tenths % 10}`;
+}