Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Fix generation timing and resolution

Bitti39e142338ed6
diff --git a/src/components/workspace/GeneratorView.test.tsx b/src/components/workspace/GeneratorView.test.tsx
index 119502f..0b74fee 100644
--- a/src/components/workspace/GeneratorView.test.tsx
+++ b/src/components/workspace/GeneratorView.test.tsx
@@ -20,6 +20,7 @@ const readyRuntime: VidfabStatus = {
state: "ready",
dllPath: "test",
version: "test",
+ platform: "CUDA 13",
detail: "Ready",
models: [],
};
@@ -126,7 +127,9 @@ describe("Generator scene controls", () => {
});
it("shows scene generation controls and sends their values in the render snapshot", async () => {
- const state = setup();
+ const initial = project();
+ initial.settings = { ...initial.settings, resolution: "416p", frameRate: 60 };
+ const state = setup(initial);
const generation = screen.getByRole("region", { name: "Generation" });
expect(within(generation).getByRole("spinbutton", { name: "Generation step count" })).toHaveValue(20);
expect(within(generation).getByRole("spinbutton", { name: "Generation seed" })).toHaveValue(-1);
@@ -137,7 +140,13 @@ describe("Generator scene controls", () => {
fireEvent.click(within(screen.getByRole("region", { name: "First scene" })).getByRole("button", { name: "Generate" }));
await waitFor(() => expect(state.latest().generationJobs[0].generationSnapshot).toEqual(expect.any(String)));
- expect(JSON.parse(state.latest().generationJobs[0].generationSnapshot!)).toEqual(expect.objectContaining({ steps: 28, seed: 9173 }));
+ expect(JSON.parse(state.latest().generationJobs[0].generationSnapshot!)).toEqual(expect.objectContaining({
+ steps: 28,
+ seed: 9173,
+ frames: 144,
+ canvasWidth: 736,
+ canvasHeight: 416,
+ }));
});
it("edits shot speech and sends it with the selected language in dialogue tags", async () => {
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index 66c0552..b3067f6 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -1,7 +1,8 @@
import { Plus, Sparkles, Square, WandSparkles } from "lucide-react";
import { invoke } from "@tauri-apps/api/core";
import { useEffect, useMemo, useRef, useState } from "react";
-import { actionReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, sceneDurationSeconds, sceneGenerationReferences, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableImageReferences, type GenerationJob, type ProjectConfig, type ProjectReference, type PromptSegment, type SceneShot } from "../../lib/project";
+import { actionReferenceIds, compileGenerationJobPrompt, compileGenerationJobSegments, createDraftGenerationJob, danglingReferenceTokens, GENERATION_FRAME_RATE, sceneDurationSeconds, sceneGenerationReferences, sceneGenerationSeed, sceneGenerationSnapshot, sceneGenerationSteps, sceneShots, SCENE_MAX_SECONDS, SCENE_MIN_SECONDS, projectItemPath, usableImageReferences, type GenerationJob, type ProjectConfig, type ProjectReference, type PromptSegment, type SceneShot } from "../../lib/project";
+import { generationDimensions } from "../../lib/export";
import { isTauri } from "../../lib/persistence";
import { cancelVidfabGeneration, enqueueVidfabGeneration, resolveVidfabPlan, type VidfabGenerationRequest, type VidfabStatus } from "../../lib/runtime";
import { SceneBoard, type GeneratorSelection } from "./SceneBoard";
@@ -231,6 +232,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
// compiled prompt, because vidfab.rs adds reference_paths sequentially — so
// array index 0 must be the asset the prompt calls <Picture 1>.
const bound = sceneGenerationReferences(job, configRef.current.references);
+ const canvas = generationDimensions(config.settings.resolution, config.settings.aspectRatio);
return {
jobId: job.id,
// Recompiled from current state so edits to a bound reference or a
@@ -239,10 +241,11 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
// the same string the compiled-prompt panel shows.
prompt: compileGenerationJobPrompt(job, bound),
// The scene's own length, not a fixed six seconds.
- frames: Math.round(sceneDurationSeconds(job) * config.settings.frameRate),
+ frames: Math.round(sceneDurationSeconds(job) * GENERATION_FRAME_RATE),
steps: sceneGenerationSteps(job),
seed: sceneGenerationSeed(job),
- aspectRatio: config.settings.aspectRatio,
+ canvasWidth: canvas.width,
+ canvasHeight: canvas.height,
referencePaths: usableImageReferences(bound)
.map((reference) => projectItemPath(folderPath, reference) ?? "")
.filter((path) => path.length > 0),
diff --git a/src/lib/export.test.ts b/src/lib/export.test.ts
index cf2e3d9..c8f1386 100644
--- a/src/lib/export.test.ts
+++ b/src/lib/export.test.ts
@@ -9,6 +9,7 @@ import {
fitRect,
formatBytes,
formatDuration,
+ generationDimensions,
outputDimensions,
resolutionLabel,
sourceTimeMsForFrame,
@@ -157,8 +158,8 @@ describe("output geometry", () => {
describe("defaults", () => {
it("starts from the project's own settings instead of inventing numbers", () => {
- const config = project([], [], { resolution: "4k", frameRate: 24 });
- expect(defaultExportSettings(config)).toEqual({ resolution: "4k", frameRate: 24, codec: "h264", quality: "balanced" });
+ const config = project([], [], { resolution: "4k", frameRate: 60 });
+ expect(defaultExportSettings(config)).toEqual({ resolution: "4k", frameRate: 60, codec: "h264", quality: "balanced" });
});
it("derives the bitrate from the frame size and rate", () => {
@@ -222,6 +223,12 @@ describe("the plan", () => {
});
});
+ it("expands legacy generation canvases to H3's 32-pixel grid without changing export pixels", () => {
+ expect(generationDimensions("1080p", "16:9")).toEqual({ width: 1920, height: 1088 });
+ expect(generationDimensions("720p", "9:16")).toEqual({ width: 736, height: 1280 });
+ expect(generationDimensions("768p", "4:5")).toEqual(outputDimensions("768p", "4:5"));
+ });
+
it("reveals wipe transitions in the selected direction", () => {
const base = clipVisualSettings(clip("a", "track-story", 0, 1_000));
expect(clipFrameStyle({ ...base, transition: { type: "wipe-left", durationMs: 1_000 } }, 250))
diff --git a/src/lib/export.ts b/src/lib/export.ts
index a16d5cc..05fb4fa 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -183,6 +183,17 @@ export function outputDimensions(resolution: Resolution, aspectRatio: AspectRati
return { width, height };
}
+/** The canvas sent to MiniMax H3. Current project sizes already satisfy the
+ * model's 32-pixel grid. Legacy export sizes remain exact on disk and at
+ * export, but their generation canvas is expanded to the nearest valid grid. */
+export function generationDimensions(resolution: Resolution, aspectRatio: AspectRatio): { width: number; height: number } {
+ const { width, height } = outputDimensions(resolution, aspectRatio);
+ return {
+ width: Math.ceil(width / 32) * 32,
+ height: Math.ceil(height / 32) * 32,
+ };
+}
+
/** A frame size in the only terms that mean anything to the person choosing it.
* Everywhere a resolution is SHOWN uses this rather than the id: "416p" names
* the short edge and leaves the other one a guess, and at 16:9 the guess is
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 1cb081a..489e58f 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -8,6 +8,9 @@ export const PROJECT_FILE_NAME = "polstudio.json";
export const LEGACY_PROJECT_FILE_NAMES = ["pols.json", "polstudio.project.json"] as const;
export const CURRENT_SCHEMA_VERSION = 1 as const;
export const DEFAULT_GENERATION_STEPS = 20;
+/** Vidfab's native generation clock. Project frame rate is an editing/export
+ * target; higher rates are produced later by interpolation and retiming. */
+export const GENERATION_FRAME_RATE = 24;
export const MAX_GENERATION_STEPS = 2_147_483_647;
export const RANDOM_GENERATION_SEED = -1;
@@ -775,7 +778,8 @@ export interface SceneGenerationInput {
frames: number;
steps: number;
seed: number;
- aspectRatio: string;
+ canvasWidth: number;
+ canvasHeight: number;
referencePaths: readonly string[];
}
@@ -793,7 +797,8 @@ export function sceneGenerationSnapshot(job: GenerationJob, input: SceneGenerati
frames: input.frames,
steps: input.steps,
seed: input.seed,
- aspectRatio: input.aspectRatio,
+ canvasWidth: input.canvasWidth,
+ canvasHeight: input.canvasHeight,
referencePaths: input.referencePaths,
shots,
});
diff --git a/src/lib/runtime.test.ts b/src/lib/runtime.test.ts
index e2438fc..d8e6b85 100644
--- a/src/lib/runtime.test.ts
+++ b/src/lib/runtime.test.ts
@@ -24,9 +24,12 @@ describe("deterministic browser runtime", () => {
expect(status.vidfab.state).toBe("demo");
const plan = await resolveVidfabPlan({
jobId: "job-plan", prompt: "three field prompt", frames: 180, steps: 50,
- seed: 1, aspectRatio: "16:9", referencePaths: [],
+ seed: 1, canvasWidth: 1920, canvasHeight: 1088, referencePaths: [],
}, record.config);
expect(plan.alignedFrames).toBe(192);
+ expect(plan.canvasWidth).toBe(1920);
+ expect(plan.canvasHeight).toBe(1088);
+ expect(plan.durationSeconds).toBe(8);
expect(plan.boundary).toContain("No weights");
});
diff --git a/src/lib/runtime.ts b/src/lib/runtime.ts
index a315085..b4599ed 100644
--- a/src/lib/runtime.ts
+++ b/src/lib/runtime.ts
@@ -1,6 +1,6 @@
import { invoke } from "@tauri-apps/api/core";
import { isTauri, saveProject } from "./persistence";
-import { compileMiniMaxH3Prompt, parseProjectConfig, type GenerationJob, type ProjectConfig, type ProjectRecord } from "./project";
+import { compileMiniMaxH3Prompt, GENERATION_FRAME_RATE, parseProjectConfig, type GenerationJob, type ProjectConfig, type ProjectRecord } from "./project";
import {
agentEndpointProviderSettings, endpointProviderSetting, engineProviderSetting,
loadEngineSettings, withAgentEndpointSettings, withEngineSettings,
@@ -60,7 +60,8 @@ export interface VidfabGenerationRequest {
frames: number;
steps: number;
seed: number;
- aspectRatio: string;
+ canvasWidth: number;
+ canvasHeight: number;
referencePaths: string[];
}
@@ -159,18 +160,16 @@ export async function cancelAgentTurn(requestId: string): Promise<boolean> {
export async function resolveVidfabPlan(request: VidfabGenerationRequest, config: ProjectConfig): Promise<ResolvedPlan> {
if (isTauri()) return invoke<ResolvedPlan>("resolve_vidfab_plan", { request, config: withEngineSettings(config) });
- const [ratioWidth, ratioHeight] = request.aspectRatio.split(":").map(Number);
- const vertical = ratioHeight > ratioWidth;
const alignedFrames = Math.ceil(Math.max(5, request.frames - 5) / 17) * 17 + 5;
return {
- canvasWidth: vertical ? 768 : request.aspectRatio === "1:1" ? 768 : 1344,
- canvasHeight: vertical ? 1344 : 768,
+ canvasWidth: request.canvasWidth,
+ canvasHeight: request.canvasHeight,
alignedFrames,
- durationSeconds: alignedFrames / config.settings.frameRate,
+ durationSeconds: alignedFrames / GENERATION_FRAME_RATE,
modelEvaluations: request.steps - 1,
sequenceRowsWithoutText: 0,
latentFrames: Math.ceil(alignedFrames / 17), latentWidth: 0, latentHeight: 0,
- description: `Browser demo plan: ${alignedFrames} aligned frames at ${config.settings.frameRate} fps.`,
+ description: `Browser demo plan: ${alignedFrames} aligned frames at ${GENERATION_FRAME_RATE} fps.`,
boundary: "Deterministic plan only. No weights were opened and no media file was created.",
};
}