AI media harness for planning, generating, and editing locally
TypeScript
Improve reference icon rendering
Bittice9278fb147f
diff --git a/README.md b/README.mdindex ad36819..7aa1231 100644--- a/README.md+++ b/README.md@@ -64,14 +64,15 @@ The web fallback seeds three deterministic showcase projects on first launch. Re### Reference iconsCustom references with a prompt and no existing artwork automatically receive-256×256 JPG icons through SlopFab's dedicated still-image mode. Category-specific+256×256 JPG icons, generated at 768×768 with 20 steps through SlopFab's dedicated+still-image mode, then downscaled and encoded at JPEG quality 95. Category-specificcompositions include the user's prompt. All icon work appears in one Work Queueentry, and waiting video generations run before the next icon. Generated iconsare saved under `references/icons/` in the project and are not sent to the videoengine as image attachments. Built-in presets are added as new references fromthe Add a reference dialog.-`npm run reference-icons` renders every missing character, product, location, and style preset with MiniMax H3 at 30 steps. Its DLL backend uses the dedicated still-image mode and writes a high-quality 256×256 JPEG without creating an intermediate video or audio file. Character entries use controlled headshot lighting; product and location entries use a complete-subject view; style entries use a representative composition with lighting and rendering tailored to the named style.+`npm run reference-icons` renders every missing character, product, location, and style preset with MiniMax H3 at 768×768 and 20 steps. Its DLL backend uses the dedicated still-image mode, downsamples each 3×3 pixel block to produce a 256×256 icon, and encodes it at JPEG quality 95 without creating an intermediate video or audio file. Character entries use controlled headshot lighting; product and location entries use a complete-subject view; style entries use a representative composition with lighting and rendering tailored to the named style.SlopFab, Cargo, and FFmpeg must be installed. The script uses the development paths under `D:\Projects\slopfab` by default; `SLOPFAB_EXE`, `SLOPFAB_TRANSFORMER`, `SLOPFAB_TEXT_ENCODER`, `SLOPFAB_VIDEO_VAE`, `SLOPFAB_AUDIO_VAE`, `SLOPFAB_BACKEND`, `CARGO_EXE`, and `FFMPEG_EXE` can override them. Use `node scripts/generate-reference-icons.mjs --id=<preset-id> --force` to regenerate one entry.diff --git a/scripts/generate-reference-icons.mjs b/scripts/generate-reference-icons.mjsindex 02facfe..3fb1ff3 100644--- a/scripts/generate-reference-icons.mjs+++ b/scripts/generate-reference-icons.mjs@@ -172,11 +172,11 @@ async function generate(entry, temporaryDirectory, inferenceBackend) {"--prompt-file", promptPath,"--out", rawBase,"--raw",- "--resolution", "256x256",+ "--resolution", "768x768",// H3's decoder cannot accept one temporal frame. Six is the smallest legal// request and resolves to 22; only frame zero is retained below."--frames", "6",- "--steps", "30",+ "--steps", "20","--seed", String(stableSeed(entry.id)),"--inference-backend", inferenceBackend,"--attention", inferenceBackend === "cuda" ? "sage2" : "exact",@@ -187,7 +187,7 @@ async function generate(entry, temporaryDirectory, inferenceBackend) {];if (dryRun) args.push("--dry-run");- if (dryRun) console.log(`\n[${entry.name}] MiniMax H3, 30 steps, ${inferenceBackend}`);+ if (dryRun) console.log(`\n[${entry.name}] MiniMax H3, 768x768, 20 steps, ${inferenceBackend}`);await (dryRun ? run : runQuiet)(settings.executable, args, { cwd: repository });if (dryRun) return;await run(settings.ffmpeg, [diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex fc66d95..ff59c41 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -12,6 +12,7 @@ mod agent;mod agent_commands;mod diagnostics;mod export;+mod reference_icons;mod rendered;mod slopfab;@@ -2576,22 +2577,15 @@ fn generated_summary(job_id: String) -> Option<rendered::RenderedSummary> {#[tauri::command]fn save_reference_icon(folder_path: String, job_id: String) -> Result<String, String> {let summary = rendered::summary(&job_id).ok_or("The reference icon is no longer available.")?;- if summary.width != 256 || summary.height != 256 || summary.frame_count != 1 {- return Err("A reference icon must be a single 256x256 frame.".into());+ if summary.width != reference_icons::RENDER_SIZE || summary.height != reference_icons::RENDER_SIZE || summary.frame_count != 1 {+ return Err("A reference icon must be a single 768x768 frame.".into());}let rgba = rendered::frame(&job_id, 0)?.ok_or("The reference icon has no image.")?;write_reference_icon_frame(&folder_path, &job_id, &rgba)}fn write_reference_icon_frame(folder_path: &str, job_id: &str, rgba: &[u8]) -> Result<String, String> {- if rgba.len() != 256 * 256 * 4 {- return Err("The reference icon has an invalid pixel buffer.".into());- }- let rgb: Vec<u8> = rgba.chunks_exact(4).flat_map(|pixel| [pixel[0], pixel[1], pixel[2]]).collect();- let mut bytes = Vec::new();- jpeg_encoder::Encoder::new(&mut bytes, 95)- .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb)- .map_err(|error| format!("Could not encode reference icon: {error}"))?;+ let bytes = reference_icons::encode_jpeg(rgba)?;let root = project_root(folder_path)?;let stem = generated_file_stem(job_id)?;let directory = root.join("references").join("icons");@@ -3552,11 +3546,16 @@ mod tests {fn reference_icon_files_are_256_square_jpegs_confined_to_the_project() {let root = tempfile::tempdir().unwrap();write_project(root.path(), &fixture()).unwrap();- let rgba = vec![128; 256 * 256 * 4];+ let rgba = vec![128; 768 * 768 * 4];let folder = root.path().to_str().unwrap();let relative = write_reference_icon_frame(folder, "icon-test", &rgba).unwrap();assert_eq!(relative, "references/icons/icon-test.jpg");let jpeg = fs::read(root.path().join(&relative)).unwrap();+ let mut expected = Vec::new();+ jpeg_encoder::Encoder::new(&mut expected, 95)+ .encode(&vec![128; 256 * 256 * 3], 256, 256, jpeg_encoder::ColorType::Rgb)+ .unwrap();+ assert_eq!(jpeg, expected);assert_eq!(&jpeg[..2], &[0xff, 0xd8]);assert_eq!(&jpeg[jpeg.len() - 2..], &[0xff, 0xd9]);let frame = jpeg.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap();diff --git a/src-tauri/src/reference_icons.rs b/src-tauri/src/reference_icons.rsnew file mode 100644index 0000000..82959d5--- /dev/null+++ b/src-tauri/src/reference_icons.rs@@ -0,0 +1,73 @@+pub const RENDER_SIZE: u32 = 768;+pub const ICON_SIZE: u16 = 256;+pub const STEPS: i32 = 20;+const JPEG_QUALITY: u8 = 95;++fn downscale_rgb(rgba: &[u8]) -> Result<Vec<u8>, String> {+ let render_size = RENDER_SIZE as usize;+ let icon_size = ICON_SIZE as usize;+ if rgba.len() != render_size * render_size * 4 {+ return Err("The reference icon has an invalid pixel buffer.".into());+ }+ let scale = render_size / icon_size;+ let samples = (scale * scale) as u32;+ let mut rgb = Vec::with_capacity(icon_size * icon_size * 3);+ for row in 0..icon_size {+ for column in 0..icon_size {+ let mut sums = [0_u32; 3];+ for offset_row in 0..scale {+ for offset_column in 0..scale {+ let index =+ ((row * scale + offset_row) * render_size + column * scale + offset_column)+ * 4;+ for channel in 0..3 {+ sums[channel] += u32::from(rgba[index + channel]);+ }+ }+ }+ rgb.extend(sums.map(|sum| ((sum + samples / 2) / samples) as u8));+ }+ }+ Ok(rgb)+}++pub fn encode_jpeg(rgba: &[u8]) -> Result<Vec<u8>, String> {+ let rgb = downscale_rgb(rgba)?;+ let mut bytes = Vec::new();+ jpeg_encoder::Encoder::new(&mut bytes, JPEG_QUALITY)+ .encode(&rgb, ICON_SIZE, ICON_SIZE, jpeg_encoder::ColorType::Rgb)+ .map_err(|error| format!("Could not encode reference icon: {error}"))?;+ Ok(bytes)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn downscale_averages_each_three_by_three_block() {+ let mut rgba = vec![0; RENDER_SIZE as usize * RENDER_SIZE as usize * 4];+ for row in 0..RENDER_SIZE as usize {+ for column in 0..RENDER_SIZE as usize {+ let index = (row * RENDER_SIZE as usize + column) * 4;+ rgba[index..index + 4].copy_from_slice(&[+ ((column % 3) * 90) as u8,+ ((row % 3) * 60) as u8,+ (column / 3) as u8,+ 255,+ ]);+ }+ }+ let rgb = downscale_rgb(&rgba).unwrap();+ assert_eq!(rgb.len(), ICON_SIZE as usize * ICON_SIZE as usize * 3);+ for (index, pixel) in rgb.chunks_exact(3).enumerate() {+ assert_eq!(pixel, [90, 60, (index % ICON_SIZE as usize) as u8]);+ }+ }++ #[test]+ fn rejects_buffers_that_are_not_768_square_rgba() {+ assert!(encode_jpeg(&[]).is_err());+ assert!(encode_jpeg(&vec![0; 256 * 256 * 4]).is_err());+ }+}diff --git a/src-tauri/src/slopfab.rs b/src-tauri/src/slopfab.rsindex e7f625d..6a89270 100644--- a/src-tauri/src/slopfab.rs+++ b/src-tauri/src/slopfab.rs@@ -816,11 +816,11 @@ fn write_reference_icon(prompt: spec.prompt.clone(),frames: 1,still_image: true,- steps: 30,+ steps: crate::reference_icons::STEPS,seed: i64::try_from(spec.seed).map_err(|_| format!("Icon '{}' seed is too large.", spec.id))?,- canvas_width: 256,- canvas_height: 256,+ canvas_width: crate::reference_icons::RENDER_SIZE as i32,+ canvas_height: crate::reference_icons::RENDER_SIZE as i32,reference_paths: Vec::new(),};let handle = RequestHandle::new(api)?;@@ -837,17 +837,15 @@ fn write_reference_icon(});}let output = api.output(generation.0)?;- if output.width != 256 || output.height != 256 || output.frames != 1 {+ let render_size = crate::reference_icons::RENDER_SIZE;+ if output.width != render_size as i32 || output.height != render_size as i32 || output.frames != 1 {return Err(format!("Icon '{}' returned {}x{} with {} frames.",spec.id, output.width, output.height, output.frames));}- let rgba = api.frame_rgba8(generation.0, 0, 256, 256)?;- let rgb = rgba- .chunks_exact(4)- .flat_map(|pixel| [pixel[0], pixel[1], pixel[2]])- .collect::<Vec<_>>();+ let rgba = api.frame_rgba8(generation.0, 0, render_size, render_size)?;+ let jpeg = crate::reference_icons::encode_jpeg(&rgba)?;if let Some(parent) = spec.destination.parent() {std::fs::create_dir_all(parent).map_err(|error| {format!(@@ -856,20 +854,12 @@ fn write_reference_icon()})?;}- let encoder = jpeg_encoder::Encoder::new_file(&spec.destination, 100).map_err(|error| {+ std::fs::write(&spec.destination, jpeg).map_err(|error| {format!("Could not create icon '{}': {error}",spec.destination.display())- })?;- encoder- .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb)- .map_err(|error| {- format!(- "Could not encode icon '{}': {error}",- spec.destination.display()- )- })+ })}pub fn generate_reference_icon_batch(@@ -1501,12 +1491,12 @@ mod tests {if !default_dll_path().is_file() { return; }let request: GenerationRequest = serde_json::from_value(serde_json::json!({"jobId": "icon-plan", "prompt": "A red toy car", "frames": 1, "stillImage": true,- "steps": 30, "seed": 1, "canvasWidth": 256, "canvasHeight": 256+ "steps": 20, "seed": 1, "canvasWidth": 768, "canvasHeight": 768})).unwrap();let plan = resolve_plan(&request, &BTreeMap::new()).unwrap();assert_eq!(plan.aligned_frames, 1);assert_eq!(plan.latent_frames, 1);- assert_eq!((plan.canvas_width, plan.canvas_height), (256, 256));+ assert_eq!((plan.canvas_width, plan.canvas_height), (768, 768));}#[test]fn generation_seed_accepts_random_and_non_negative_values() {diff --git a/src/lib/referenceIconWork.test.ts b/src/lib/referenceIconWork.test.tsindex 44d4d60..a48b677 100644--- a/src/lib/referenceIconWork.test.ts+++ b/src/lib/referenceIconWork.test.ts@@ -60,7 +60,8 @@ it("groups icons across projects and gives waiting videos priority between iconsexpect(queue.getSnapshot().filter((item) => item.kind === "reference-icons")).toHaveLength(1);expect(queue.hasActiveProject(second.record)).toBe(true);await finish(requests()[0].jobId);- expect(requests()[1]).toMatchObject({ stillImage: true, frames: 1, canvasWidth: 256, canvasHeight: 256, steps: 30 });+ expect(requests()[1]).toMatchObject({ stillImage: true, frames: 1, canvasWidth: 768, canvasHeight: 768, steps: 20 });+ expect(queue.getSnapshot().find((item) => item.kind === "reference-icons")?.settings).toMatchObject({ canvasWidth: 768, canvasHeight: 768, steps: 20 });expect(requests()[1].prompt).toContain("User description for hero.");video(queue, second);await finish(requests()[1].jobId);@@ -191,7 +192,7 @@ it("forces replacement of bundled and generated icons in one batch behind videosexpect(queue.isReferenceIconPending(session, "existing")).toBe(true);expect(session.getSnapshot().config.references[0].iconRelativePath).toBe(existing.iconRelativePath);await finish(requests()[0].jobId);- expect(requests()[1]).toMatchObject({ stillImage: true, canvasWidth: 256, canvasHeight: 256 });+ expect(requests()[1]).toMatchObject({ stillImage: true, canvasWidth: 768, canvasHeight: 768, steps: 20 });await finish(requests()[1].jobId);expect(session.getSnapshot().config.references[0].iconRelativePath).toBe(`references/icons/${requests()[1].jobId}.jpg`);expect(queue.isReferenceIconPending(session, "existing")).toBe(false);diff --git a/src/lib/referenceIconWork.ts b/src/lib/referenceIconWork.tsindex aceda24..4dfae4e 100644--- a/src/lib/referenceIconWork.ts+++ b/src/lib/referenceIconWork.ts@@ -3,7 +3,7 @@ import { releaseRendered } from "./generatedVideo";import { isTauri } from "./persistence";import type { ProjectSession } from "./projectSession";import { referenceImages, type ProjectReference } from "./project";-import { needsReferenceIcon, referenceIconPrompt, REFERENCE_ICON_SIZE, REFERENCE_ICON_STEPS } from "./referenceIcons";+import { needsReferenceIcon, referenceIconPrompt, REFERENCE_ICON_RENDER_SIZE, REFERENCE_ICON_STEPS } from "./referenceIcons";import { cancelSlopfabGeneration, enqueueSlopfabGeneration, saveReferenceIcon } from "./runtime";import { withEngineSettings } from "./settings";import type { WorkItem } from "./workQueue";@@ -123,7 +123,7 @@ export class ReferenceIconWork {id: `icons-${crypto.randomUUID()}`, kind: "reference-icons", projectKey: "", folderPath: "", projectName: "", sceneId: "",title: "Reference icons", submittedAt: new Date().toISOString(), status: "queued", progress: 0,detail: "Waiting to generate icons", error: null, completionAt: null, cancelling: false, needsSave: false,- settings: { frames: 1, steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_SIZE, canvasHeight: REFERENCE_ICON_SIZE },+ settings: { frames: 1, steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE },};}if (!this.item) return;@@ -168,7 +168,7 @@ export class ReferenceIconWork {const done = new Promise<void>((resolve) => { task.finish = resolve; });await enqueueSlopfabGeneration({jobId: task.nativeId, prompt: task.prompt, stillImage: true, frames: 1,- steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_SIZE, canvasHeight: REFERENCE_ICON_SIZE, referencePaths: [],+ steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE, referencePaths: [],}, withEngineSettings(task.session.getSnapshot().config));task.submitted = true;if (task.cancelled) await this.cancelNative(task);diff --git a/src/lib/referenceIcons.ts b/src/lib/referenceIcons.tsindex 0d11e3b..131566d 100644--- a/src/lib/referenceIcons.ts+++ b/src/lib/referenceIcons.ts@@ -2,7 +2,8 @@ import { referenceImages, type ProjectReference } from "./project";import { referenceType, selectedReferencePreset } from "./reference-presets";export const REFERENCE_ICON_SIZE = 256;-export const REFERENCE_ICON_STEPS = 30;+export const REFERENCE_ICON_RENDER_SIZE = 768;+export const REFERENCE_ICON_STEPS = 20;const COMPOSITIONS = {character: "A clear head-and-shoulders character portrait. Emphasize recognizable facial features, hair, clothing, and personality against a simple background.",