Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Improve reference icon rendering

Bittice9278fb147f
diff --git a/README.md b/README.md
index ad36819..7aa1231 100644
--- a/README.md
+++ b/README.md
@@ -64,14 +64,15 @@ The web fallback seeds three deterministic showcase projects on first launch. Re
### Reference icons
Custom references with a prompt and no existing artwork automatically receive
-256×256 JPG icons through SlopFab's dedicated still-image mode. Category-specific
+256×256 JPG icons, generated at 768×768 with 20 steps through SlopFab's dedicated
+still-image mode, then downscaled and encoded at JPEG quality 95. Category-specific
compositions include the user's prompt. All icon work appears in one Work Queue
entry, and waiting video generations run before the next icon. Generated icons
are saved under `references/icons/` in the project and are not sent to the video
engine as image attachments. Built-in presets are added as new references from
the Add a reference dialog.
-`npm run reference-icons` renders every missing character, product, location, and style preset with MiniMax H3 at 30 steps. Its DLL backend uses the dedicated still-image mode and writes a high-quality 256×256 JPEG without creating an intermediate video or audio file. Character entries use controlled headshot lighting; product and location entries use a complete-subject view; style entries use a representative composition with lighting and rendering tailored to the named style.
+`npm run reference-icons` renders every missing character, product, location, and style preset with MiniMax H3 at 768×768 and 20 steps. Its DLL backend uses the dedicated still-image mode, downsamples each 3×3 pixel block to produce a 256×256 icon, and encodes it at JPEG quality 95 without creating an intermediate video or audio file. Character entries use controlled headshot lighting; product and location entries use a complete-subject view; style entries use a representative composition with lighting and rendering tailored to the named style.
SlopFab, Cargo, and FFmpeg must be installed. The script uses the development paths under `D:\Projects\slopfab` by default; `SLOPFAB_EXE`, `SLOPFAB_TRANSFORMER`, `SLOPFAB_TEXT_ENCODER`, `SLOPFAB_VIDEO_VAE`, `SLOPFAB_AUDIO_VAE`, `SLOPFAB_BACKEND`, `CARGO_EXE`, and `FFMPEG_EXE` can override them. Use `node scripts/generate-reference-icons.mjs --id=<preset-id> --force` to regenerate one entry.
diff --git a/scripts/generate-reference-icons.mjs b/scripts/generate-reference-icons.mjs
index 02facfe..3fb1ff3 100644
--- a/scripts/generate-reference-icons.mjs
+++ b/scripts/generate-reference-icons.mjs
@@ -172,11 +172,11 @@ async function generate(entry, temporaryDirectory, inferenceBackend) {
"--prompt-file", promptPath,
"--out", rawBase,
"--raw",
- "--resolution", "256x256",
+ "--resolution", "768x768",
// H3's decoder cannot accept one temporal frame. Six is the smallest legal
// request and resolves to 22; only frame zero is retained below.
"--frames", "6",
- "--steps", "30",
+ "--steps", "20",
"--seed", String(stableSeed(entry.id)),
"--inference-backend", inferenceBackend,
"--attention", inferenceBackend === "cuda" ? "sage2" : "exact",
@@ -187,7 +187,7 @@ async function generate(entry, temporaryDirectory, inferenceBackend) {
];
if (dryRun) args.push("--dry-run");
- if (dryRun) console.log(`\n[${entry.name}] MiniMax H3, 30 steps, ${inferenceBackend}`);
+ if (dryRun) console.log(`\n[${entry.name}] MiniMax H3, 768x768, 20 steps, ${inferenceBackend}`);
await (dryRun ? run : runQuiet)(settings.executable, args, { cwd: repository });
if (dryRun) return;
await run(settings.ffmpeg, [
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index fc66d95..ff59c41 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -12,6 +12,7 @@ mod agent;
mod agent_commands;
mod diagnostics;
mod export;
+mod reference_icons;
mod rendered;
mod slopfab;
@@ -2576,22 +2577,15 @@ fn generated_summary(job_id: String) -> Option<rendered::RenderedSummary> {
#[tauri::command]
fn save_reference_icon(folder_path: String, job_id: String) -> Result<String, String> {
let summary = rendered::summary(&job_id).ok_or("The reference icon is no longer available.")?;
- if summary.width != 256 || summary.height != 256 || summary.frame_count != 1 {
- return Err("A reference icon must be a single 256x256 frame.".into());
+ if summary.width != reference_icons::RENDER_SIZE || summary.height != reference_icons::RENDER_SIZE || summary.frame_count != 1 {
+ return Err("A reference icon must be a single 768x768 frame.".into());
}
let rgba = rendered::frame(&job_id, 0)?.ok_or("The reference icon has no image.")?;
write_reference_icon_frame(&folder_path, &job_id, &rgba)
}
fn write_reference_icon_frame(folder_path: &str, job_id: &str, rgba: &[u8]) -> Result<String, String> {
- if rgba.len() != 256 * 256 * 4 {
- return Err("The reference icon has an invalid pixel buffer.".into());
- }
- let rgb: Vec<u8> = rgba.chunks_exact(4).flat_map(|pixel| [pixel[0], pixel[1], pixel[2]]).collect();
- let mut bytes = Vec::new();
- jpeg_encoder::Encoder::new(&mut bytes, 95)
- .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb)
- .map_err(|error| format!("Could not encode reference icon: {error}"))?;
+ let bytes = reference_icons::encode_jpeg(rgba)?;
let root = project_root(folder_path)?;
let stem = generated_file_stem(job_id)?;
let directory = root.join("references").join("icons");
@@ -3552,11 +3546,16 @@ mod tests {
fn reference_icon_files_are_256_square_jpegs_confined_to_the_project() {
let root = tempfile::tempdir().unwrap();
write_project(root.path(), &fixture()).unwrap();
- let rgba = vec![128; 256 * 256 * 4];
+ let rgba = vec![128; 768 * 768 * 4];
let folder = root.path().to_str().unwrap();
let relative = write_reference_icon_frame(folder, "icon-test", &rgba).unwrap();
assert_eq!(relative, "references/icons/icon-test.jpg");
let jpeg = fs::read(root.path().join(&relative)).unwrap();
+ let mut expected = Vec::new();
+ jpeg_encoder::Encoder::new(&mut expected, 95)
+ .encode(&vec![128; 256 * 256 * 3], 256, 256, jpeg_encoder::ColorType::Rgb)
+ .unwrap();
+ assert_eq!(jpeg, expected);
assert_eq!(&jpeg[..2], &[0xff, 0xd8]);
assert_eq!(&jpeg[jpeg.len() - 2..], &[0xff, 0xd9]);
let frame = jpeg.windows(2).position(|bytes| bytes == [0xff, 0xc0]).unwrap();
diff --git a/src-tauri/src/reference_icons.rs b/src-tauri/src/reference_icons.rs
new file mode 100644
index 0000000..82959d5
--- /dev/null
+++ b/src-tauri/src/reference_icons.rs
@@ -0,0 +1,73 @@
+pub const RENDER_SIZE: u32 = 768;
+pub const ICON_SIZE: u16 = 256;
+pub const STEPS: i32 = 20;
+const JPEG_QUALITY: u8 = 95;
+
+fn downscale_rgb(rgba: &[u8]) -> Result<Vec<u8>, String> {
+ let render_size = RENDER_SIZE as usize;
+ let icon_size = ICON_SIZE as usize;
+ if rgba.len() != render_size * render_size * 4 {
+ return Err("The reference icon has an invalid pixel buffer.".into());
+ }
+ let scale = render_size / icon_size;
+ let samples = (scale * scale) as u32;
+ let mut rgb = Vec::with_capacity(icon_size * icon_size * 3);
+ for row in 0..icon_size {
+ for column in 0..icon_size {
+ let mut sums = [0_u32; 3];
+ for offset_row in 0..scale {
+ for offset_column in 0..scale {
+ let index =
+ ((row * scale + offset_row) * render_size + column * scale + offset_column)
+ * 4;
+ for channel in 0..3 {
+ sums[channel] += u32::from(rgba[index + channel]);
+ }
+ }
+ }
+ rgb.extend(sums.map(|sum| ((sum + samples / 2) / samples) as u8));
+ }
+ }
+ Ok(rgb)
+}
+
+pub fn encode_jpeg(rgba: &[u8]) -> Result<Vec<u8>, String> {
+ let rgb = downscale_rgb(rgba)?;
+ let mut bytes = Vec::new();
+ jpeg_encoder::Encoder::new(&mut bytes, JPEG_QUALITY)
+ .encode(&rgb, ICON_SIZE, ICON_SIZE, jpeg_encoder::ColorType::Rgb)
+ .map_err(|error| format!("Could not encode reference icon: {error}"))?;
+ Ok(bytes)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn downscale_averages_each_three_by_three_block() {
+ let mut rgba = vec![0; RENDER_SIZE as usize * RENDER_SIZE as usize * 4];
+ for row in 0..RENDER_SIZE as usize {
+ for column in 0..RENDER_SIZE as usize {
+ let index = (row * RENDER_SIZE as usize + column) * 4;
+ rgba[index..index + 4].copy_from_slice(&[
+ ((column % 3) * 90) as u8,
+ ((row % 3) * 60) as u8,
+ (column / 3) as u8,
+ 255,
+ ]);
+ }
+ }
+ let rgb = downscale_rgb(&rgba).unwrap();
+ assert_eq!(rgb.len(), ICON_SIZE as usize * ICON_SIZE as usize * 3);
+ for (index, pixel) in rgb.chunks_exact(3).enumerate() {
+ assert_eq!(pixel, [90, 60, (index % ICON_SIZE as usize) as u8]);
+ }
+ }
+
+ #[test]
+ fn rejects_buffers_that_are_not_768_square_rgba() {
+ assert!(encode_jpeg(&[]).is_err());
+ assert!(encode_jpeg(&vec![0; 256 * 256 * 4]).is_err());
+ }
+}
diff --git a/src-tauri/src/slopfab.rs b/src-tauri/src/slopfab.rs
index e7f625d..6a89270 100644
--- a/src-tauri/src/slopfab.rs
+++ b/src-tauri/src/slopfab.rs
@@ -816,11 +816,11 @@ fn write_reference_icon(
prompt: spec.prompt.clone(),
frames: 1,
still_image: true,
- steps: 30,
+ steps: crate::reference_icons::STEPS,
seed: i64::try_from(spec.seed)
.map_err(|_| format!("Icon '{}' seed is too large.", spec.id))?,
- canvas_width: 256,
- canvas_height: 256,
+ canvas_width: crate::reference_icons::RENDER_SIZE as i32,
+ canvas_height: crate::reference_icons::RENDER_SIZE as i32,
reference_paths: Vec::new(),
};
let handle = RequestHandle::new(api)?;
@@ -837,17 +837,15 @@ fn write_reference_icon(
});
}
let output = api.output(generation.0)?;
- if output.width != 256 || output.height != 256 || output.frames != 1 {
+ let render_size = crate::reference_icons::RENDER_SIZE;
+ if output.width != render_size as i32 || output.height != render_size as i32 || output.frames != 1 {
return Err(format!(
"Icon '{}' returned {}x{} with {} frames.",
spec.id, output.width, output.height, output.frames
));
}
- let rgba = api.frame_rgba8(generation.0, 0, 256, 256)?;
- let rgb = rgba
- .chunks_exact(4)
- .flat_map(|pixel| [pixel[0], pixel[1], pixel[2]])
- .collect::<Vec<_>>();
+ let rgba = api.frame_rgba8(generation.0, 0, render_size, render_size)?;
+ let jpeg = crate::reference_icons::encode_jpeg(&rgba)?;
if let Some(parent) = spec.destination.parent() {
std::fs::create_dir_all(parent).map_err(|error| {
format!(
@@ -856,20 +854,12 @@ fn write_reference_icon(
)
})?;
}
- let encoder = jpeg_encoder::Encoder::new_file(&spec.destination, 100).map_err(|error| {
+ std::fs::write(&spec.destination, jpeg).map_err(|error| {
format!(
"Could not create icon '{}': {error}",
spec.destination.display()
)
- })?;
- encoder
- .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb)
- .map_err(|error| {
- format!(
- "Could not encode icon '{}': {error}",
- spec.destination.display()
- )
- })
+ })
}
pub fn generate_reference_icon_batch(
@@ -1501,12 +1491,12 @@ mod tests {
if !default_dll_path().is_file() { return; }
let request: GenerationRequest = serde_json::from_value(serde_json::json!({
"jobId": "icon-plan", "prompt": "A red toy car", "frames": 1, "stillImage": true,
- "steps": 30, "seed": 1, "canvasWidth": 256, "canvasHeight": 256
+ "steps": 20, "seed": 1, "canvasWidth": 768, "canvasHeight": 768
})).unwrap();
let plan = resolve_plan(&request, &BTreeMap::new()).unwrap();
assert_eq!(plan.aligned_frames, 1);
assert_eq!(plan.latent_frames, 1);
- assert_eq!((plan.canvas_width, plan.canvas_height), (256, 256));
+ assert_eq!((plan.canvas_width, plan.canvas_height), (768, 768));
}
#[test]
fn generation_seed_accepts_random_and_non_negative_values() {
diff --git a/src/lib/referenceIconWork.test.ts b/src/lib/referenceIconWork.test.ts
index 44d4d60..a48b677 100644
--- a/src/lib/referenceIconWork.test.ts
+++ b/src/lib/referenceIconWork.test.ts
@@ -60,7 +60,8 @@ it("groups icons across projects and gives waiting videos priority between icons
expect(queue.getSnapshot().filter((item) => item.kind === "reference-icons")).toHaveLength(1);
expect(queue.hasActiveProject(second.record)).toBe(true);
await finish(requests()[0].jobId);
- expect(requests()[1]).toMatchObject({ stillImage: true, frames: 1, canvasWidth: 256, canvasHeight: 256, steps: 30 });
+ expect(requests()[1]).toMatchObject({ stillImage: true, frames: 1, canvasWidth: 768, canvasHeight: 768, steps: 20 });
+ expect(queue.getSnapshot().find((item) => item.kind === "reference-icons")?.settings).toMatchObject({ canvasWidth: 768, canvasHeight: 768, steps: 20 });
expect(requests()[1].prompt).toContain("User description for hero.");
video(queue, second);
await finish(requests()[1].jobId);
@@ -191,7 +192,7 @@ it("forces replacement of bundled and generated icons in one batch behind videos
expect(queue.isReferenceIconPending(session, "existing")).toBe(true);
expect(session.getSnapshot().config.references[0].iconRelativePath).toBe(existing.iconRelativePath);
await finish(requests()[0].jobId);
- expect(requests()[1]).toMatchObject({ stillImage: true, canvasWidth: 256, canvasHeight: 256 });
+ expect(requests()[1]).toMatchObject({ stillImage: true, canvasWidth: 768, canvasHeight: 768, steps: 20 });
await finish(requests()[1].jobId);
expect(session.getSnapshot().config.references[0].iconRelativePath).toBe(`references/icons/${requests()[1].jobId}.jpg`);
expect(queue.isReferenceIconPending(session, "existing")).toBe(false);
diff --git a/src/lib/referenceIconWork.ts b/src/lib/referenceIconWork.ts
index aceda24..4dfae4e 100644
--- a/src/lib/referenceIconWork.ts
+++ b/src/lib/referenceIconWork.ts
@@ -3,7 +3,7 @@ import { releaseRendered } from "./generatedVideo";
import { isTauri } from "./persistence";
import type { ProjectSession } from "./projectSession";
import { referenceImages, type ProjectReference } from "./project";
-import { needsReferenceIcon, referenceIconPrompt, REFERENCE_ICON_SIZE, REFERENCE_ICON_STEPS } from "./referenceIcons";
+import { needsReferenceIcon, referenceIconPrompt, REFERENCE_ICON_RENDER_SIZE, REFERENCE_ICON_STEPS } from "./referenceIcons";
import { cancelSlopfabGeneration, enqueueSlopfabGeneration, saveReferenceIcon } from "./runtime";
import { withEngineSettings } from "./settings";
import type { WorkItem } from "./workQueue";
@@ -123,7 +123,7 @@ export class ReferenceIconWork {
id: `icons-${crypto.randomUUID()}`, kind: "reference-icons", projectKey: "", folderPath: "", projectName: "", sceneId: "",
title: "Reference icons", submittedAt: new Date().toISOString(), status: "queued", progress: 0,
detail: "Waiting to generate icons", error: null, completionAt: null, cancelling: false, needsSave: false,
- settings: { frames: 1, steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_SIZE, canvasHeight: REFERENCE_ICON_SIZE },
+ settings: { frames: 1, steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE },
};
}
if (!this.item) return;
@@ -168,7 +168,7 @@ export class ReferenceIconWork {
const done = new Promise<void>((resolve) => { task.finish = resolve; });
await enqueueSlopfabGeneration({
jobId: task.nativeId, prompt: task.prompt, stillImage: true, frames: 1,
- steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_SIZE, canvasHeight: REFERENCE_ICON_SIZE, referencePaths: [],
+ steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE, referencePaths: [],
}, withEngineSettings(task.session.getSnapshot().config));
task.submitted = true;
if (task.cancelled) await this.cancelNative(task);
diff --git a/src/lib/referenceIcons.ts b/src/lib/referenceIcons.ts
index 0d11e3b..131566d 100644
--- a/src/lib/referenceIcons.ts
+++ b/src/lib/referenceIcons.ts
@@ -2,7 +2,8 @@ import { referenceImages, type ProjectReference } from "./project";
import { referenceType, selectedReferencePreset } from "./reference-presets";
export const REFERENCE_ICON_SIZE = 256;
-export const REFERENCE_ICON_STEPS = 30;
+export const REFERENCE_ICON_RENDER_SIZE = 768;
+export const REFERENCE_ICON_STEPS = 20;
const COMPOSITIONS = {
character: "A clear head-and-shoulders character portrait. Emphasize recognizable facial features, hair, clothing, and personality against a simple background.",