Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Clarify image refmod conditioning and diagnostics

Bitti6d8ee605a30a
diff --git a/src-tauri/src/commands/generation.rs b/src-tauri/src/commands/generation.rs
index f3ecf7f..042f5e2 100644
--- a/src-tauri/src/commands/generation.rs
+++ b/src-tauri/src/commands/generation.rs
@@ -87,7 +87,7 @@ pub(crate) fn resolve_slopfab_plan(
let context = serde_json::json!({
"jobId": request.job_id, "frames": request.frames, "steps": request.steps,
"canvasWidth": request.canvas_width, "canvasHeight": request.canvas_height,
- "referenceCount": request.reference_count(), "inputChars": request.prompt.chars().count(),
+ "referenceCount": request.reference_count(), "refmodCount": request.refmod_count(), "inputChars": request.prompt.chars().count(),
});
diagnostics::info(
"slopfab",
diff --git a/src-tauri/src/slopfab/runtime.rs b/src-tauri/src/slopfab/runtime.rs
index b39f474..458ddcb 100644
--- a/src-tauri/src/slopfab/runtime.rs
+++ b/src-tauri/src/slopfab/runtime.rs
@@ -39,6 +39,7 @@ impl Default for SlopfabRuntime {
"canvasWidth": item.request.canvas_width,
"canvasHeight": item.request.canvas_height,
"referenceCount": item.request.reference_count(),
+ "refmodCount": item.request.refmod_count(),
"randomSeed": item.request.seed == -1,
}));
let result = run_generation(&item);
@@ -137,6 +138,7 @@ impl SlopfabRuntime {
"canvasWidth": request.canvas_width,
"canvasHeight": request.canvas_height,
"referenceCount": request.reference_count(),
+ "refmodCount": request.refmod_count(),
"inputChars": request.prompt.chars().count(),
}),
);
diff --git a/src-tauri/src/slopfab/tests.rs b/src-tauri/src/slopfab/tests.rs
index aeb62dd..8a37b57 100644
--- a/src-tauri/src/slopfab/tests.rs
+++ b/src-tauri/src/slopfab/tests.rs
@@ -511,6 +511,7 @@ fn refmods_attach_for_plans_and_icons_or_report_an_older_dll() {
};
let probe = RequestHandle::new(&api).unwrap();
let added = api.add_refmod(&probe, &file, 0.7, 2);
+ assert_eq!(request.refmod_count(), 1);
if let Err(error) = &added {
assert!(error.contains("does not support refmods"), "{error}");
eprintln!(
@@ -551,6 +552,7 @@ fn refmods_attach_for_plans_and_icons_or_report_an_older_dll() {
}
}
request.refmods[0].strength = 0.0;
+ assert_eq!(request.refmod_count(), 0);
std::fs::remove_file(&file).unwrap();
let handle = RequestHandle::new(&api).unwrap();
configure_request(
diff --git a/src-tauri/src/slopfab/types.rs b/src-tauri/src/slopfab/types.rs
index 53c2976..d9f8ce3 100644
--- a/src-tauri/src/slopfab/types.rs
+++ b/src-tauri/src/slopfab/types.rs
@@ -76,6 +76,9 @@ impl GenerationRequest {
pub fn reference_count(&self) -> usize {
self.reference_paths.len() + self.reference_video_ids.len()
}
+ pub fn refmod_count(&self) -> usize {
+ self.refmods.iter().filter(|refmod| refmod.strength > 0.0).count()
+ }
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
diff --git a/src/lib/imagePrompt.test.ts b/src/lib/imagePrompt.test.ts
index 53985fe..0d6551a 100644
--- a/src/lib/imagePrompt.test.ts
+++ b/src/lib/imagePrompt.test.ts
@@ -76,4 +76,15 @@ describe("MiniMax H3 still-image prompts", () => {
config.brief.prompt = "";
expect(compileImagePrompt(config).prompt).toBe("");
});
+ it("describes an active refmod as visual identity conditioning, with no invented picture label", () => {
+ const config = photo();
+ config.references = [{ id: "identity", kind: "text", name: "Woman", description: "", intendedUse: ["character"], createdAt: config.createdAt,
+ refmods: [{ id: "encoded", name: "Identity", sourcePath: "D:/identity.safetensors", strength: 1, copies: 1 }] }];
+ config.imageScene!.referenceIds = ["identity"];
+ const prompt = compileImagePrompt(config).prompt;
+ expect(prompt).toContain("Use the supplied reference conditioning for Woman's identity and appearance in this still image");
+ expect(prompt).not.toMatch(/Additional visual guidance: Woman|<Picture|<Video|safetensors/);
+ config.references[0].refmods![0].strength = 0;
+ expect(compileImagePrompt(config).prompt).not.toContain("Woman");
+ });
});
diff --git a/src/lib/imagePrompt.ts b/src/lib/imagePrompt.ts
index c849c30..98d23d0 100644
--- a/src/lib/imagePrompt.ts
+++ b/src/lib/imagePrompt.ts
@@ -1,5 +1,5 @@
import { createImageScene, imageScenePrompt } from "./imageScene";
-import { referenceDefinition, referenceImages, type ProjectConfig } from "./project";
+import { activeReferenceRefmods, referenceDefinition, referenceImages, type ProjectConfig } from "./project";
import { SHOT_TAG_GROUPS } from "./shot-tags";
/** Shared by the generation request and its debug preview. */
@@ -15,12 +15,22 @@ export function compileImagePrompt(config: ProjectConfig) {
const referencePrompts = references.map((reference) => {
const labels = referenceImages(reference).map(() => `<Picture ${++picture}>`);
const definition = referenceDefinition(reference);
+ const refmods = activeReferenceRefmods(reference);
+ if (reference.refmods?.length && !refmods.length && !labels.length && !definition) return "";
const content = definition || reference.name;
const description = /[.!?。!?]$/.test(content) ? content : `${content}.`;
+ if (refmods.length) {
+ const role = reference.intendedUse.includes("character") || reference.intendedUse.includes("animal") ? "identity and appearance" : reference.intendedUse.includes("style") ? "visual style" : "appearance";
+ return [
+ `Use the supplied reference conditioning for ${reference.name}'s ${role} in this still image, while following the requested composition and styling.`,
+ ...(definition ? [description] : []),
+ ...(labels.length ? [`Use ${labels.join(", ")} as additional visual guidance for ${reference.name}.`] : []),
+ ].join(" ");
+ }
return labels.length
? `Use ${labels.join(", ")} as visual guidance for ${reference.name}. ${description}`
: `Additional visual guidance: ${description}`;
- });
+ }).filter(Boolean);
if (picture > 9) throw new Error("MiniMax H3 supports at most nine reference images per generation.");
const look = SHOT_TAG_GROUPS.find((group) => group.id === "visualStyle")?.options.find((option) => option.id === config.settings.defaultLook)?.label;
return {
diff --git a/src/lib/workQueue.test.ts b/src/lib/workQueue.test.ts
index ed0021a..b82e255 100644
--- a/src/lib/workQueue.test.ts
+++ b/src/lib/workQueue.test.ts
@@ -56,6 +56,25 @@ const finish = async (queue: WorkQueue, id: string) => {
describe("image generation work", () => {
const template: GeneratorTemplate = { id: "image-test", name: "MiniMax H3", defaultSteps: 20, attention: "sage2", paths: { ...EMPTY_ENGINE_SETTINGS, transformer: "C:/h3.safetensors" } };
const imageProject = (): ProjectRecord => ({ folderPath: "C:/Image", config: createProjectConfig({ name: "Poster", prompt: "An ocean poster", generationType: "image", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 60 }) });
+ it("forwards selected refmods to still-image planning and generation with their strength and copies", async () => {
+ const { queue } = setup();
+ const project = imageProject();
+ project.config.references = [{ id: "person", kind: "text", name: "Person", description: "", intendedUse: [], createdAt: project.config.createdAt, refmods: [
+ { id: "identity", name: "Identity", relativePath: "references/person.safetensors", strength: 0.75, copies: 2 },
+ { id: "outfit", name: "Outfit", sourcePath: "D:/Refmods/outfit.safetensors", strength: 1, copies: 1 },
+ { id: "disabled", name: "Disabled", sourcePath: "D:/Refmods/off.safetensors", strength: 0, copies: 3 },
+ ] }];
+ project.config.imageScene!.referenceIds = ["person"];
+ queue.enqueueImage(queue.project(project), template);
+ await waitFor(() => expect(enqueueSlopfabGeneration).toHaveBeenCalledOnce());
+ const request = vi.mocked(enqueueSlopfabGeneration).mock.calls[0][0];
+ expect(request).toMatchObject({ stillImage: true, frames: 1, referencePaths: [], refmods: [
+ { path: "C:/Image/references/person.safetensors", strength: 0.75, copies: 2 },
+ { path: "D:/Refmods/outfit.safetensors", strength: 1, copies: 1 },
+ ] });
+ expect(vi.mocked(resolveSlopfabPlan).mock.calls[0][0].refmods).toEqual(request.refmods);
+ await queue.cancel(request.jobId);
+ });
it("submits the complete debug prompt with project look and ordered reference pictures", async () => {
const { queue } = setup();
const project = imageProject();