Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Select Yue2 weights by VRAM

Bittid76f383e09a1
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index cf09e8d..7e2e551 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -4,7 +4,7 @@ Changes since 0.3.2.
## What's new
-- **Music.** Generate with YuE2 or MiniMax Music3, browse songs in grid or list view, use bottom playback controls, and create references or timeline audio.
+- **Music.** Generate with YuE2 or MiniMax Music3, browse grid/list views, play songs, and create references or timeline audio. YuE2 selects BF16 above 8 GiB VRAM, otherwise INT8.
- **Motions.** Create JavaScript animations with the agent, edit elements and parameters, and export them on the timeline or as reference videos.
- **Scene history.** Keep up to four takes and an editable Draft, restore take settings, choose the primary timeline video, and duplicate scenes.
- **Agent frame inspection.** Agents can view timeline frames, scenes, takes, references, generated images and Motions at a requested time.
diff --git a/src/lib/settings.test.ts b/src/lib/settings.test.ts
index 7e2bef4..46e6e20 100644
--- a/src/lib/settings.test.ts
+++ b/src/lib/settings.test.ts
@@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it } from "vitest";
import { TURBO_LORA, VIGGLE_ANIMATE_LORA, loadLoras, saveLoras } from "./loras";
import {
agentEndpointProviderSettings,
+ chooseWeightSource,
selectedImageGeneratorTemplate, createGeneratorTemplate, musicTemplates, mossImageTemplate, fiboImageTemplate, fiboBaseTemplate, templateSupportsEditing, templateSupportsOutpainting, defaultGeneratorTemplate, generatorPathFields, templateSupportsVideo,
engineProviderSetting,
loadInferenceBackend,
@@ -25,6 +26,48 @@ import {
describe("generator templates", () => {
beforeEach(() => localStorage.clear());
+ it("selects YuE2 BF16 only above 8 GiB, with INT8 for smaller or unknown GPUs", () => {
+ const sources = musicTemplates()[0].sources!.transformer!;
+ for (const [gib, filename] of [[4, "yue2_3b_int8_convrot.safetensors"], [8, "yue2_3b_int8_convrot.safetensors"],
+ [8.01, "yue2_3b_bf16.safetensors"], [12, "yue2_3b_bf16.safetensors"], [31.4, "yue2_3b_bf16.safetensors"]] as const) {
+ expect(chooseWeightSource(sources, [{ name: "GPU", memoryBytes: gib * 1024 ** 3 }])?.url)
+ .toBe(`https://huggingface.co/Comfy-Org/YuE2/resolve/main/checkpoints/${filename}`);
+ }
+ expect(chooseWeightSource(sources, [])?.url).toBe(musicTemplates()[0].paths.transformer);
+ });
+
+ it("uses the supplied MiniMax Music3 weights and only its three required model roles", () => {
+ const template = musicTemplates()[1];
+ const root = "https://huggingface.co/Comfy-Org/MiniMax-Music-3/resolve/main/";
+ expect(template.paths).toMatchObject({
+ transformer: root + "diffusion_models/minimax_music3_dit_int8_convrot.safetensors",
+ textEncoder: root + "text_encoders/minimax_music3_text_encoder_pruned_int8_convrot.safetensors",
+ audioVae: root + "vae/minimax_music3_dav.safetensors",
+ });
+ expect(generatorPathFields(template).map(({ id }) => id)).toEqual(["transformer", "textEncoder", "audioVae"]);
+ });
+
+ it("upgrades existing YuE2 sources once while preserving downloaded and custom choices", () => {
+ const template = musicTemplates()[0];
+ const int8 = template.sources!.transformer![1];
+ const cached = "D:/weights/yue2-int8.safetensors";
+ const previous = { ...template, name: "My music", paths: { ...template.paths, transformer: cached },
+ sources: { transformer: [{ ...int8, downloadedPath: cached }] } };
+ saveGeneratorTemplateSettings({ templates: [previous], defaultTemplateId: previous.id, catalogVersion: 16 });
+ const upgraded = loadGeneratorTemplateSettings();
+ expect(upgraded.catalogVersion).toBe(17);
+ expect(upgraded.templates[0]).toMatchObject({ name: "My music", paths: { transformer: cached },
+ sources: { transformer: [template.sources!.transformer![0], { ...int8, downloadedPath: cached }] } });
+ expect(loadGeneratorTemplateSettings()).toEqual(upgraded);
+
+ const custom = { ...previous, sources: { transformer: [{ url: "https://example.com/custom.safetensors", gpuModel: "", minVramGb: 0 }] } };
+ saveGeneratorTemplateSettings({ templates: [custom], defaultTemplateId: custom.id, catalogVersion: 16 });
+ expect(loadGeneratorTemplateSettings().templates).toEqual([custom]);
+ const remaining = musicTemplates()[1];
+ saveGeneratorTemplateSettings({ ...upgraded, templates: [remaining], defaultTemplateId: "" });
+ expect(loadGeneratorTemplateSettings().templates).toEqual([remaining]);
+ });
+
it("starts image projects with the remembered Fibo generator's four-step default", async () => {
const h3 = createGeneratorTemplate("Video"), fibo = createGeneratorTemplate("Fibo", "fibo2");
saveGeneratorTemplateSettings({ templates: [h3, fibo], defaultTemplateId: h3.id, catalogVersion: 14 });
@@ -215,7 +258,7 @@ describe("generator templates", () => {
expect(settings.templates[2]).toEqual(minimaxFastTemplate());
expect(settings.templates[3]).toEqual(minimaxSingularityTemplate());
expect(settings.defaultTemplateId).toBe(template.id);
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(loadGeneratorTemplateSettings()).toEqual(settings);
saveGeneratorTemplateSettings({ ...settings, templates: [settings.templates[0]] });
expect(loadGeneratorTemplateSettings().templates).toHaveLength(1);
@@ -236,7 +279,7 @@ describe("generator templates", () => {
const settings = loadGeneratorTemplateSettings();
expect(settings.templates).toEqual([expect.objectContaining(custom), minimaxSingularityTemplate(), viggleAnimateTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()]);
expect(settings.defaultTemplateId).toBe(custom.id);
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(loadGeneratorTemplateSettings()).toEqual(settings);
saveGeneratorTemplateSettings({ ...settings, templates: [custom] });
expect(loadGeneratorTemplateSettings().templates).toEqual([expect.objectContaining(custom)]);
@@ -301,7 +344,7 @@ describe("generator templates", () => {
localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 8 }));
const settings = loadGeneratorTemplateSettings();
expect(settings.templates).toEqual([minimaxFastTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()]);
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(templateNeedsDownload(settings.templates[0])).toBe(true);
expect(loadGeneratorTemplateSettings()).toEqual(settings);
});
@@ -316,7 +359,7 @@ describe("generator templates", () => {
const otherPaths = { ...template.paths };
localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 10 }));
const settings = loadGeneratorTemplateSettings();
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(settings.templates).toHaveLength(5);
expect(settings.templates[0]).toMatchObject({ id: template.id, name: "FastH3 Omni", defaultSteps: 8,
paths: { ...otherPaths, transformer: minimaxFastTemplate().paths.transformer },
@@ -412,7 +455,7 @@ describe("generator templates", () => {
template.paths[field] = downloadedPath;
localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 1 }));
const settings = loadGeneratorTemplateSettings();
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(settings.templates[0].paths[field]).toBe(downloadedPath);
expect(settings.templates[0].sources![field]).toEqual(minimaxOriginalTemplate().sources![field]!.map((source, index) =>
index === 0 ? { ...source, downloadedPath } : source));
@@ -447,7 +490,7 @@ describe("generator templates", () => {
localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], defaultTemplateId: template.id, catalogVersion: 2 }));
const settings = loadGeneratorTemplateSettings();
const migrated = settings.templates[0];
- expect(settings.catalogVersion).toBe(16);
+ expect(settings.catalogVersion).toBe(17);
expect(migrated.sources!.textEncoder).toEqual([
{ url: originalUrl, gpuModel: "", minVramGb: 0, ...(selection === "cached original" ? { downloadedPath: cachedOriginal } : {}) },
custom,
diff --git a/src/lib/settings.ts b/src/lib/settings.ts
index 8de1f0e..945d0bb 100644
--- a/src/lib/settings.ts
+++ b/src/lib/settings.ts
@@ -257,7 +257,12 @@ export function musicTemplates(): GeneratorTemplate[] {
textEncoder: root + "text_encoders/minimax_music3_text_encoder_pruned_int8_convrot.safetensors",
audioVae: root + "vae/minimax_music3_dav.safetensors" }) };
return { ...createGeneratorTemplate(generatorModel(modelType).label, modelType), id: modelType, paths,
- sources: Object.fromEntries(Object.entries(paths).filter(([, url]) => url).map(([id, url]) => [id, [{ url, gpuModel: "", minVramGb: 0 }]])) };
+ sources: Object.fromEntries(Object.entries(paths).filter(([, url]) => url).map(([id, url]) => [id, [
+ // Selection rounds detected GiB up: a minimum tier of 9 means strictly
+ // more than 8 GiB, keeping 8 GiB cards on the INT8 fallback.
+ ...(modelType === "yue2" && id === "transformer" ? [{ url: "https://huggingface.co/Comfy-Org/YuE2/resolve/main/checkpoints/yue2_3b_bf16.safetensors", gpuModel: "", minVramGb: 9 }] : []),
+ { url, gpuModel: "", minVramGb: 0 },
+ ]])) };
});
}
@@ -366,7 +371,7 @@ export function viggleAnimateTemplate(): GeneratorTemplate {
const initialTemplateSettings = (paths = EMPTY_ENGINE_SETTINGS): GeneratorTemplateSettings => ({
templates: [{ id: "default", name: "Default", modelType: MINIMAX_H3_MODEL_TYPE, defaultSteps: DEFAULT_GENERATION_STEPS, attention: "sage2", paths: copyPaths(paths) }, minimaxOriginalTemplate(), minimaxReferencesTemplate(), minimaxFastTemplate(), minimaxSingularityTemplate(), viggleAnimateTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()],
defaultTemplateId: "default",
- catalogVersion: 16,
+ catalogVersion: 17,
});
const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings | null => {
@@ -429,7 +434,7 @@ const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings |
templates,
defaultTemplateId: templates.find((template) => template.id === requestedDefault && templateUsable(template))?.id
?? templates.find(templateUsable)?.id ?? "",
- catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(record.catalogVersion) ? record.catalogVersion : 0,
+ catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17].includes(record.catalogVersion) ? record.catalogVersion : 0,
};
};
@@ -628,6 +633,19 @@ export function loadGeneratorTemplateSettings(): GeneratorTemplateSettings {
normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 16 })!;
localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));
}
+ if ((normalized.catalogVersion ?? 0) < 17) {
+ const yue = musicTemplates()[0];
+ const bf16 = yue.sources!.transformer![0];
+ normalized.templates = normalized.templates.map((template) => {
+ const sources = template.sources?.transformer ?? [];
+ if (template.id !== yue.id || template.modelType !== yue.modelType
+ || !sources.some((source) => source.url === yue.paths.transformer)
+ || sources.some((source) => source.url === bf16.url)) return template;
+ return { ...template, sources: { ...template.sources, transformer: [bf16, ...sources] } };
+ });
+ normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 17 })!;
+ localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));
+ }
return normalized;
}
}