AI media harness for planning, generating, and editing locally
TypeScript
Select Yue2 weights by VRAM
Bittid76f383e09a1
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.mdindex cf09e8d..7e2e551 100644--- a/RELEASE_NOTES.md+++ b/RELEASE_NOTES.md@@ -4,7 +4,7 @@ Changes since 0.3.2.## What's new-- **Music.** Generate with YuE2 or MiniMax Music3, browse songs in grid or list view, use bottom playback controls, and create references or timeline audio.+- **Music.** Generate with YuE2 or MiniMax Music3, browse grid/list views, play songs, and create references or timeline audio. YuE2 selects BF16 above 8 GiB VRAM, otherwise INT8.- **Motions.** Create JavaScript animations with the agent, edit elements and parameters, and export them on the timeline or as reference videos.- **Scene history.** Keep up to four takes and an editable Draft, restore take settings, choose the primary timeline video, and duplicate scenes.- **Agent frame inspection.** Agents can view timeline frames, scenes, takes, references, generated images and Motions at a requested time.diff --git a/src/lib/settings.test.ts b/src/lib/settings.test.tsindex 7e2bef4..46e6e20 100644--- a/src/lib/settings.test.ts+++ b/src/lib/settings.test.ts@@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it } from "vitest";import { TURBO_LORA, VIGGLE_ANIMATE_LORA, loadLoras, saveLoras } from "./loras";import {agentEndpointProviderSettings,+ chooseWeightSource,selectedImageGeneratorTemplate, createGeneratorTemplate, musicTemplates, mossImageTemplate, fiboImageTemplate, fiboBaseTemplate, templateSupportsEditing, templateSupportsOutpainting, defaultGeneratorTemplate, generatorPathFields, templateSupportsVideo,engineProviderSetting,loadInferenceBackend,@@ -25,6 +26,48 @@ import {describe("generator templates", () => {beforeEach(() => localStorage.clear());+ it("selects YuE2 BF16 only above 8 GiB, with INT8 for smaller or unknown GPUs", () => {+ const sources = musicTemplates()[0].sources!.transformer!;+ for (const [gib, filename] of [[4, "yue2_3b_int8_convrot.safetensors"], [8, "yue2_3b_int8_convrot.safetensors"],+ [8.01, "yue2_3b_bf16.safetensors"], [12, "yue2_3b_bf16.safetensors"], [31.4, "yue2_3b_bf16.safetensors"]] as const) {+ expect(chooseWeightSource(sources, [{ name: "GPU", memoryBytes: gib * 1024 ** 3 }])?.url)+ .toBe(`https://huggingface.co/Comfy-Org/YuE2/resolve/main/checkpoints/${filename}`);+ }+ expect(chooseWeightSource(sources, [])?.url).toBe(musicTemplates()[0].paths.transformer);+ });++ it("uses the supplied MiniMax Music3 weights and only its three required model roles", () => {+ const template = musicTemplates()[1];+ const root = "https://huggingface.co/Comfy-Org/MiniMax-Music-3/resolve/main/";+ expect(template.paths).toMatchObject({+ transformer: root + "diffusion_models/minimax_music3_dit_int8_convrot.safetensors",+ textEncoder: root + "text_encoders/minimax_music3_text_encoder_pruned_int8_convrot.safetensors",+ audioVae: root + "vae/minimax_music3_dav.safetensors",+ });+ expect(generatorPathFields(template).map(({ id }) => id)).toEqual(["transformer", "textEncoder", "audioVae"]);+ });++ it("upgrades existing YuE2 sources once while preserving downloaded and custom choices", () => {+ const template = musicTemplates()[0];+ const int8 = template.sources!.transformer![1];+ const cached = "D:/weights/yue2-int8.safetensors";+ const previous = { ...template, name: "My music", paths: { ...template.paths, transformer: cached },+ sources: { transformer: [{ ...int8, downloadedPath: cached }] } };+ saveGeneratorTemplateSettings({ templates: [previous], defaultTemplateId: previous.id, catalogVersion: 16 });+ const upgraded = loadGeneratorTemplateSettings();+ expect(upgraded.catalogVersion).toBe(17);+ expect(upgraded.templates[0]).toMatchObject({ name: "My music", paths: { transformer: cached },+ sources: { transformer: [template.sources!.transformer![0], { ...int8, downloadedPath: cached }] } });+ expect(loadGeneratorTemplateSettings()).toEqual(upgraded);++ const custom = { ...previous, sources: { transformer: [{ url: "https://example.com/custom.safetensors", gpuModel: "", minVramGb: 0 }] } };+ saveGeneratorTemplateSettings({ templates: [custom], defaultTemplateId: custom.id, catalogVersion: 16 });+ expect(loadGeneratorTemplateSettings().templates).toEqual([custom]);+ const remaining = musicTemplates()[1];+ saveGeneratorTemplateSettings({ ...upgraded, templates: [remaining], defaultTemplateId: "" });+ expect(loadGeneratorTemplateSettings().templates).toEqual([remaining]);+ });+it("starts image projects with the remembered Fibo generator's four-step default", async () => {const h3 = createGeneratorTemplate("Video"), fibo = createGeneratorTemplate("Fibo", "fibo2");saveGeneratorTemplateSettings({ templates: [h3, fibo], defaultTemplateId: h3.id, catalogVersion: 14 });@@ -215,7 +258,7 @@ describe("generator templates", () => {expect(settings.templates[2]).toEqual(minimaxFastTemplate());expect(settings.templates[3]).toEqual(minimaxSingularityTemplate());expect(settings.defaultTemplateId).toBe(template.id);- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(loadGeneratorTemplateSettings()).toEqual(settings);saveGeneratorTemplateSettings({ ...settings, templates: [settings.templates[0]] });expect(loadGeneratorTemplateSettings().templates).toHaveLength(1);@@ -236,7 +279,7 @@ describe("generator templates", () => {const settings = loadGeneratorTemplateSettings();expect(settings.templates).toEqual([expect.objectContaining(custom), minimaxSingularityTemplate(), viggleAnimateTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()]);expect(settings.defaultTemplateId).toBe(custom.id);- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(loadGeneratorTemplateSettings()).toEqual(settings);saveGeneratorTemplateSettings({ ...settings, templates: [custom] });expect(loadGeneratorTemplateSettings().templates).toEqual([expect.objectContaining(custom)]);@@ -301,7 +344,7 @@ describe("generator templates", () => {localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 8 }));const settings = loadGeneratorTemplateSettings();expect(settings.templates).toEqual([minimaxFastTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()]);- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(templateNeedsDownload(settings.templates[0])).toBe(true);expect(loadGeneratorTemplateSettings()).toEqual(settings);});@@ -316,7 +359,7 @@ describe("generator templates", () => {const otherPaths = { ...template.paths };localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 10 }));const settings = loadGeneratorTemplateSettings();- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(settings.templates).toHaveLength(5);expect(settings.templates[0]).toMatchObject({ id: template.id, name: "FastH3 Omni", defaultSteps: 8,paths: { ...otherPaths, transformer: minimaxFastTemplate().paths.transformer },@@ -412,7 +455,7 @@ describe("generator templates", () => {template.paths[field] = downloadedPath;localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], catalogVersion: 1 }));const settings = loadGeneratorTemplateSettings();- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(settings.templates[0].paths[field]).toBe(downloadedPath);expect(settings.templates[0].sources![field]).toEqual(minimaxOriginalTemplate().sources![field]!.map((source, index) =>index === 0 ? { ...source, downloadedPath } : source));@@ -447,7 +490,7 @@ describe("generator templates", () => {localStorage.setItem("slopus.generator-templates.v1", JSON.stringify({ templates: [template], defaultTemplateId: template.id, catalogVersion: 2 }));const settings = loadGeneratorTemplateSettings();const migrated = settings.templates[0];- expect(settings.catalogVersion).toBe(16);+ expect(settings.catalogVersion).toBe(17);expect(migrated.sources!.textEncoder).toEqual([{ url: originalUrl, gpuModel: "", minVramGb: 0, ...(selection === "cached original" ? { downloadedPath: cachedOriginal } : {}) },custom,diff --git a/src/lib/settings.ts b/src/lib/settings.tsindex 8de1f0e..945d0bb 100644--- a/src/lib/settings.ts+++ b/src/lib/settings.ts@@ -257,7 +257,12 @@ export function musicTemplates(): GeneratorTemplate[] {textEncoder: root + "text_encoders/minimax_music3_text_encoder_pruned_int8_convrot.safetensors",audioVae: root + "vae/minimax_music3_dav.safetensors" }) };return { ...createGeneratorTemplate(generatorModel(modelType).label, modelType), id: modelType, paths,- sources: Object.fromEntries(Object.entries(paths).filter(([, url]) => url).map(([id, url]) => [id, [{ url, gpuModel: "", minVramGb: 0 }]])) };+ sources: Object.fromEntries(Object.entries(paths).filter(([, url]) => url).map(([id, url]) => [id, [+ // Selection rounds detected GiB up: a minimum tier of 9 means strictly+ // more than 8 GiB, keeping 8 GiB cards on the INT8 fallback.+ ...(modelType === "yue2" && id === "transformer" ? [{ url: "https://huggingface.co/Comfy-Org/YuE2/resolve/main/checkpoints/yue2_3b_bf16.safetensors", gpuModel: "", minVramGb: 9 }] : []),+ { url, gpuModel: "", minVramGb: 0 },+ ]])) };});}@@ -366,7 +371,7 @@ export function viggleAnimateTemplate(): GeneratorTemplate {const initialTemplateSettings = (paths = EMPTY_ENGINE_SETTINGS): GeneratorTemplateSettings => ({templates: [{ id: "default", name: "Default", modelType: MINIMAX_H3_MODEL_TYPE, defaultSteps: DEFAULT_GENERATION_STEPS, attention: "sage2", paths: copyPaths(paths) }, minimaxOriginalTemplate(), minimaxReferencesTemplate(), minimaxFastTemplate(), minimaxSingularityTemplate(), viggleAnimateTemplate(), mossImageTemplate(), fiboImageTemplate(), fiboBaseTemplate(), ...musicTemplates()],defaultTemplateId: "default",- catalogVersion: 16,+ catalogVersion: 17,});const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings | null => {@@ -429,7 +434,7 @@ const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings |templates,defaultTemplateId: templates.find((template) => template.id === requestedDefault && templateUsable(template))?.id?? templates.find(templateUsable)?.id ?? "",- catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(record.catalogVersion) ? record.catalogVersion : 0,+ catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17].includes(record.catalogVersion) ? record.catalogVersion : 0,};};@@ -628,6 +633,19 @@ export function loadGeneratorTemplateSettings(): GeneratorTemplateSettings {normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 16 })!;localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));}+ if ((normalized.catalogVersion ?? 0) < 17) {+ const yue = musicTemplates()[0];+ const bf16 = yue.sources!.transformer![0];+ normalized.templates = normalized.templates.map((template) => {+ const sources = template.sources?.transformer ?? [];+ if (template.id !== yue.id || template.modelType !== yue.modelType+ || !sources.some((source) => source.url === yue.paths.transformer)+ || sources.some((source) => source.url === bf16.url)) return template;+ return { ...template, sources: { ...template.sources, transformer: [bf16, ...sources] } };+ });+ normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 17 })!;+ localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));+ }return normalized;}}