Skip to content

Unrestricted Open-source alternative to AI video platforms — Free AI image & video generation studio with 600+ models (Flux, Midjourney, Kling, Sora, Veo). No content filters. Self-hosted, MIT licensed.

Merge pull request #336 from antilya/video-model-workflows

jaiprasad04c90e9080fcf4
diff --cc packages/studio/src/models.js
index 24088a6b,939813e0..4bc0c1ea
--- a/packages/studio/src/models.js
+++ b/packages/studio/src/models.js
@@@ -1,10 -1,6 +1,12 @@@
+ import { getMediaCapability } from "./modelCapabilities.js";
+
// Auto-generated from models_dump.json
+import {
+ getAspectRatioOptions,
+ I2I_DIMENSION_RATIOS,
+ T2I_DIMENSION_RATIOS,
+} from './imageSizing.js';
+
export const t2iModels = [
{
"id": "nano-banana",
@@@ -7864,13 -7839,10 +7885,10 @@@ const getInputOptions = (input) =>
export const getDurationsForModel = (modelId) => {
const model = getVideoModelById(modelId);
if (!model) return [5];
- const durInput = model.inputs?.duration;
- if (durInput && durInput.enum) return durInput.enum;
- if (durInput) return [durInput.default || 5];
- return [];
+ return getInputOptions(model.inputs?.duration);
};
-export const getResolutionsForVideoModel = (modelId) => {
+export const getResolutionsForVideoModel = (modelId, selections = {}) => {
const model = getVideoModelById(modelId);
if (!model) return [];
const resInput = model.inputs?.resolution;
@@@ -13716,7 -13706,7 +13755,8 @@@ export const i2vModels =
"family": "kling-v2.6",
"imageField": "image_url",
"hasPrompt": true,
+ "promptRequired": true,
+ "parameterNotice": "This integration supports 5 or 10 seconds.",
"inputs": {
"prompt": {
"type": "string",
@@@ -14044,8 -14028,7 +14087,9 @@@
"imageField": "image_url",
"lastImageField": "last_image",
"hasPrompt": true,
+ "promptRequired": true,
+ "aspectRatioMode": "inherited",
+ "parameterNotice": "Aspect ratio is inherited from the input image.",
"inputs": {
"prompt": {
"type": "string",
@@@ -14119,8 -14102,7 +14163,9 @@@
"imageField": "image_url",
"lastImageField": "last_image",
"hasPrompt": true,
+ "promptRequired": true,
+ "aspectRatioMode": "inherited",
+ "parameterNotice": "Aspect ratio is inherited from the input image.",
"inputs": {
"prompt": {
"type": "string",
@@@ -15093,10 -15084,8 +15147,11 @@@
"family": "pixverse-v6",
"imageField": "image_url",
"lastImageField": "last_image",
+ "endImageRequired": true,
"hasPrompt": true,
+ "promptRequired": true,
+ "aspectRatioMode": "inherited",
+ "parameterNotice": "Start and end frames are required. Aspect ratio is determined by the input frames.",
"inputs": {
"prompt": {
"type": "string",
@@@ -15719,9 -15715,8 +15781,10 @@@
"endpoint": "seedance-2-first-last-frame-fast",
"family": "sd-2",
"imageField": "images_list",
+ "lastImageField": "images_list",
+ "endImageRequired": true,
"hasPrompt": true,
+ "promptRequired": true,
"inputs": {
"prompt": {
"type": "string",
@@@ -16058,9 -16057,8 +16125,10 @@@
"endpoint": "seedance-2-vip-first-last-frame",
"family": "sd-2",
"imageField": "images_list",
+ "lastImageField": "images_list",
+ "endImageRequired": true,
"hasPrompt": true,
+ "promptRequired": true,
"inputs": {
"prompt": {
"type": "string",
@@@ -16128,9 -16126,8 +16196,10 @@@
"endpoint": "seedance-2-vip-first-last-frame-fast",
"family": "sd-2",
"imageField": "images_list",
+ "lastImageField": "images_list",
+ "endImageRequired": true,
"hasPrompt": true,
+ "promptRequired": true,
"inputs": {
"prompt": {
"type": "string",
@@@ -19490,19 -19516,9 +19589,14 @@@ export const getI2VModelById = (id) =>
export const getMaxImagesForI2VModel = (modelId) => {
const model = getI2VModelById(modelId);
- if (!model) return 1;
- if (model.maxImages) return model.maxImages;
- const imageInput = model.inputs?.[model.imageField];
- if (imageInput?.type === 'array') return imageInput.maxItems || 1;
- if (model.lastImageField) return 2;
- return 1;
+ return model ? getMediaCapability(model, 'image').maxItems : 1;
};
+export const getSelectableAspectRatiosForI2IModel = (modelId) => {
+ const model = getI2IModelById(modelId);
+ return getAspectRatioOptions(model, I2I_DIMENSION_RATIOS);
+};
+
export const getAspectRatiosForI2IModel = (modelId) => {
const model = getI2IModelById(modelId);
if (!model) return ['1:1'];
@@@ -19521,19 -19536,10 +19615,10 @@@ export const getAspectRatiosForI2VMode
export const getDurationsForI2VModel = (modelId) => {
const model = getI2VModelById(modelId);
if (!model) return [];
- const dur = model.inputs && model.inputs.duration;
- if (!dur) return [];
- if (dur.enum) return dur.enum;
- if (dur.minValue !== undefined && dur.maxValue !== undefined && dur.step) {
- const vals = [];
- for (let v = dur.minValue; v <= dur.maxValue; v += dur.step) vals.push(v);
- return vals;
- }
- if (dur.default) return [dur.default];
- return [];
+ return getInputOptions(model.inputs?.duration);
};
-export const getResolutionsForI2VModel = (modelId) => {
+export const getResolutionsForI2VModel = (modelId, selections = {}) => {
const model = getI2VModelById(modelId);
if (!model) return [];
const res = model.inputs && model.inputs.resolution;
diff --cc packages/studio/src/muapi.js
index 2262b0bf,56241d91..ec8cfed0
--- a/packages/studio/src/muapi.js
+++ b/packages/studio/src/muapi.js
@@@ -1,11 -1,6 +1,13 @@@
import { getModelById, getVideoModelById, getI2IModelById, getI2VModelById, getV2VModelById, getRecastModelById, getLipSyncModelById, getAudioModelById } from './models.js';
+import {
+ buildVideoToolPayload,
+ serializeVideoToolOptions,
+} from './videoToolCapabilities.js';
+import { buildImageSizePayload } from './imageSizing.js';
+import { buildImageInputPayload, getImageInputValidationError, normalizePrimaryImageUrls } from './imageInputContracts.js';
+import { pollForGenerationResult } from './utils/generationLifecycle.js';
+ import { mapReferenceParams } from './modelCapabilities.js';
+ import { buildSupplementalInputPayload } from './modelParameters.js';
// In an http(s) browser we route through the host app's proxy (Next.js routes
// under /api/* re-issue the call server-side) so api.muapi.ai CORS is bypassed.
@@@ -23,24 -16,44 +25,39 @@@ function notifyAuthRequired(status, det
window.dispatchEvent(new CustomEvent('muapi:auth-required', { detail: { status, message: detail } }));
}
+ function assertRequiredPrompt(model, params) {
+ if (model?.promptRequired && !String(params.prompt || '').trim()) {
+ throw new Error('Prompt is required for this model.');
+ }
+ }
+
+ function includeRequiredArrayDefaults(model, payload) {
+ const defaults = {};
+ for (const field of model?.required || []) {
+ if (payload[field] !== undefined || model?.inputs?.[field]?.type !== 'array') continue;
+ defaults[field] = [];
+ }
+ return Object.keys(defaults).length > 0 ? { ...defaults, ...payload } : payload;
+ }
+
async function pollForResult(requestId, key, maxAttempts = 900, interval = 2000) {
- const pollUrl = `${BASE_URL}/api/v1/predictions/${requestId}/result`;
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
- await new Promise(resolve => setTimeout(resolve, interval));
- try {
- const response = await fetch(pollUrl, {
- headers: { 'Content-Type': 'application/json', 'x-api-key': key }
- });
- if (!response.ok) {
- const errText = await response.text();
- if (response.status >= 500) continue;
- notifyAuthRequired(response.status, errText);
- throw new Error(`Poll Failed: ${response.status} - ${errText.slice(0, 100)}`);
- }
- const data = await response.json();
- const status = data.status?.toLowerCase();
- if (status === 'completed' || status === 'succeeded' || status === 'success') return data;
- if (status === 'failed' || status === 'error') throw new Error(`Generation failed: ${data.error || 'Unknown error'}`);
- } catch (error) {
- if (attempt === maxAttempts) throw error;
- }
- }
- throw new Error('Generation timed out after polling.');
+ return pollForGenerationResult({
+ baseUrl: BASE_URL,
+ requestId,
+ apiKey: key,
+ maxAttempts,
+ interval,
+ onAuthRequired: notifyAuthRequired,
+ });
+}
+
+function normalizePredictionResult(submitData, result, outputUrl) {
+ const requestId = submitData?.request_id || submitData?.id || result?.request_id || result?.id;
+ return {
+ ...result,
+ ...(requestId ? { request_id: requestId } : {}),
+ url: outputUrl,
+ };
}
async function submitAndPoll(endpoint, payload, key, onRequestId, maxAttempts = 60) {
@@@ -67,9 -80,11 +84,12 @@@
export async function generateImage(apiKey, params) {
const modelInfo = getModelById(params.model);
const endpoint = modelInfo?.endpoint || params.model;
- const payload = { prompt: params.prompt };
+ const payload = {
+ ...buildSupplementalInputPayload(modelInfo, params),
+ prompt: params.prompt
+ };
- if (params.aspect_ratio) payload.aspect_ratio = params.aspect_ratio;
+ if (modelInfo) Object.assign(payload, buildImageSizePayload(modelInfo, params.aspect_ratio));
+ else if (params.aspect_ratio) payload.aspect_ratio = params.aspect_ratio;
if (params.resolution) payload.resolution = params.resolution;
if (params.quality) payload.quality = params.quality;
if (params.image_url) {
@@@ -87,21 -102,15 +107,25 @@@
export async function generateI2I(apiKey, params) {
const modelInfo = getI2IModelById(params.model);
const endpoint = modelInfo?.endpoint || params.model;
+ const imageField = modelInfo?.imageField || 'image_url';
+ const imagesList = normalizePrimaryImageUrls(params.images_list, params.image_url);
+ const inputError = getImageInputValidationError(modelInfo, 'i2i', {
+ prompt: params.prompt,
+ primaryImageUrls: imagesList,
+ auxiliaryImageUrls: params,
+ });
+ if (inputError) throw new Error(inputError);
- const payload = buildImageInputPayload(modelInfo, 'i2i', params);
+ const payload = {
+ ...mapReferenceParams(modelInfo, params),
- ...buildSupplementalInputPayload(modelInfo, params)
++ ...buildSupplementalInputPayload(modelInfo, params),
++ ...buildImageInputPayload(modelInfo, 'i2i', params)
+ };
- if (params.prompt) payload.prompt = params.prompt;
- if (modelInfo?.swapField && params.swap_url) {
- payload[modelInfo.swapField] = params.swap_url;
+ if (imagesList.length > 0) {
+ if (imageField === 'images_list') payload.images_list = imagesList;
+ else payload[imageField] = imagesList[0];
}
- if (params.aspect_ratio) payload.aspect_ratio = params.aspect_ratio;
+ if (modelInfo) Object.assign(payload, buildImageSizePayload(modelInfo, params.aspect_ratio));
+ else if (params.aspect_ratio) payload.aspect_ratio = params.aspect_ratio;
if (params.resolution) payload.resolution = params.resolution;
if (params.quality) payload.quality = params.quality;
if (modelInfo?.inputs?.name) {
@@@ -135,11 -148,7 +163,12 @@@ export async function generateVideo(api
if (params.resolution) payload.resolution = params.resolution;
if (params.quality) payload.quality = params.quality;
if (params.mode) payload.mode = params.mode;
+ if (params.image_url) payload.image_url = params.image_url;
+ if (params.images_list?.length > 0) payload.images_list = params.images_list;
+ if (params.videos_list?.length > 0) payload.videos_list = params.videos_list;
+ if (params.video_files?.length > 0) payload.video_files = params.video_files;
+ Object.assign(payload, serializeVideoToolOptions(modelInfo, params.options));
+ payload = includeRequiredArrayDefaults(modelInfo, payload);
return submitAndPoll(endpoint, payload, apiKey, params.onRequestId, 900);
}
@@@ -197,7 -188,15 +208,16 @@@ export async function generateMarketing
export async function processV2V(apiKey, params) {
const modelInfo = getV2VModelById(params.model);
const endpoint = modelInfo?.endpoint || params.model;
- const payload = buildVideoToolPayload(modelInfo, params);
- assertRequiredPrompt(modelInfo, params);
++ const toolPayload = buildVideoToolPayload(modelInfo, params);
+ let payload = {
+ ...mapReferenceParams(modelInfo, params),
- ...buildSupplementalInputPayload(modelInfo, params)
++ ...buildSupplementalInputPayload(modelInfo, params),
++ ...toolPayload,
+ };
+ if (modelInfo?.hasPrompt && params.prompt) {
+ payload.prompt = params.prompt;
+ }
+ payload = includeRequiredArrayDefaults(modelInfo, payload);
return submitAndPoll(endpoint, payload, apiKey, params.onRequestId, 900);
}