Skip to content

Unrestricted Open-source alternative to AI video platforms — Free AI image & video generation studio with 600+ models (Flux, Midjourney, Kling, Sora, Veo). No content filters. Self-hosted, MIT licensed.

fix(video-studio): support kling motion-control + correct higgsfield endpoints

Anil Matchaaa4a4202b5f8
diff --git a/packages/studio/src/components/VideoStudio.jsx b/packages/studio/src/components/VideoStudio.jsx
index fb718f3c..883b2ace 100644
--- a/packages/studio/src/components/VideoStudio.jsx
+++ b/packages/studio/src/components/VideoStudio.jsx
@@ -1,7 +1,7 @@
"use client";
import { useState, useEffect, useRef, useCallback } from "react";
-import { generateVideo, generateI2V, uploadFile } from "../muapi.js";
+import { generateVideo, generateI2V, processV2V, uploadFile } from "../muapi.js";
import {
t2vModels,
i2vModels,
@@ -144,7 +144,7 @@ function ModelDropdown({ imageMode, selectedModel, onSelect, onClose }) {
</span>
{isV2V && (
<span className="text-[9px] text-orange-400/70">
- Upload a video to use
+ {m.imageField ? "Upload a video and image" : "Upload a video to use"}
</span>
)}
</div>
@@ -349,6 +349,15 @@ export default function VideoStudio({
[getCurrentModels, selectedModel],
);
+ const isMotionControlSelection = useCallback(
+ (modelId, isV2v) => {
+ if (!isV2v) return false;
+ const m = v2vModels.find((x) => x.id === modelId);
+ return !!m?.imageField;
+ },
+ [],
+ );
+
// ── update controls when model/mode changes ──────────────────────────────
const applyControlsForModel = useCallback(
(modelId, isImageMode, isV2vMode) => {
@@ -635,23 +644,28 @@ export default function VideoStudio({
});
setUploadedImageUrl(url);
- // Clear v2v if active
- setUploadedVideoUrl(null);
- setUploadedVideoName(null);
- setV2vMode(false);
-
- if (!imageMode) {
- const currentT2V = t2vModels.find((m) => m.id === selectedModel);
- const sibling = currentT2V?.family
- ? i2vModels.find((m) => m.family === currentT2V.family)
- : null;
- const target = sibling || i2vModels[0];
- setImageMode(true);
- setSelectedModel(target.id);
- setSelectedModelName(target.name);
- applyControlsForModel(target.id, true, false);
+ // Motion-control v2v: image is a second input, not a mode switch
+ if (isMotionControlSelection(selectedModel, v2vMode)) {
+ setPromptDisabled(false);
+ } else {
+ // Clear v2v if active
+ setUploadedVideoUrl(null);
+ setUploadedVideoName(null);
+ setV2vMode(false);
+
+ if (!imageMode) {
+ const currentT2V = t2vModels.find((m) => m.id === selectedModel);
+ const sibling = currentT2V?.family
+ ? i2vModels.find((m) => m.family === currentT2V.family)
+ : null;
+ const target = sibling || i2vModels[0];
+ setImageMode(true);
+ setSelectedModel(target.id);
+ setSelectedModelName(target.name);
+ applyControlsForModel(target.id, true, false);
+ }
+ setPromptDisabled(false);
}
- setPromptDisabled(false);
} catch (err) {
console.error("[VideoStudio] Image upload failed:", err);
alert(`Image upload failed: ${err.message}`);
@@ -665,6 +679,8 @@ export default function VideoStudio({
const clearImageUpload = () => {
setUploadedImageUrl(null);
setUploadedEndImageUrl(null);
+ // Motion-control v2v: keep model and video; just drop the image
+ if (isMotionControlSelection(selectedModel, v2vMode)) return;
setImageMode(false);
const first = t2vModels[0];
setSelectedModel(first.id);
@@ -716,18 +732,23 @@ export default function VideoStudio({
setUploadedVideoUrl(url);
setUploadedVideoName(file.name);
- // Clear image mode if active
- if (imageMode) {
- setUploadedImageUrl(null);
- setImageMode(false);
+ if (isMotionControlSelection(selectedModel, v2vMode)) {
+ // Already in motion-control mode — keep model and image, allow prompt
+ setPromptDisabled(false);
+ } else {
+ // Default v2v flow (e.g. watermark remover) — auto-pick the first v2v model
+ if (imageMode) {
+ setUploadedImageUrl(null);
+ setImageMode(false);
+ }
+ setV2vMode(true);
+ const firstV2V = v2vModels[0];
+ setSelectedModel(firstV2V.id);
+ setSelectedModelName(firstV2V.name);
+ applyControlsForModel(firstV2V.id, false, true);
+ setPrompt("");
+ setPromptDisabled(true);
}
- setV2vMode(true);
- const firstV2V = v2vModels[0];
- setSelectedModel(firstV2V.id);
- setSelectedModelName(firstV2V.name);
- applyControlsForModel(firstV2V.id, false, true);
- setPrompt("");
- setPromptDisabled(true);
} catch (err) {
console.error("[VideoStudio] Video upload failed:", err);
alert(`Video upload failed: ${err.message}`);
@@ -755,13 +776,22 @@ export default function VideoStudio({
if (isV2V) {
setV2vMode(true);
setImageMode(false);
- setUploadedImageUrl(null);
- setUploadedImagePreview(null);
+ const isMC = !!m.imageField;
+ if (!isMC) {
+ // Single-input v2v (watermark remover etc.) — drop any image
+ setUploadedImageUrl(null);
+ setUploadedImagePreview(null);
+ }
setSelectedModel(m.id);
setSelectedModelName(m.name);
applyControlsForModel(m.id, false, true);
- setPrompt("");
- setPromptDisabled(true);
+ if (isMC) {
+ // Motion-control: prompt is editable, video+image are needed
+ setPromptDisabled(false);
+ } else {
+ setPrompt("");
+ setPromptDisabled(true);
+ }
} else {
if (v2vMode) {
setV2vMode(false);
@@ -801,6 +831,14 @@ export default function VideoStudio({
alert("Please upload a video first.");
return;
}
+ if (currentModel?.imageField && !uploadedImageUrl) {
+ alert("Please upload a reference image for motion control.");
+ return;
+ }
+ if (currentModel?.promptRequired && !trimmedPrompt) {
+ alert("Please describe the motion you want.");
+ return;
+ }
} else if (isExtendMode) {
if (!lastGenerationId) {
alert(
@@ -829,11 +867,19 @@ export default function VideoStudio({
let res;
if (v2vMode) {
- // V2V: use generateVideo with video_url (the v2v models use the video endpoint)
- res = await generateVideo(apiKey, {
+ // V2V: dedicated processV2V handles single-input tools (e.g. watermark
+ // remover) and motion-control models (which take video + image + prompt)
+ const v2vParams = {
model: selectedModel,
video_url: uploadedVideoUrl,
- });
+ };
+ if (currentModel?.imageField && uploadedImageUrl) {
+ v2vParams.image_url = uploadedImageUrl;
+ }
+ if (currentModel?.hasPrompt && trimmedPrompt) {
+ v2vParams.prompt = trimmedPrompt;
+ }
+ res = await processV2V(apiKey, v2vParams);
if (!res?.url) throw new Error("No video URL returned by API");
const genId = res.id || Date.now().toString();
@@ -842,7 +888,7 @@ export default function VideoStudio({
const entry = {
id: genId,
url: res.url,
- prompt: "",
+ prompt: currentModel?.hasPrompt ? trimmedPrompt : "",
model: selectedModel,
timestamp: new Date().toISOString(),
};
@@ -852,7 +898,7 @@ export default function VideoStudio({
onGenerationComplete({
url: res.url,
model: selectedModel,
- prompt: "",
+ prompt: currentModel?.hasPrompt ? trimmedPrompt : "",
type: "video",
});
} else if (imageMode) {
@@ -1021,7 +1067,11 @@ export default function VideoStudio({
const isExtendMode = currentModelObj?.requiresRequestId;
const promptPlaceholder = v2vMode
- ? "Video ready — click Generate to remove watermark"
+ ? currentModelObj?.imageField
+ ? currentModelObj?.promptRequired
+ ? "Describe the motion"
+ : "Describe the motion (optional)"
+ : "Video ready — click Generate to remove watermark"
: imageMode
? "Describe the motion or effect (optional)"
: isExtendMode
diff --git a/packages/studio/src/models.js b/packages/studio/src/models.js
index 6a93c794..880f8278 100644
--- a/packages/studio/src/models.js
+++ b/packages/studio/src/models.js
@@ -3846,7 +3846,7 @@ export const i2iModels = [
{
"id": "higgsfield-soul-image-to-image",
"name": "Higgsfield Soul Image To Image",
- "endpoint": "higgsfield-soul-image-to-image",
+ "endpoint": "hf-soul-image-to-image",
"family": "higgsfield",
"imageField": "image_url",
"hasPrompt": true,
@@ -6586,7 +6586,7 @@ export const i2vModels = [
{
"id": "higgsfield-dop-image-to-video",
"name": "Higgsfield Dop Image To Video",
- "endpoint": "higgsfield-dop-image-to-video",
+ "endpoint": "hf-dop-image-to-video",
"family": "higgsfield",
"imageField": "image_url",
"hasPrompt": true,
@@ -8237,6 +8237,7 @@ export const v2vModels = [
"videoField": "video_url",
"imageField": "image_url",
"hasPrompt": true,
+ "promptRequired": true,
"description": "Kling v2.6 Pro Motion Control allows precise control over camera movement, subject motion, and scene dynamics during video generation."
},
{
diff --git a/packages/studio/src/muapi.js b/packages/studio/src/muapi.js
index 9550b454..690e1eb7 100644
--- a/packages/studio/src/muapi.js
+++ b/packages/studio/src/muapi.js
@@ -131,6 +131,20 @@ export async function generateMarketingStudioAd(apiKey, params) {
return submitAndPoll(endpoint, payload, apiKey, params.onRequestId, 900);
}
+export async function processV2V(apiKey, params) {
+ const modelInfo = getV2VModelById(params.model);
+ const endpoint = modelInfo?.endpoint || params.model;
+ const videoField = modelInfo?.videoField || 'video_url';
+ const payload = { [videoField]: params.video_url };
+ if (modelInfo?.imageField && params.image_url) {
+ payload[modelInfo.imageField] = params.image_url;
+ }
+ if (modelInfo?.hasPrompt && params.prompt) {
+ payload.prompt = params.prompt;
+ }
+ return submitAndPoll(endpoint, payload, apiKey, params.onRequestId, 900);
+}
+
export async function processLipSync(apiKey, params) {
const modelInfo = getLipSyncModelById(params.model);
const endpoint = modelInfo?.endpoint || params.model;
diff --git a/src/components/VideoStudio.js b/src/components/VideoStudio.js
index 8418a63c..7f6ebbcf 100644
--- a/src/components/VideoStudio.js
+++ b/src/components/VideoStudio.js
@@ -68,6 +68,7 @@ export function VideoStudio() {
};
const getCurrentModes = (id) => getModesForModel(id);
const getCurrentModel = () => getCurrentModels().find(m => m.id === selectedModel);
+ const isMotionControlV2V = () => v2vMode && !!getCurrentModel()?.imageField;
const getQualitiesForModel = (id) => {
const model = getCurrentModels().find(m => m.id === id);
return model?.inputs?.quality?.enum || [];
@@ -122,6 +123,14 @@ export function VideoStudio() {
anchorContainer: container,
onSelect: ({ url }) => {
uploadedImageUrl = url;
+ // Motion-control v2v: image is a second input alongside the video, not a mode switch
+ if (isMotionControlV2V()) {
+ textarea.disabled = false;
+ textarea.placeholder = uploadedVideoUrl
+ ? (getCurrentModel()?.promptRequired ? 'Describe the motion' : 'Describe the motion (optional)')
+ : 'Now upload a reference video using the 🎥 button';
+ return;
+ }
// Clear video mode if active
if (v2vMode) {
uploadedVideoUrl = null;
@@ -145,6 +154,8 @@ export function VideoStudio() {
},
onClear: () => {
uploadedImageUrl = null;
+ // Motion-control v2v: keep the model selection; just lose the image
+ if (isMotionControlV2V()) return;
imageMode = false;
// Clearing the start frame invalidates any selected end frame.
uploadedEndImageUrl = null;
@@ -258,8 +269,13 @@ export function VideoStudio() {
const clearVideoUpload = () => {
uploadedVideoUrl = null;
- v2vMode = false;
showVideoIcon();
+ // Motion-control v2v: keep the model and image; user can re-upload a video
+ if (isMotionControlV2V()) {
+ textarea.placeholder = 'Upload a reference video using the 🎥 button';
+ return;
+ }
+ v2vMode = false;
selectedModel = allT2V[0].id;
selectedModelName = allT2V[0].name;
document.getElementById('v-model-btn-label').textContent = selectedModelName;
@@ -293,19 +309,27 @@ export function VideoStudio() {
uploadedVideoUrl = url;
showVideoReady(file.name);
- // Switch to v2v mode
- if (imageMode) {
- picker.reset();
- uploadedImageUrl = null;
- imageMode = false;
+ // If a motion-control v2v model is already selected, keep it and the image upload
+ if (isMotionControlV2V()) {
+ textarea.disabled = false;
+ textarea.placeholder = uploadedImageUrl
+ ? (getCurrentModel()?.promptRequired ? 'Describe the motion' : 'Describe the motion (optional)')
+ : 'Now upload a reference image using the 🖼 button';
+ } else {
+ // Default v2v flow (e.g. watermark remover) — auto-pick the first v2v model
+ if (imageMode) {
+ picker.reset();
+ uploadedImageUrl = null;
+ imageMode = false;
+ }
+ v2vMode = true;
+ selectedModel = v2vModels[0].id;
+ selectedModelName = v2vModels[0].name;
+ document.getElementById('v-model-btn-label').textContent = selectedModelName;
+ updateControlsForModel(selectedModel);
+ textarea.placeholder = 'Video ready — click Generate to remove watermark';
+ textarea.disabled = true;
}
- v2vMode = true;
- selectedModel = v2vModels[0].id;
- selectedModelName = v2vModels[0].name;
- document.getElementById('v-model-btn-label').textContent = selectedModelName;
- updateControlsForModel(selectedModel);
- textarea.placeholder = 'Video ready — click Generate to remove watermark';
- textarea.disabled = true;
} catch (err) {
console.error('[VideoStudio] Video upload failed:', err);
showVideoIcon();
@@ -550,7 +574,7 @@ export function VideoStudio() {
<div class="w-10 h-10 ${iconColor} border border-white/5 rounded-xl flex items-center justify-center font-black text-sm shadow-inner uppercase">${m.name.charAt(0)}</div>
<div class="flex flex-col gap-0.5">
<span class="text-xs font-bold text-white tracking-tight">${m.name}</span>
- ${isV2V ? '<span class="text-[9px] text-orange-400/70">Upload a video to use</span>' : ''}
+ ${isV2V ? `<span class="text-[9px] text-orange-400/70">${m.imageField ? 'Upload a video and image' : 'Upload a video to use'}</span>` : ''}
</div>
</div>
${selectedModel === m.id ? '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="#d9ff00" stroke-width="4"><polyline points="20 6 9 17 4 12"/></svg>' : ''}
@@ -561,14 +585,25 @@ export function VideoStudio() {
// Switch to v2v mode
v2vMode = true;
imageMode = false;
- picker.reset();
- uploadedImageUrl = null;
+ const isMC = !!m.imageField;
+ if (!isMC) {
+ // Single-input v2v (watermark remover etc.) — drop any image
+ picker.reset();
+ uploadedImageUrl = null;
+ }
selectedModel = m.id;
selectedModelName = m.name;
document.getElementById('v-model-btn-label').textContent = selectedModelName;
updateControlsForModel(selectedModel);
- textarea.placeholder = 'Upload a video using the 🎥 button, then click Generate';
- textarea.disabled = true;
+ if (isMC) {
+ textarea.placeholder = m.promptRequired
+ ? 'Upload a reference video and image, then describe the motion'
+ : 'Upload a reference video and image, then describe the motion (optional)';
+ textarea.disabled = false;
+ } else {
+ textarea.placeholder = 'Upload a video using the 🎥 button, then click Generate';
+ textarea.disabled = true;
+ }
} else {
// Leaving v2v mode if was in it
if (v2vMode) {
@@ -1054,6 +1089,14 @@ export function VideoStudio() {
alert('Please upload a video first.');
return;
}
+ if (model?.imageField && !uploadedImageUrl) {
+ alert('Please upload a reference image for motion control.');
+ return;
+ }
+ if (model?.promptRequired && !prompt) {
+ alert('Please describe the motion you want.');
+ return;
+ }
} else if (isExtendMode) {
if (!lastGenerationId) {
alert('No Seedance 2.0 generation found to extend. Generate a video first.');
@@ -1132,14 +1175,17 @@ export function VideoStudio() {
}
if (v2vMode) {
- const res = await muapi.processV2V({ model: selectedModel, video_url: uploadedVideoUrl, onRequestId });
+ const v2vParams = { model: selectedModel, video_url: uploadedVideoUrl, onRequestId };
+ if (model?.imageField && uploadedImageUrl) v2vParams.image_url = uploadedImageUrl;
+ if (model?.hasPrompt && prompt) v2vParams.prompt = prompt;
+ const res = await muapi.processV2V(v2vParams);
console.log('[VideoStudio] V2V response:', res);
if (res && res.url) {
if (capturedRequestId) removePendingJob(capturedRequestId);
const genId = res.id || capturedRequestId || Date.now().toString();
lastGenerationId = null;
lastGenerationModel = null;
- addToHistory({ id: genId, url: res.url, prompt: '', model: selectedModel, timestamp: new Date().toISOString() });
+ addToHistory({ id: genId, url: res.url, prompt: model?.hasPrompt ? prompt : '', model: selectedModel, timestamp: new Date().toISOString() });
showVideoInCanvas(res.url, selectedModel);
} else {
throw new Error('No video URL returned by API');
diff --git a/src/lib/muapi.js b/src/lib/muapi.js
index 270b8d3e..ebabe9c0 100644
--- a/src/lib/muapi.js
+++ b/src/lib/muapi.js
@@ -407,10 +407,14 @@ export class MuapiClient {
}
/**
- * Processes a video through a Video-to-Video model (e.g. watermark remover).
+ * Processes a video through a Video-to-Video model.
+ * Single-input tools (e.g. watermark remover) only need `video_url`.
+ * Motion-control models additionally need `image_url` and (often) `prompt`.
* @param {Object} params
* @param {string} params.model - v2vModel id
* @param {string} params.video_url - The uploaded video URL
+ * @param {string} [params.image_url] - Reference image URL (motion-control models)
+ * @param {string} [params.prompt] - Motion description (motion-control models)
*/
async processV2V(params) {
const key = this.getKey();
@@ -421,6 +425,13 @@ export class MuapiClient {
const videoField = modelInfo?.videoField || 'video_url';
const finalPayload = { [videoField]: params.video_url };
+ if (modelInfo?.imageField && params.image_url) {
+ finalPayload[modelInfo.imageField] = params.image_url;
+ }
+ if (modelInfo?.hasPrompt && params.prompt) {
+ finalPayload.prompt = params.prompt;
+ }
+
console.log('[Muapi] V2V Request:', url);
console.log('[Muapi] V2V Payload:', finalPayload);