Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Prompt enhancement: show a rewrite as the scene and its sound

Bowen Xueb341d12cff49
diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs
index d237cbe..f7c357c 100644
--- a/scripts/test_freevideo_entry.mjs
+++ b/scripts/test_freevideo_entry.mjs
@@ -116,6 +116,7 @@ test('main browser entry registers both views and preserves node hooks', async t
source = source.replace("import { beginWorkspaceRestore, finishWorkspaceRestore, waitForLauncherRestore } from './workspace_restore.js';", 'const {beginWorkspaceRestore,finishWorkspaceRestore,waitForLauncherRestore} = globalThis.__freevideoEntryTest;');
source = source.replace("import { openEditorView } from './editor_loader.js';", 'const openEditorView = (...args) => globalThis.__freevideoEntryTest.openEditorView?.(...args);');
source = source.replace("from './issue_button.js'", `from '${new URL('../web/issue_button.js', import.meta.url).href}'`);
+ source = source.replace("from './prompt_sections.js'", `from '${new URL('../web/prompt_sections.js', import.meta.url).href}'`);
source = source
.replace("import { createErrorPanel, errorText, createErrorReport, adviceFor } from './error_panel.js';", 'const {createErrorPanel,errorText,createErrorReport,adviceFor} = globalThis.__freevideoEntryTest;')
.replace('import { app } from "../../scripts/app.js";', 'const {app} = globalThis.__freevideoEntryTest;')
diff --git a/web/error_panel.js b/web/error_panel.js
index 806f953..7910e1f 100644
--- a/web/error_panel.js
+++ b/web/error_panel.js
@@ -201,10 +201,12 @@ function reportRedactor(detail, context) {
for (const node of list(context.graph?._nodes)) {
remember(node.properties?.freevideo_prompt_versions?.original);
remember(node.properties?.freevideo_prompt_versions?.rewritten);
+ remember(node.properties?.freevideo_prompt_versions?.model);
for (const widget of list(node.widgets)) remember(widget.value);
}
remember(context.node?.properties?.freevideo_prompt_versions?.original);
remember(context.node?.properties?.freevideo_prompt_versions?.rewritten);
+ remember(context.node?.properties?.freevideo_prompt_versions?.model);
for (const widget of list(context.node?.widgets)) remember(widget.value);
for (const node of Object.values(record(detail?.response?.node_errors ?? detail?.node_errors)))
for (const reason of list(node?.errors)) remember(reason?.extra_info?.received_value);
diff --git a/web/freevideo.js b/web/freevideo.js
index dd9fb93..4b696b9 100644
--- a/web/freevideo.js
+++ b/web/freevideo.js
@@ -18,6 +18,7 @@ import { openFreeVideoWorkflow } from './launcher.js';
import { agentInterfaceState } from './agent_connect.js';
import { attachReferencePicker, referenceItems, syncReferencePrompt, referenceInputs, mediaRows, mediaNodeOf } from './prompt_references.js';
import { expandCast } from './cast_mention.js';
+import { requestText } from './prompt_sections.js';
import { outputDownloadURL } from './output_download.js';
import { shareButton } from './share.js';
import { reportFileName } from './report_issue.js';
@@ -821,12 +822,14 @@ app.registerExtension({
force.value = false; force.serializeValue = () => false;
}
// Characters mentioned with @ travel as <Subject N> plus their definitions; the draft keeps the typed names.
+ // A rewrite the box shows in short goes back in the model's sections.
const prompt = this.widgets?.find(w => w.name === 'text');
if (prompt) {
const owner = this;
// Replace person asks for its prefix here, where every way of running the node reads the text.
prompt.serializeValue = () => {
- const body = expandCast(prompt.value, mediaRows(owner), {continued: Boolean(owner.freevideoContinues?.())}).text;
+ const body = expandCast(requestText(prompt.value, owner.properties?.freevideo_prompt_versions),
+ mediaRows(owner), {continued: Boolean(owner.freevideoContinues?.())}).text;
return swapState(mediaNodeOf(owner)) ? withPrefix(body) : body;
};
}
diff --git a/web/health.js b/web/health.js
index fc54131..6a1687e 100644
--- a/web/health.js
+++ b/web/health.js
@@ -16,7 +16,7 @@ const launchVisit = query.get('freevideo') === 'launch';
export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js',
'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js',
'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js',
- 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js',
+ 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_sections.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js',
'result_actions.js', 'result_reuse.js', 'sampling_effort.js', 'setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'replace_person.js', 'replace_state.js', 'optional_inputs.js', 'studio_queue.js', 'studio_size.js',
'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js', 'whitebox_encode.js', 'whitebox_mp4.js', 'whitebox_recording.js', 'whitebox_render.js', 'whitebox_scenes.js', 'whitebox_worker.js'];
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
diff --git a/web/prompt_enhance.js b/web/prompt_enhance.js
index 5ad99eb..9b152de 100644
--- a/web/prompt_enhance.js
+++ b/web/prompt_enhance.js
@@ -14,16 +14,18 @@ export class PromptVersions {
edit(text, restart = false) {
this.value.edits = (this.value.edits || 0) + 1;
if (restart && this.value.selected === 'rewritten') {
- Object.assign(this.value, {original: text, rewritten: '', selected: 'original', context: '', useOriginal: false});
+ Object.assign(this.value, {original: text, rewritten: '', model: '', selected: 'original', context: '', useOriginal: false});
return;
}
this.value[this.value.selected] = text;
if (this.value.selected === 'original') this.value.context = '';
}
key(context) { return JSON.stringify({text: this.value.original, ...context}); }
- accept(key, context, text) {
+ // `model` is the rewrite as the model wrote it, kept when the box shows it in a shorter form.
+ accept(key, context, text, model = text) {
if (this.key(context) !== key || !this.value.enabled || this.value.useOriginal) return false;
- this.value.rewritten = text; this.value.selected = 'rewritten'; this.value.context = key;
+ this.value.rewritten = text; this.value.model = model === text ? '' : model;
+ this.value.selected = 'rewritten'; this.value.context = key;
return true;
}
fresh(context) { return Boolean(this.value.rewritten && this.value.context === this.key(context)); }
@@ -67,8 +69,9 @@ export function promptPrestart(api, nodeId, enabled, alive = () => true) {
// An empty box is never rewritten, the untouched starter prompt included:
// enhancement waits for the user's own scene.
// `prepare` turns the user's text into what the model reads (character mentions written out with their pictures);
-// `restore` turns the rewrite back into what the box shows (the mentions written as the user wrote them).
-export function createPromptEnhancer({node, input, editor, api, context, setText, t, starter = '', prepare = text => text, restore = text => text, onReport = () => {}, onChange = () => {}, issueButton = null}) {
+// `restore` turns the rewrite back into the user's terms (the mentions written as the user wrote them), and `display`
+// into what the box shows; the restored rewrite is kept for the request (see prompt_sections.js).
+export function createPromptEnhancer({node, input, editor, api, context, setText, t, starter = '', prepare = text => text, restore = text => text, display = text => text, onReport = () => {}, onChange = () => {}, issueButton = null}) {
const saved = node.properties?.freevideo_prompt_versions;
// Drafts saved before the starter became a placeholder hold it as the original.
const versions = new PromptVersions(starter && saved?.original === starter ? {...saved, original: ''} : saved, input.value);
@@ -355,7 +358,8 @@ export function createPromptEnhancer({node, input, editor, api, context, setText
}
const result = await poll('rewrite', {text: prepare(original), ...scene});
if (disposed || cancelling) throw new Error('cancelled');
- if (edits !== versions.value.edits || !versions.accept(key, context(), restore(result.text))) throw new Error('stale');
+ const restored = restore(result.text);
+ if (edits !== versions.value.edits || !versions.accept(key, context(), display(restored), restored)) throw new Error('stale');
apply(); show('');
};
running = Promise.resolve().then(execute).catch(error => {
diff --git a/web/prompt_references.css b/web/prompt_references.css
index ed5a861..b92ca2c 100644
--- a/web/prompt_references.css
+++ b/web/prompt_references.css
@@ -1,5 +1,5 @@
.fv-prompt-editor{position:relative;display:flex;flex-direction:column;height:170px;min-height:148px;max-height:340px;resize:vertical;overflow:hidden;background:var(--fv-card)}
-.fv-studio .fv-prompt-editor textarea{flex:1;min-height:0;height:0;max-height:none;resize:none;padding-bottom:8px;outline:none}
+.fv-studio .fv-prompt-editor textarea{flex:1;min-height:0;height:0;max-height:none;resize:none;padding-bottom:14px;outline:none}
.fv-prompt-tools{display:flex;flex:none;align-items:center;justify-content:flex-end;height:36px;padding-right:9px}
.fv-studio .fv-prompt-draft-note{min-height:27px;height:27px;margin:0 auto 0 9px;padding:0 9px;border:0;background:transparent;color:var(--fv-warning);font-size:12px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.fv-studio .fv-prompt-draft-note[hidden]{display:none}
diff --git a/web/prompt_sections.js b/web/prompt_sections.js
new file mode 100644
index 0000000..e42a419
--- /dev/null
+++ b/web/prompt_sections.js
@@ -0,0 +1,107 @@
+// A rewrite in H3's sections, as the prompt box shows it: the scene, then a "Sound:" paragraph (any music joins it).
+// The section names, the summary and the retention analysis stay out of sight; `modelText` puts them back when the
+// prompt is sent, so an untouched rewrite reaches the model exactly as the model wrote it.
+const ORDERS = [
+ ['subject_definitions', 'summary', 'retention_analysis', 'detailed_description', 'overall_soundscape', 'non_diegetic_music'],
+ ['integrated_multimodal_description', 'overall_soundscape', 'non_diegetic_music'],
+];
+const SCENE = new Set(['detailed_description', 'integrated_multimodal_description']);
+const LABELS = [['en', 'Sound: '], ['zh', '声音:']];
+const NO_MUSIC = /^n\/?a\.?$/i;
+// "[Shot 2] At 00:03.000, ..." in the model's words; "Shot 2 (0:03): ..." or "镜头 2(0:03):..." in the box.
+const AT = /^At (\d{1,2}):(\d{2}(?:\.\d+)?),[ \t]*/;
+const SHOT_HEAD = /^(?:Shot|镜头) (\d+)(?: ?[((](\d{1,2}):(\d{2}(?:\.\d+)?)[))])?[::][ \t]*/;
+
+// [[name, body]] when the text is exactly H3's sections in order (subject definitions may be absent), else null.
+function sections(text) {
+ const parts = String(text ?? '').trim().split(/\n\n+(?=[a-z_]+:)/);
+ const rows = parts.map(part => /^([a-z_]+):[ \t]*([\s\S]*)$/.exec(part)).map(match => match && [match[1], match[2].trim()]);
+ if (!rows.length || rows.some(row => !row)) return null;
+ const names = rows.map(([name]) => name).join();
+ return [ORDERS[0], ORDERS[0].slice(1), ORDERS[1]].some(order => order.join() === names) ? rows : null;
+}
+
+// The scene's shots in order as {at, minutes, seconds, text} (a scene without markers is one shot), or null when the
+// markers are not [Shot 1], [Shot 2], ... from the start.
+function shots(scene) {
+ const pieces = scene.split(/[ \t]*\[Shot (\d+)\][ \t]*/);
+ if (pieces.length === 1) return [{at: '', minutes: 0, seconds: '00', text: scene.trim()}];
+ if (pieces[0].trim()) return null;
+ const list = [];
+ for (let i = 1; i < pieces.length; i += 2) {
+ if (Number(pieces[i]) !== list.length + 1) return null;
+ const at = AT.exec(pieces[i + 1]);
+ list.push({at: at ? at[0].trim() : '', minutes: at ? Number(at[1]) : 0, seconds: at ? at[2] : '00', text: pieces[i + 1].slice(at ? at[0].length : 0).trim()});
+ }
+ return list;
+}
+
+const clock = (minutes, seconds) => `${minutes}:${seconds.replace(/(\.\d*?)0+$/, '$1').replace(/\.$/, '')}`;
+
+// What the box shows for a rewrite, or null to show it as it is. A single shot shows without its marker; several shots
+// each start a paragraph with their number and start time.
+export function shownRewrite(text, language = 'en') {
+ const rows = sections(text);
+ if (!rows) return null;
+ const get = name => rows.find(([key]) => key === name)?.[1] ?? '';
+ const list = shots(rows.find(([key]) => SCENE.has(key))[1]);
+ if (!list) return null;
+ const zh = language === 'zh';
+ const scene = list.length === 1 ? list[0].text : list.map((shot, index) => {
+ const time = clock(shot.minutes, shot.seconds);
+ return (zh ? `镜头 ${index + 1}(${time}):` : `Shot ${index + 1} (${time}): `) + shot.text;
+ }).join('\n\n');
+ const music = get('non_diegetic_music');
+ const sound = [get('overall_soundscape'), NO_MUSIC.test(music) ? '' : music].filter(Boolean).join(' ');
+ return scene + '\n\n' + (zh ? LABELS[1][1] : LABELS[0][1]) + sound;
+}
+
+// The scene section for the scene part of an edited box. Paragraphs that open with a shot number become that shot,
+// keeping the model's own time when the shown one is unchanged; anything before the first is the first shot.
+function sceneFromBox(box, original) {
+ const shot = [];
+ for (const paragraph of box.split(/\n\n+/)) {
+ const head = SHOT_HEAD.exec(paragraph);
+ if (head || !shot.length) shot.push({head, text: paragraph.slice(head ? head[0].length : 0).trim()});
+ else shot[shot.length - 1].text += ' ' + paragraph.trim();
+ }
+ if (shot.length === 1 && !shot[0].head) return '[Shot 1] ' + shot[0].text;
+ return shot.map(({head, text}, index) => {
+ const old = original?.[index];
+ let at = '';
+ if (head?.[2] !== undefined) {
+ const time = clock(Number(head[2]), head[3]);
+ at = old && clock(old.minutes, old.seconds) === time ? old.at
+ : time === '0:00' && index === 0 ? '' : `At ${head[2].padStart(2, '0')}:${head[3]},`;
+ } else if (old?.at) at = old.at;
+ return `[Shot ${index + 1}] ${at ? at + ' ' : ''}${text}`;
+ }).join(' ');
+}
+
+// The text sent to the model for what the box holds, given the rewrite it was shown from. An edited box keeps the hidden
+// definitions and retention analysis but drops the summary, which may no longer match the edited scene.
+export function modelText(shown, rewrite) {
+ const rows = sections(rewrite);
+ if (!rows || LABELS.some(([code]) => shownRewrite(rewrite, code) === shown)) return rows ? rewrite : shown;
+ const get = name => rows.find(([key]) => key === name)?.[1] ?? '';
+ const [sceneName, sceneText] = rows.find(([key]) => SCENE.has(key));
+ const box = String(shown ?? '').trim();
+ // The last paragraph that opens with "Sound:" or "声音:" (in either language) is the sound; the music goes on while
+ // its words still end it. Without one, the whole box is the scene and no sound is described, as the box shows.
+ let scene = box, soundscape = 'N/A', music = get('non_diegetic_music');
+ const heads = [...box.matchAll(/(?:^|\n\n+)(?:Sound:|声音:)[ \t]*/g)];
+ const head = heads[heads.length - 1];
+ let sound = head ? box.slice(head.index + head[0].length).trim() : '';
+ if (head) scene = box.slice(0, head.index).trim();
+ if (!NO_MUSIC.test(music) && sound.endsWith(music)) sound = sound.slice(0, -music.length).trim();
+ else music = 'N/A';
+ soundscape = sound || soundscape;
+ const keep = rows.filter(([name]) => name === 'subject_definitions' || name === 'retention_analysis');
+ return [...keep, [sceneName, sceneFromBox(scene, shots(sceneText))], ['overall_soundscape', soundscape], ['non_diegetic_music', music]]
+ .map(([name, body]) => `${name}: ${body}`).join('\n\n');
+}
+
+// The prompt widget's text for the request: a rewrite the box shows in short goes back in the model's sections.
+export function requestText(text, versions) {
+ return versions?.selected === 'rewritten' && versions.model ? modelText(text, versions.model) : text;
+}
diff --git a/web/studio.js b/web/studio.js
index f90ddba..3049486 100644
--- a/web/studio.js
+++ b/web/studio.js
@@ -34,6 +34,7 @@ import { continuationUpscale, createSamplingEffort, effortFor, effortsFrom, pass
import { resultActions, previewBadge, icon, actionsUnavailable } from './result_actions.js';
import { framesOf } from './result_reuse.js';
import { createPromptEnhancer } from './prompt_enhance.js';
+import { shownRewrite } from './prompt_sections.js';
import { createRequestAssets, enabledLoras, referenceFiles } from './request_assets.js';
import { promptDraftTrouble, rememberPromptDraft, restorePromptDraft, savePromptDraft, trackPromptDraft, watchPromptDraftTrouble } from './prompt_draft.js';
import { canvasDimensions, imageSizeSelection } from './studio_size.js';
@@ -821,7 +822,7 @@ export function openStudio(node, {extendFrom = null} = {}) {
const enhancer = createPromptEnhancer({node, input: prompt, editor: promptEditor, api, t, starter, issueButton,
// One rewrite runs at a time, so the subjects of the text it was given are the ones its numbers refer to.
prepare: text => { const out = expandCast(text, mediaRows(node), {continued: Boolean(extension.source)}); rewriteSubjects = out.subjects; return out.text; },
- restore: text => restoreCast(text, rewriteSubjects),
+ restore: text => restoreCast(text, rewriteSubjects), display: text => shownRewrite(text, t('en', 'zh')) ?? text,
// The rewrite has its own report; the video's report entry is not touched.
onReport: row => { node.freevideoPromptReportId = row.rewrite_report_id || row.report_id; },
onChange: () => { rememberPromptDraft(api, node); requestAssets?.refresh(); },