Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Editing view: a block can carry a short note, shown on the strip under the pictures

Bowen Xue9fd875ba4a3c
diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.py
index 6af6b6f..2f793a3 100644
--- a/freevideo_engine/agent/editor_runtime.py
+++ b/freevideo_engine/agent/editor_runtime.py
@@ -137,6 +137,7 @@ globalThis.__batch = text => {
case 'move': record = actions.moveParts(record, op.clip_ids, {frames: op.frames, track: null}); break;
case 'describe': record = model.describe(record, actions.linkedIds(record, op.clip_ids), op.text, by); break;
case 'set_ref': record = model.setRef(record, actions.linkedIds(record, op.clip_ids), op.ref); break;
+ case 'note': record = model.setNote(record, actions.linkedIds(record, op.clip_ids), op.text); break;
case 'use_version': record = model.restoreVersion(record, op.clip_id, op.index, by); break;
// One undo on the page; rejects as subtitles / range / text / locked / overlap.
case 'subtitles': record = texts.addSubtitles(record, op.cues, {replace: op.replace === true}); break;
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index ad420ec..3d683b8 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -108,6 +108,7 @@ GUIDE = {
'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.',
'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
+ 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
],
'project_routes': [dict(operation=name, method=method, path=path, description=summary)
diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py
index afd7180..26a0252 100644
--- a/freevideo_engine/agent/projects.py
+++ b/freevideo_engine/agent/projects.py
@@ -33,6 +33,19 @@ RULE_EXPECTED = {
'source': 'a source that has a picture or sound',
'invalid': 'an op named in the operations schema with its fields',
}
+# The note operation shares the page's 'text' code with describe; what it needed is its own.
+NOTE_EXPECTED = 'picture clips and a text of at most 120 characters on one line (an empty text removes the note)'
+# An agent acts on the last answer it read, so the answer that brings new picture clips asks for their notes.
+NOTE_NEXT = ('Picture clips %s have no note yet. Write each one now with a note operation: in the user\'s language, '
+ 'what is in the picture, subject first; not its position and not its progress.')
+
+
+def note_next(result, project, ids):
+ missing = [clip['id'] for track in project['tracks'] if track['kind'] == 'video' for clip in track['clips']
+ if clip['id'] in ids and not clip.get('note')]
+ if missing:
+ result['next'] = NOTE_NEXT % ', '.join(missing)
+ return result
def new_project(given):
@@ -168,6 +181,8 @@ class Projects:
def _rejected(self, error, stored, path, received, *, operation=None):
if error['code'] in STATE_CODES:
raise ProjectError(STATE_CODES[error['code']], project=stored, operation=operation)
+ if error['code'] == 'text' and isinstance(received, dict) and received.get('op') == 'note':
+ raise _invalid(path, NOTE_EXPECTED, received)
raise _invalid(path, RULE_EXPECTED.get(error['code'], 'an operation this timeline allows'), received)
# Reads ---------------------------------------------------------------------------------------------
@@ -270,6 +285,7 @@ class Projects:
operations[index] if index is not None else None, operation=index)
saved = self._save(reply['project'], ref=ref, client=client)
result = dict(project=saved, changed_clip_ids=changed_clip_ids(stored, saved), results=reply['results'])
+ note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored)))
self._remember(ref, digest, result, client)
return result
@@ -331,7 +347,8 @@ class Projects:
raise
with self.lock:
self.prepared[job['id']] = saved
- result = dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids'])
+ result = note_next(dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']),
+ saved, set(plan['ids']))
self._remember(ref, digest, result, client)
return result
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 1ca0174..2fa2313 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -753,7 +753,7 @@ def project_schema(name):
if name != 'operations':
return EMPTY
fields = {
- 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end.'),
+ 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end. Then write the new picture clip\'s note.'),
'split': (dict(clip_ids=ids, frame=EDITOR_FRAME), ('clip_ids', 'frame'), 'Cut selected clips and their linked partners. The frame must be inside a selected segment.'),
'trim': (dict(clip_ids=ids, frame=EDITOR_FRAME, edge=enum(('start', 'end'))), ('clip_ids', 'frame', 'edge'), 'Trim a selected edge at the frame. The frame must be inside a selected segment.'),
'remove': (dict(clip_ids=ids), ('clip_ids',), 'Remove selected clips and linked partners. Use clip IDs from this revision.'),
@@ -763,6 +763,7 @@ def project_schema(name):
'move': (dict(clip_ids=ids, frames=number(-10**7, 10**7, integer=True)), ('clip_ids', 'frames'), 'Move selected clips by a signed relative frame offset. Use clip IDs from this revision; linked partners move together.'),
'describe': (dict(clip_ids=ids, text=dict(type='string', maxLength=8000)), ('clip_ids', 'text'), 'Set the description of selected clips. Use clip IDs from this revision and one paragraph of text.'),
'set_ref': (dict(clip_ids=ids, ref=dict(type='string', maxLength=128, pattern=r'^[A-Za-z0-9_-]{0,128}$')), ('clip_ids', 'ref'), 'Set an outside shot reference on selected clips. Use clip IDs from this revision; an empty ref removes it.'),
+ 'note': (dict(clip_ids=ids, text=dict(type='string', maxLength=120)), ('clip_ids', 'text'), 'Write the one line the user sees under selected picture clips; without it they see the start of the generation text. Do it after you add, generate or change a clip: in the user\'s language, what is in the picture, subject first ("Train through morning mist"); a short clip shows only the first few characters. Not the clip\'s position (opening, part 2) and not its progress. Up to 120 characters on one line; an empty text removes it. It never changes the description used for generation.'),
'use_version': (dict(clip_id=CLIP, index=number(0, 49, integer=True)), ('clip_id', 'index'), 'Restore a stored clip version. First get the clip ID and its versions array; index is zero based.'),
'subtitles': (dict(cues=array(obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True), text=dict(type='string', minLength=1, maxLength=400)),
('start', 'frames', 'text')), 1, 500), replace=dict(type='boolean')), ('cues',),
diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py
index 852d9e3..df799b4 100644
--- a/freevideo_engine/comfy_editor.py
+++ b/freevideo_engine/comfy_editor.py
@@ -39,6 +39,8 @@ _CLIP = re.compile(r'c[1-9][0-9]{0,8}')
_TOKEN = re.compile(r'[A-Za-z0-9_-]{1,128}')
_REASON = re.compile(r'[a-z_]{1,80}')
_REF = re.compile(r'[A-Za-z0-9_-]{1,128}')
+# A note is one line: the page refuses the same characters (editor_model.js NOTE_BREAK) and the same ends.
+_NOTE_BREAK = re.compile('[\x00-\x1f\x7f-\x9f\u2028\u2029]')
_STATE = threading.RLock()
_TRANSCODING = threading.Lock()
_FAILED_IMPORTS = {} # terminal statuses awaiting writable storage
@@ -279,6 +281,12 @@ def validate(record):
if not isinstance(raw['ref'], str) or not _REF.fullmatch(raw['ref']):
raise ValueError('Invalid block reference')
clip['ref'] = raw['ref']
+ if 'note' in raw:
+ note = raw['note']
+ if (not _text(note, 120) or not note or note != note.strip() or '\ufeff' in (note[0], note[-1])
+ or _NOTE_BREAK.search(note)):
+ raise ValueError('Invalid block note')
+ clip['note'] = note
if source is None and ('take' in raw or 'made_text' in raw):
raise ValueError('Invalid clip versions')
_made(raw, clip)
diff --git a/freevideo_engine/editor_text.py b/freevideo_engine/editor_text.py
index 0872060..84fea21 100644
--- a/freevideo_engine/editor_text.py
+++ b/freevideo_engine/editor_text.py
@@ -79,7 +79,7 @@ def validate_text_project(raw, validate_media):
_check(all(c.get(k) is None for k in ('source','pending','link','part','job','reason')) and c.get('extend',False) is False
and ('versions' not in c or isinstance(c['versions'],list) and not c['versions']))
_check(type(c.get('in',0)) is int and c.get('in',0) == 0 and type(c.get('length',c['frames'])) is int and c.get('length',c['frames']) == c['frames'])
- _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','take','made_text')))
+ _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','note','take','made_text')))
clip = dict(id=identity, start=c['start'], frames=c['frames'], length=c['frames'], source=None,
link=None, text=text, part=None, pending=None, reason=None, job=None, extend=False, versions=[], text_kind=kind, **{'in':0})
_check(kind != 'subtitle' or 'text_style' not in c)
diff --git a/web/editor_model.js b/web/editor_model.js
index 6703160..26e821d 100644
--- a/web/editor_model.js
+++ b/web/editor_model.js
@@ -24,6 +24,10 @@ const check = (yes, code = 'invalid', message = code) => { if (!yes) reject(code
const ID = /^c[1-9]\d{0,8}$/;
// Who last wrote a block's description (its note on the timeline), and an outside reference (a shot id).
const TEXT_BY = ['user', 'agent'], REF = /^[A-Za-z0-9_-]{1,128}$/;
+// A short line about a block for whoever looks at the timeline; regenerating never reads it.
+// One line: no control or line-break character anywhere, nothing at its ends that either side would strip.
+const NOTE_BREAK = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/;
+const validNote = text => validText(text) && text === text.trim() && !NOTE_BREAK.test(text) && Array.from(text).length >= 1 && Array.from(text).length <= 120;
const WORK = /^\d{4}-\d{2}-\d{2}\/[0-9a-f]{32}$/;
const PENDING = [null, 'draft', 'queued', 'running', 'failed', 'stopped'];
const key = source => `${source.kind}:${source.id}`;
@@ -168,6 +172,7 @@ export function parse(value) {
// Who last wrote the description, and an outside reference (a shot id) for callers.
if (item.text_by !== undefined) { check(TEXT_BY.includes(item.text_by), 'text'); clip.text_by = item.text_by; }
if (item.ref !== undefined) { check(typeof item.ref === 'string' && REF.test(item.ref), 'ref'); clip.ref = item.ref; }
+ if (item.note !== undefined) { check(validNote(item.note), 'text'); clip.note = item.note; }
// The part this one was pulled out of, or the part before it in the same range.
if (item.follows !== undefined) { check(kind === 'video' && typeof item.follows === 'string' && ID.test(item.follows) && item.follows !== item.id, 'follows'); clip.follows = item.follows; }
check(source !== null || (item.take === undefined && item.made_text === undefined), 'versions'); madeFields(item, clip);
@@ -335,14 +340,31 @@ export function setRef(record, ids, ref) {
check(typeof ref === 'string' && (!ref || REF.test(ref)), 'ref');
return onBlocks(record, ids, clip => { if (ref) clip.ref = ref; else delete clip.ref; }, false);
}
+// A note kept with the block; an empty value removes it. A block being generated can still be noted.
+export function setNote(record, ids, text) {
+ check(typeof text === 'string' && (!text.trim() || validNote(text.trim())), 'text');
+ return onBlocks(record, ids, clip => { if (text.trim()) clip.note = text.trim(); else delete clip.note; }, false, {picture: true, kept: true});
+}
// Only an agent's description is marked; no mark means the user wrote it, as in every earlier edit.
-function written(clip, text, by) { clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by; }
-function onBlocks(record, ids, change, refuseBusy) {
+// A note was written about the description it came with, so it goes when the user writes another one.
+function written(clip, text, by) {
+ if (by === 'user' && text !== clip.text) delete clip.note;
+ clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by;
+}
+function onBlocks(record, ids, change, refuseBusy, {picture = false, kept = false} = {}) {
check(Array.isArray(ids) && ids.length > 0 && ids.every(id => typeof id === 'string'), 'id');
for (const id of ids) check(find(record, id), 'missing');
const selected = linkedIds(record, ids), clips = all(record).filter(clip => selected.has(clip.id));
- check(!clips.some(isText), 'text'); assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips);
+ check(!clips.some(isText), 'text');
+ // A sound block takes the change only together with the picture it is linked to.
+ check(!picture || clips.every(clip => trackOf(record, clip.id).kind === 'video' || clip.link), 'text');
+ assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips);
const next = copy(record); for (const clip of all(next)) if (selected.has(clip.id)) change(clip);
+ // The blocks set aside while a part is being replaced take the change too: whichever of them comes back shows it.
+ if (kept) for (const tx of next.replacements || []) for (const row of [...tx.originals, ...tx.clips]) {
+ const shown = clips.filter(clip => trackOf(record, clip.id).id === row.id);
+ for (const clip of row.clips) if (shown.some(v => v.id === clip.id || (clip.start >= v.start && end(clip) <= end(v)))) change(clip);
+ }
return finish(next);
}
export function linkedIds(record, ids) {
diff --git a/web/editor_notes.js b/web/editor_notes.js
index 54bdc67..46e7132 100644
--- a/web/editor_notes.js
+++ b/web/editor_notes.js
@@ -12,8 +12,10 @@ function glyph(d) {
const AGAIN = 'M13.5 2.5v3.5H10M13.2 6A5.5 5.5 0 1 0 13.5 9.5', PENCIL = 'M10.5 2.5l3 3-8 8h-3v-3z';
const generating = clip => ['queued', 'running'].includes(clip?.pending);
const firstLine = text => (text || '').trim().split('\n')[0];
-// Only the picture track grows a description strip, and only once one block has a description.
-export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && clip.text?.trim());
+// A note written about a block (by an agent working on the project) is what the strip shows; without one, the description.
+export const stripText = clip => clip.note || firstLine(clip.text);
+// Only the picture track grows a description strip, and only once one block has something to show.
+export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && stripText(clip));
const agentWrote = record => record.tracks.some(track => track.clips.some(clip => clip.text_by === 'agent'));
// Status words appear only in the hover tip and the panels (Iris 10-11).
export function blockStatus(clip, t, fraction = 0) {
@@ -101,7 +103,7 @@ export function createBlockNotes({state, t, actions}) {
function showTip(item) {
const view = item.element.closest('.fv-ed-track-view'), clip = item.clip; if (!view || !item.element.isConnected) return;
const fraction = Number(item.element.style.getPropertyValue('--fv-ed-progress')) || 0;
- const text = [clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · ');
+ const text = [clip.note || clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · ');
if (!text) return;
if (!tip) { tip = el('div', 'fv-ed-note-tip'); tip.setAttribute('role', 'tooltip'); }
if (tip.parentElement !== view) view.append(tip);
@@ -120,7 +122,7 @@ export function createBlockNotes({state, t, actions}) {
items.set(item.clip.id, item);
item.element.dataset.noted = String(noted);
item.note.hidden = item.number.hidden = !noted;
- item.note.textContent = firstLine(item.clip.text); item.number.textContent = String(index + 1);
+ item.note.textContent = stripText(item.clip); item.number.textContent = String(index + 1);
// The pencil marks blocks an agent leaves alone; a project nobody else wrote in has none.
item.mark.hidden = !agentWrote(record) || !!item.clip.text_by;
}
diff --git a/web/editor_text_model.js b/web/editor_text_model.js
index 5862653..cf42183 100644
--- a/web/editor_text_model.js
+++ b/web/editor_text_model.js
@@ -48,7 +48,7 @@ export function readTextProject(raw, parseMedia) {
check(typeof c.text === 'string' && Array.from(c.text).length <= TEXT_LIMIT && !/[\u0000-\u0008\u000b-\u001f\u007f\ud800-\udfff]/u.test(c.text));
check(['source','pending','link','part','job','reason'].every(k => c[k] == null) && (c.extend === undefined || c.extend === false) && (c.versions === undefined || (Array.isArray(c.versions) && c.versions.length === 0)));
check((c.in === undefined || c.in === 0) && (c.length === undefined || c.length === c.frames));
- check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'take', 'made_text'].some(k => c[k] !== undefined));
+ check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'note', 'take', 'made_text'].some(k => c[k] !== undefined));
const clip = {id: c.id, start: c.start, frames: c.frames, in: 0, length: c.frames, source: null, link: null,
text: c.text, part: null, pending: null, reason: null, job: null, extend: false, versions: [], text_kind: track.kind};
check(track.kind !== 'subtitle' || c.text_style === undefined);