Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Read a voice-only reference video as an audio reference
Bowen Xue5e9f4d009ba2
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.pyindex 61514ba..d4c78e2 100644--- a/freevideo_engine/agent/guide.py+++ b/freevideo_engine/agent/guide.py@@ -74,6 +74,7 @@ GUIDE = {"User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",'Tags are numbered per kind in reference order: <Picture 1>, <Picture 2>, <Video 1>, <Audio 1>. ''For keyframes the first frame is <Picture 1> and the last frame the next picture.',+ 'A reference video marked voice_only is an audio reference: its sound is <Audio N> and it has no <Video N> tag.','Name subjects <Subject 1>, <Subject 2> in subject_definitions and reuse those labels in every section.','Clips are cut to the generated length (at most 15 s); the job result lists any cut.',],diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.pyindex 5fb613e..cc2ce11 100644--- a/freevideo_engine/agent/schema.py+++ b/freevideo_engine/agent/schema.py@@ -139,8 +139,8 @@ def media(image=False, *, local=False, clip=False, video=False, early=False):'A reference video only: it guides the first pass for this many steps and is dropped afterwards, ''including from the second pass. Omitted: the video guides every step.')properties['voice_only'] = {'type': 'boolean', 'description':- 'A reference video used only for its voice: the picture is encoded small and at a low frame rate, '- 'the sound is unchanged. Not with early_steps.'}+ 'A reference video used only for its voice: only its sound track is read, as an audio reference, and no '+ 'picture is encoded. A clip without sound is refused. Not with early_steps.'}if clip:properties.pop('frame')properties.pop('edit')diff --git a/freevideo_engine/agent/skill/freevideo/references/film-script.txt b/freevideo_engine/agent/skill/freevideo/references/film-script.txtindex a18cde6..01646cd 100644--- a/freevideo_engine/agent/skill/freevideo/references/film-script.txt+++ b/freevideo_engine/agent/skill/freevideo/references/film-script.txt@@ -87,6 +87,6 @@ Five faults that came back in real test generations. Check the prompt and the re2. Faces in wide shots. A face that is 20 to 30 pixels tall comes out smooth and plastic. A person who matters is in a medium shot or a close-up; the wide shot shows people who do not need to be recognised.3. Reading size. Text the audience must read is at least 14 pixels tall in the finished video. Take interface screenshots at 175% to 200% zoom and attach that picture; an 8 pixel line is unreadable and the shot is sent back.4. Third-party marks. Write props as neutral and unbranded: "a neutral silver laptop with a smooth lid", "a plain white takeaway box". Brand logos and trademarks stay out of the prompt and out of the references unless the story needs that exact prop.-5. Voice reference. The first 6 seconds of the voice file carry the timbre: at least 4 of them are the person speaking, and the file does not open with music. When the person's first frame has the mouth hidden (a closed visor or a mask), attach a voice reference whose picture shows the mouth.+5. Voice reference. The first 6 seconds of the voice file carry the timbre: at least 4 of them are the person speaking, and the file does not open with music. When the person's first frame has the mouth hidden (a closed visor or a mask), attach a voice reference whose picture shows the mouth. A video that is attached only for its voice carries `voice_only: true`: its sound track is read as an audio reference and its picture is ignored, so it does not serve as the mouth picture.Example, fixing fault 1 and 4 together. Before: "Lena opens her silver laptop and drinks from a coffee cup." After: "Lena opens a neutral silver laptop with a smooth lid and lifts a plain matte grey mug."diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.pyindex a7a7d65..e1d5a1d 100644--- a/freevideo_engine/media_encoding.py+++ b/freevideo_engine/media_encoding.py@@ -14,7 +14,6 @@ from pathlib import PathREFERENCE_FRAMES = 360 # 15 s at 24 fps, the longest H3 generation-VOICE_ONLY_CANVAS = {'width': 256, 'height': 144} # a clip kept only for its voice is encoded this smalldef reference_frames(canvas):@@ -161,6 +160,9 @@ def prepare(media, canvas, directory):'width': canvas['width'], 'height': canvas['height'], 'crop_mode': 'fit'})for index, row in enumerate(media.get('references', [])):kind, path = row['kind'], row['path']+ if kind == 'video' and row.get('voice_only'):+ # Only the sound is wanted, so the clip is read as an audio reference and no picture is encoded.+ kind = 'audio'if row.get('use') == 'original':from .original_audio import model_audio, inspectnumbers[kind] += 1@@ -198,8 +200,7 @@ def prepare(media, canvas, directory):if not container.streams.video:raise ValueError('Reference video has no video stream')stream = container.streams.video[0]- voice_only = bool(row.get('voice_only'))- width, height = reference_size(stream.width, stream.height, VOICE_ONLY_CANVAS if voice_only else canvas)+ width, height = reference_size(stream.width, stream.height, canvas)length = clip_seconds(container, stream)pixels = np.lib.format.open_memmap(target, mode='w+', dtype=np.uint8, shape=(limit, height, width, 3))origin = None@@ -224,23 +225,12 @@ def prepare(media, canvas, directory):count += 1previous = imagedel previous- step = 1- if voice_only:- # Every third frame (8 fps), or every second (12 fps) when that would leave too few frames.- step = next((n for n in (3, 2) if count // n >= 22), 1)- kept = np.array(pixels[:count:step])- del pixels- pixels, count = kept, len(kept)count = count - (count - 5) % 17if count < 22:raise ValueError('The reference video is too short; it needs to be at least about 1 s long.')- if voice_only:- np.save(target, pixels[:count], allow_pickle=False)- else:- pixels.flush()- ref.update(pixels=str(target), frames=count, normalized_fps=24 // step, width=width, height=height,- crop_mode='voice_%dfps' % (24 // step) if voice_only else 'video_24fps')- used = count * step / 24.+ pixels.flush()+ ref.update(pixels=str(target), frames=count, normalized_fps=24, width=width, height=height, crop_mode='video_24fps')+ used = count / 24.# The clip's own sound is a reference too, unless the request turned it off.soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False or row.get('guide') else \_audio(path, directory / ('reference-%02d-audio.npy' % index), limit / 24.)@@ -248,7 +238,7 @@ def prepare(media, canvas, directory):if soundtrack:ref['audio'] = soundtrackref_items.append({'type': 'audio'})- selected = list(range(0, count, 12 // step))+ selected = list(range(0, count, 12))sampled = torch.from_numpy(np.array(pixels[selected])).float().div_(255)if not row.get('guide'):# A guide only supplies motion latents; it is not described to the@@ -258,7 +248,8 @@ def prepare(media, canvas, directory):else:ref['audio'], longer, length = _audio(path, target, limit / 24.)if ref['audio'] is None:- raise ValueError('Reference audio file has no audio stream')+ raise ValueError('The voice-only reference video has no audio stream' if row['kind'] == 'video'+ else 'Reference audio file has no audio stream')ref.update(width=0, height=0, frames=0, crop_mode='audio_32khz')ref_items.append({'type': 'audio'})if longer:diff --git a/freevideo_engine/media_request.py b/freevideo_engine/media_request.pyindex 4bc1dd0..c19fca2 100644--- a/freevideo_engine/media_request.py+++ b/freevideo_engine/media_request.py@@ -154,7 +154,7 @@ def validate_guide(row):def validate_voice_only(row):- """A voice-only video supplies its sound; its picture is kept small and slow because only the voice is wanted."""+ """A voice-only video supplies only its sound track, read as an audio reference; its picture is not encoded."""if row.get('kind') != 'video':raise ValueError('voice_only is only allowed on video references')if type(row['voice_only']) is not bool:diff --git a/freevideo_engine/sampling_assets.py b/freevideo_engine/sampling_assets.pyindex 4911f5a..06bdf4b 100644--- a/freevideo_engine/sampling_assets.py+++ b/freevideo_engine/sampling_assets.py@@ -177,7 +177,8 @@ def engine_task(media, base=None):A reference video counts as audio when it has an audio stream, exactly asthe encoder decides, unless the request turned that video's sound off;- tables therefore match before the request starts.+ tables therefore match before the request starts. A voice-only reference+ video is read as audio and never counts as a picture."""from .media_request import task_fortask = task_for(media)@@ -185,8 +186,9 @@ def engine_task(media, base=None):return taskvisual = audio = Falsefor ref in media.get('references', []):- visual |= ref['kind'] in ('image', 'video')- audio |= ref['kind'] == 'audio'+ voice = ref['kind'] == 'video' and ref.get('voice_only')+ visual |= ref['kind'] in ('image', 'video') and not voice+ audio |= ref['kind'] == 'audio' or bool(voice)if ref['kind'] == 'video' and not audio and ref.get('sound') is not False and not ref.get('guide'):path = Path(ref['path'])if base is not None and not path.is_absolute():