Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Audio references: a clip can be used as is for the video's sound
Bowen Xued73e253adc59
diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.pyindex 5be6329..71edc9a 100644--- a/freevideo_engine/adaptive.py+++ b/freevideo_engine/adaptive.py@@ -712,6 +712,17 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry):from .continuation import validate_hide_tailhidden = validate_hide_tail(metrics.get('hide_tail_frames', 0))output_frames = expected - hidden+ original_audio = metrics.get('original_audio')+ if original_audio is not None:+ if (not isinstance(original_audio, dict) or original_audio.get('sample_identical') is not True+ or type(original_audio.get('samples_per_channel')) is not int+ or type(original_audio.get('sample_rate')) is not int+ or original_audio['samples_per_channel'] <= 0 or original_audio['sample_rate'] <= 0):+ raise ValueError('Original audio export was not verified')+ count, rate = original_audio['samples_per_channel'], original_audio['sample_rate']+ output_frames = (count * canvas['fps'] + rate - 1) // rate+ if hidden or not 0 < output_frames <= expected:+ raise ValueError('Original audio exceeds the generated canvas')if continuation is not None:from .continuation import OVERLAP_FRAMESif (not isinstance(continuation, dict) or continuation.get('overlap_frames') != OVERLAP_FRAMES@@ -739,10 +750,11 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry):rgb_shape, dtype = _retained_array_header(Path(artifacts)/'rgb.npy')if str(dtype) != 'uint8' or rgb_shape != (output_frames, canvas['height'], canvas['width'], 3):raise ValueError('Retained video frames do not match the complete request.')- audio_shape, audio_dtype = _retained_array_header(Path(artifacts)/'audio.npy')- if (str(audio_dtype) != 'float32' or len(audio_shape) != 2 or audio_shape[0] != 2- or abs(audio_shape[1]/32000. - output_frames/canvas['fps']) > 1/canvas['fps']):- raise ValueError('Retained stereo audio does not match the request duration.')+ if original_audio is None:+ audio_shape, audio_dtype = _retained_array_header(Path(artifacts)/'audio.npy')+ if (str(audio_dtype) != 'float32' or len(audio_shape) != 2 or audio_shape[0] != 2+ or abs(audio_shape[1]/32000. - output_frames/canvas['fps']) > 1/canvas['fps']):+ raise ValueError('Retained stereo audio does not match the request duration.')import avwith av.open(str(output)) as container:video, audio = container.streams.video, container.streams.audio@@ -752,8 +764,13 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry):# A request rendered at a larger size is saved at the size asked for.shown = metrics.get('output_size') or canvasif ((stream.width, stream.height, stream.frames) != (shown['width'],shown['height'],output_frames)- or float(stream.average_rate) != canvas['fps'] or audio[0].codec_context.channels != 2):+ or (original_audio is None and float(stream.average_rate) != canvas['fps'])+ or audio[0].codec_context.channels != (original_audio['channels'] if original_audio else 2)):raise ValueError('MP4 metadata does not match the requested output.')+ if original_audio is not None:+ # A skipped audio decode is not an ordinary request timing observation;+ # retain the existing conservative planner budgets.+ return Noneif metrics.get('sampling_reused') is True:# The output is validated, but these observations cover only a decoder# retry. Do not teach placement/ETA that a full request cost this littlediff --git a/freevideo_engine/audio_use_request.py b/freevideo_engine/audio_use_request.pynew file mode 100644index 0000000..e98eccf--- /dev/null+++ b/freevideo_engine/audio_use_request.py@@ -0,0 +1,87 @@+"""File-backed original-audio preflight for editor and background requests."""+from copy import deepcopy+import json++from .original_audio import AudioUseError, fields, inspect, canvas_for, original+from .chain_plan import MAX_SECONDS+++def selection(value, input_directory):+ from .comfy_assets import resolve_assets+ if not isinstance(value, dict) or set(value) - {'file', 'use', 'range'}:+ raise AudioUseError('audio_use_invalid')+ assets = resolve_assets(json.dumps([dict(value, role='reference')]), input_directory)+ row = original(assets)+ if row is None:+ return dict(use='reference')+ info = inspect(row, max_seconds=MAX_SECONDS)+ canvas = canvas_for(info, 256, 256, max_seconds=MAX_SECONDS)+ samples, rate = info['end_sample'] - info['start_sample'], info['sample_rate']+ return dict(use='original', source_id=value['file'], sample_rate=rate,+ range=dict(start_sample=info['start_sample'], end_sample=info['end_sample']),+ samples=samples, seconds=samples / rate, frames=canvas['frames'],+ output_frames=(samples * 24 + rate - 1) // rate, source_sha256=info['sha256'])+++def graph_audio(prompt, node):+ """Inspect only connected known media nodes; never log the graph or text."""+ from .chain_runner import node_prompt+ from .comfy_assets import media_kind+ rows = []+ for item in node_prompt(prompt, node).values():+ if not isinstance(item, dict):+ continue+ inputs = item.get('inputs') or {}+ kind = item.get('class_type')+ if kind == 'FreeVideoMedia':+ assets = inputs.get('assets', '[]')+ if not isinstance(assets, str):+ continue # Dynamic sockets are validated when executed.+ assets = json.loads(assets)+ if not isinstance(assets, list):+ raise AudioUseError('audio_use_invalid')+ for asset in assets:+ if not isinstance(asset, dict):+ raise AudioUseError('audio_use_invalid')+ if not asset.get('enabled', True):+ continue+ if 'use' in asset or 'range' in asset:+ options = fields(dict(asset, kind=media_kind(asset.get('file', ''))))+ if options:+ rows.append(dict(kind='audio', file=asset['file'], **options))+ use = inputs.get('audio_use', 'reference')+ if isinstance(use, str) and fields(dict(kind='audio', use=use)):+ rows.append(dict(kind='audio', use='original'))+ elif kind == 'FreeVideoReference':+ use = inputs.get('use', 'reference')+ if isinstance(use, str) and fields(dict(kind='audio', use=use)):+ rows.append(dict(kind='audio', use='original'))+ original(dict(references=rows))+ return rows+++def prepare_chain(payload, input_directory):+ rows = graph_audio(payload['prompt'], payload['node'])+ if not rows:+ return payload+ value = deepcopy(payload)+ blocks = value.get('blocks')+ if blocks is not None and (not isinstance(blocks, list) or any(not isinstance(row, dict) for row in blocks)):+ raise AudioUseError('audio_original_route')+ if 'file' not in rows[0]:+ # Tensor sockets are resolved in the node, not by the HTTP thread.+ return value+ from .comfy_assets import resolve_assets+ asset = {k: v for k, v in rows[0].items() if k != 'kind'}+ media = resolve_assets(json.dumps([dict(asset, role='reference')]), input_directory)+ info = inspect(original(media), max_seconds=MAX_SECONDS)+ selected = canvas_for(info, 256, 256, max_seconds=MAX_SECONDS)+ value['original_audio'] = dict(file=asset['file'], use='original', info=info)+ if blocks is not None:+ if len(blocks) == 1 and not any(blocks[0].get(k) for k in ('after', 'before')):+ if blocks[0]['frames'] < (info['end_sample'] - info['start_sample']) * 24 / info['sample_rate']:+ raise AudioUseError('audio_original_slot')+ blocks[0]['frames'] = selected['frames']+ else:+ value['seconds'] = (info['end_sample'] - info['start_sample']) / info['sample_rate']+ return valuediff --git a/freevideo_engine/capabilities.py b/freevideo_engine/capabilities.pyindex a2c027d..cd32564 100644--- a/freevideo_engine/capabilities.py+++ b/freevideo_engine/capabilities.py@@ -53,7 +53,7 @@ def normalize_references(task, references=None):if not isinstance(row, dict):raise ValueError('Invalid reference metadata.')kind = row.get('kind')- allowed = {'kind', 'seconds'} if kind == 'audio' else {'kind', 'width', 'height'}+ allowed = {'kind', 'seconds', 'use'} if kind == 'audio' else {'kind', 'width', 'height'}if kind == 'video':allowed |= {'seconds', 'audio_seconds', 'guide'}if kind not in ('image', 'video', 'audio') or set(row) - allowed:@@ -83,7 +83,16 @@ def normalize_references(task, references=None):raise ValueError('Specify positive finite soundtrack seconds, or omit it for silent video.')if audio < 1 / 32000:raise ValueError('Reference audio is empty after normalization.')- rows.append((kind, width, height, seconds, audio))+ from .original_audio import fields, AudioUseError, MAX_SECONDS+ use = fields(row)+ if use:+ if seconds < 1.625:+ raise AudioUseError('audio_original_short')+ if seconds > MAX_SECONDS:+ raise AudioUseError('audio_original_long')+ rows.append((kind, width, height, seconds, audio) + (('original',) if use else ()))+ if sum(len(row) == 6 for row in rows) > 1:+ raise AudioUseError('audio_original_multiple')visual = any(r[0] != 'audio' for r in rows)audio = any(r[0] == 'audio' or r[4] is not None for r in rows)expected = 'ref2va_av' if visual and audio else 'ref2va_audio' if audio else 'ref2va' if visual else None@@ -96,7 +105,7 @@ def reference_parameters(references):"""JSON form of a normalized cache key, also used in notifications."""if references is None:return None- return [{k: v for k, v in zip(('kind', 'width', 'height', 'seconds', 'audio_seconds'), row)+ return [{k: v for k, v in zip(('kind', 'width', 'height', 'seconds', 'audio_seconds', 'use'), row)if v is not None} for row in references]@@ -124,7 +133,13 @@ def _material(canvas, task, references):video_rows = audio_rows = 0duration = 0.limit = reference_frames(canvas)- for kind, width, height, seconds, soundtrack in references:+ for row in references:+ kind, width, height, seconds, soundtrack = row[:5]+ if len(row) == 6:+ # Original audio occupies the existing target rows. Keep all+ # ordinary model/decoder budgets, without duplicating the track.+ duration += seconds+ continueif kind != 'audio':width, height = reference_size(width, height, canvas)patches = (width // 32) * (height // 32)diff --git a/freevideo_engine/chain_join.py b/freevideo_engine/chain_join.pyindex 5abfc0d..d1fd11a 100644--- a/freevideo_engine/chain_join.py+++ b/freevideo_engine/chain_join.py@@ -5,6 +5,8 @@ needed to reach an exact requested length with matching codec parameters. Thesound is decoded and encoded once as a single stream: every block's soundfollows the previous block's sound, not its own first frame, which keeps along chain in step.+Original soundtracks instead bypass every block's sound and mux the selected+source range once, with a sample-identity receipt."""from datetime import datetimefrom fractions import Fraction@@ -160,7 +162,8 @@ def _exact_tail(source, target, wanted, cancelled=None):temporary.unlink(missing_ok=True)-def join_files(sources, target, cancelled=None, frames=None, leads=None, expected=None, speaking=None):+def join_files(sources, target, cancelled=None, frames=None, leads=None, expected=None, speaking=None, *,+ original_audio=None):"""Write `target` from `sources` in order; returns the joined frame and sample counts.`frames`: stop at this length when a compatible short tail can close it exactly,@@ -168,32 +171,38 @@ def join_files(sources, target, cancelled=None, frames=None, leads=None, expecte`leads`: optional `leads(following, previous)` returning a seam lead kept by the chain;the shared continuation record is read only when it has none.`expected`: how many seams should be repaired; only reported back, the caller decides.+ `original_audio`: mux the original selection and close the picture on its+ source-sample clock; a rounded completed prefix keeps its existing frames."""sources = list(sources)try:- return _join_ending(sources, target, cancelled, frames, leads, expected, speaking)+ return _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio)except _VoiceFailed:logging.warning('[FreeVideo] The voice of the parts could not be evened out; they are joined as they are',exc_info=True)- return _join_ending(sources, target, cancelled, frames, leads, expected, None)+ return _join_ending(sources, target, cancelled, frames, leads, expected, None, original_audio)-def _join_ending(sources, target, cancelled, frames, leads, expected, speaking):+def _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio=None):try:- return _join_files(sources, target, cancelled, frames, leads=leads, expected=expected, speaking=speaking)+ return _join_files(sources, target, cancelled, frames, leads=leads, expected=expected, speaking=speaking,+ original_audio=original_audio)except _ExactEndFailed:# A late mux or validation error must also leave the original ending available.return _join_files(sources, target, cancelled, frames, exact=False, leads=leads, expected=expected,- speaking=speaking)+ speaking=speaking, original_audio=original_audio)-def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None):+def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None,+ original_audio=None):import avfrom .comfy_metadata import _copy_stream- try:- from .comfy_continuation import audio_lead_in, apply_audio_lead_in- except ImportError:- audio_lead_in = apply_audio_lead_in = None+ audio_lead_in = apply_audio_lead_in = None+ if original_audio is None:+ try:+ from .comfy_continuation import audio_lead_in, apply_audio_lead_in+ except ImportError:+ audio_lead_in = apply_audio_lead_in = Noneif leads is not None and audio_lead_in is not None:stored = audio_lead_indef audio_lead_in(video, previous=None):@@ -203,7 +212,7 @@ def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expereturn leads(video, previous)return leads(video, previous) or stored(video, previous=previous)sources, target = [Path(source) for source in sources], Path(target)- if len(sources) < 2:+ if len(sources) < (2 if original_audio is None else 1):raise ValueError('Joining needs at least two videos')if target.exists() or target.is_symlink():raise ValueError('The joined video already exists')@@ -216,7 +225,7 @@ def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expeif len(first.streams.video) != 1:raise ValueError('A block has no single picture stream')shape = _shape(first.streams.video[0])- sound = first.streams.audio[0] if first.streams.audio else None+ sound = first.streams.audio[0] if first.streams.audio and original_audio is None else Nonerate = sound.codec_context.sample_rate if sound is not None else Nonelayout = sound.codec_context.layout.name if sound is not None else Nonebit_rate = max(sound.codec_context.bit_rate or 0, 192000) if sound is not None else None@@ -236,7 +245,7 @@ def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expewith av.open(str(source)) as container:if len(container.streams.video) != 1 or _shape(container.streams.video[0]) != shape:raise ValueError('The blocks differ in size or encoding')- if bool(container.streams.audio) != (sound is not None):+ if original_audio is None and bool(container.streams.audio) != (sound is not None):raise ValueError('Only some of the blocks have sound')video = container.streams.video[0]tail = (_exact_tail(source, target, wanted - frames, cancelled)@@ -308,8 +317,33 @@ def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expewith av.open(str(temporary)) as check:if check.streams.video[0].frames != frames:raise ValueError('The joined video is incomplete')+ receipt = None+ if original_audio is not None:+ from .original_audio import mux+ required = ((original_audio['end_sample'] - original_audio['start_sample']) * FPS+ + original_audio['sample_rate'] - 1) // original_audio['sample_rate']+ if frames < required:+ # Completed prefixes end on nearest-sample global boundaries.+ # Rounding up can put the endpoint half a sample past the+ # last frame; it does not require a further picture frame.+ count = original_audio['end_sample'] - original_audio['start_sample']+ if 2 * (count * FPS - frames * original_audio['sample_rate']) > FPS:+ raise ValueError('Original audio is longer than the joined picture')+ required = frames+ receipt = mux(temporary, temporary, original_audio, video_frames=required)+ _check(cancelled)+ rate, samples = receipt['sample_rate'], receipt['samples_per_channel']+ with av.open(str(temporary)) as check:+ video = check.streams.video[0]+ frames = video.frames+ if (video.duration is None or+ abs(video.duration * video.time_base - Fraction(samples, rate)) > Fraction(1, rate)):+ raise ValueError('The joined picture did not end on the selected audio sample')+ if frames != required:+ raise ValueError('The joined picture did not reach the selected audio endpoint')os.replace(temporary, target)- return dict(exact_end=wanted is not None and frames == wanted, frames=frames, width=shape[1], height=shape[2],+ return dict(**({'original_audio': receipt} if receipt is not None else {}),+ exact_end=wanted is not None and frames == wanted, frames=frames, width=shape[1], height=shape[2],audio_samples=samples, sample_rate=rate,seams_repaired=joined_audio.seams_repaired if joined_audio is not None else 0,seams_expected=expected,@@ -563,7 +597,7 @@ def _replace_target(root, replace, identity, sources):def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, leads=None, expected_seams=None,- speaking=None):+ speaking=None, *, original_audio=None):"""Join relative block outputs into a saved video, replacing only this chain's earlier join."""import folder_pathsroot = Path(folder_paths.get_output_directory()).resolve()@@ -594,8 +628,10 @@ def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, loriginals = {str(source): output for source, output in zip(sources, outputs)}def kept(following, previous):return leads(originals.get(str(following)), originals.get(str(previous)) if previous is not None else None)- joined = join_files(sources, staged, cancelled, frames, leads=kept, expected=expected_seams, speaking=speaking)- if expected_seams is not None and joined['seams_repaired'] < expected_seams:+ joined = join_files(sources, staged, cancelled, frames, leads=kept, expected=expected_seams, speaking=speaking,+ **({'original_audio': original_audio} if original_audio is not None else {}))+ # The original selection replaces every block's sound, so there are no seam sounds to repair.+ if original_audio is None and expected_seams is not None and joined['seams_repaired'] < expected_seams:# A seam sound left unrepaired is never reported as a finished video.raise ValueError('Only %d of %d seam sounds could be repaired' % (joined['seams_repaired'], expected_seams))report = dict(success=True, geometry=dict(width=joined['width'], height=joined['height'], frames=joined['frames'],@@ -607,6 +643,9 @@ def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, lreport['chain_join']['seams_expected'] = expected_seamsif joined.get('voice') is not None:report['chain_join']['voice'] = joined['voice']+ if original_audio is not None:+ report['video']['original_audio'] = joined['original_audio']+ report['geometry']['seconds'] = joined['original_audio']['duration_seconds']with temporary.open('w', encoding='utf-8') as stream:json.dump(report, stream, ensure_ascii=False)stream.flush()diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.pyindex 401ee78..c8a3791 100644--- a/freevideo_engine/chain_runner.py+++ b/freevideo_engine/chain_runner.py@@ -529,7 +529,8 @@ class ChainRunner:def hidden(self, exists=None):with self.lock:return {row['output'] for record in self.records.values()- if record['join'] and record['output_state'] == 'done' and record['output_blocks'] > 1+ if record['join'] and record['output_state'] == 'done'+ and (record['output_blocks'] > 1 or record.get('original_audio') is not None)and record['output'] not in {block['output'] for block in record['blocks']}and (exists is None or exists(record['output']))for row in record['blocks'][:record['output_blocks']] if row['output'] is not None}@@ -597,8 +598,9 @@ class ChainRunner:self._save(record)self.wake.set()previous = self._sources(record, block)['after']+ # A kept original soundtrack replaces every seam's sound, so there is no lead to save.keep = (record['id'], block['index'], output, previous['output'] if previous is not None else None,- bool(saved))+ bool(saved) and record.get('original_audio') is None)self._keep_lead(*keep)def _keep_lead(self, identity, index, output, previous, continues):@@ -627,6 +629,9 @@ class ChainRunner:block that continues from a block other than its neighbour cannot be repaired and counts as lost."""identity, blocks = snapshot['id'], store.leading_blocks(snapshot)wanted, kept = set(), 0+ if snapshot.get('original_audio') is not None:+ # The original selection replaces every block's sound, so no seam sound needs repair.+ return wanted, keptfor position, row in enumerate(blocks[1:] if getattr(self.host, 'repairs_seams', True) else (), 1):if not self._continues(row):continue@@ -647,7 +652,7 @@ class ChainRunner:if not blocks:record.update(output=None, output_blocks=0, output_state='pending', output_error=None)return final_state- if len(blocks) == 1:+ if len(blocks) == 1 and record.get('original_audio') is None:output = blocks[0]['output']record.update(output=output, output_blocks=1 if output else 0, output_state='done', output_error=None)return final_state@@ -763,7 +768,8 @@ class ChainRunner:self._save(record)prompt = deepcopy(record['snapshot']['prompt'])block_inputs = dict(block['inputs'])- block_inputs.update(index=block['index'], count=len(record['blocks']))+ block_inputs.update(index=block['index'], count=len(record['blocks']),+ original_audio=block.get('original_audio') is not None)sources = self._sources(record, block)extra = deepcopy(record['snapshot']['extra_data'])if record['client_id'] is not None:@@ -804,6 +810,9 @@ class ChainRunner:if hasattr(self.host, 'continued'):self.host.continued(prompt, node)prompt[node]['inputs'].update(self.inputs(block_inputs))+ if record.get('original_audio') is not None:+ from .original_audio_chain import block_prompt+ block_prompt(prompt, node, record['original_audio'], block['original_audio'])self.host.submit(prompt_id, node_prompt(prompt, node), extra)except Exception as error:# A response may be lost after the host has accepted the prompt.diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.pyindex cd4bb63..6177fc8 100644--- a/freevideo_engine/chain_store.py+++ b/freevideo_engine/chain_store.py@@ -91,9 +91,15 @@ def validate(payload):source = payload.get('after')if source is not None and not isinstance(source, dict):raise ValueError('A planned video can only continue a saved work')- planned = chain_plan.plan(payload['seconds'], payload.get('single', False), payload.get('max_frames'),- payload.get('single_limit_seconds'), payload.get('block_seconds'),- continued=source is not None)+ if payload.get('original_audio') is not None:+ from .original_audio_chain import plan as audio_plan, selection_info+ planned = audio_plan(selection_info(payload['original_audio']),+ **{key: payload[key] for key in ('single', 'max_frames', 'single_limit_seconds', 'block_seconds')+ if key in payload})+ else:+ planned = chain_plan.plan(payload['seconds'], payload.get('single', False), payload.get('max_frames'),+ payload.get('single_limit_seconds'), payload.get('block_seconds'),+ continued=source is not None)inputs = prompt[node]['inputs']text, seed = payload.get('text', inputs.get('text')), payload.get('seed', inputs.get('seed'))if not isinstance(text, str):@@ -117,6 +123,11 @@ def validate(payload):rows = payload['blocks']if not isinstance(rows, list) or not 1 <= len(rows) <= chain_plan.MAX_BLOCKS:raise ValueError('A chain requires 1 to %s blocks' % chain_plan.MAX_BLOCKS)+ from .audio_use_request import graph_audio+ from .original_audio import AudioUseError+ if graph_audio(prompt, node) and payload.get('original_audio') is None and (len(rows) != 1 or not isinstance(rows[0], dict)+ or any(rows[0].get(k) for k in ('after', 'before'))):+ raise AudioUseError('audio_original_route')blocks, ids = [], set()for index, row in enumerate(rows):if not isinstance(row, dict):@@ -175,6 +186,9 @@ def validate(payload):output_error=None, output_blocks=0, split=split)if queue:record['queue'] = True+ if payload.get('original_audio') is not None:+ from .original_audio_chain import bind+ bind(record, payload['original_audio'])try:json.dumps(record, allow_nan=False)except (ValueError, TypeError, OverflowError) as error:@@ -213,6 +227,12 @@ def detail(record):# Split parts repeat words put there by code; the page lists and edits only each part's own.from .prompt_vlm.rules import own_wordsvalue['part_words'] = own_words([str(row['inputs'].get('text', '')) for row in record['blocks']])+ if record.get('original_audio') is not None:+ selected = record['original_audio']+ info = selected['info']+ value['original_audio'] = dict(file=selected['file'], use='original',+ range={key: info[key] for key in ('start_sample', 'end_sample')},+ sample_rate=info['sample_rate'], source_sha256=info['sha256'])return value@@ -304,10 +324,13 @@ def load_all(directory, limit=200, *, identity=None):normalized = validate(dict(node=record['node'], client_id=record['client_id'], **snapshot,queue=record.get('queue', False), blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')}| row['inputs']- for row in record['blocks']], join=record['join']))+ for row in record['blocks']], join=record['join'],+ **({'original_audio': record['original_audio']} if 'original_audio' in record else {})))if record['state'] not in CHAIN_STATES:raise ValueError('Invalid chain state')for row, clean in zip(record['blocks'], normalized['blocks']):+ if record.get('original_audio') is not None and row.get('original_audio') != clean['original_audio']:+ raise ValueError('Invalid original audio block range')row.setdefault('new_frames', clean['new_frames'])if type(row['new_frames']) is not int or row['new_frames'] != clean['new_frames']:raise ValueError('Invalid block new frame count')diff --git a/freevideo_engine/cli.py b/freevideo_engine/cli.pyindex 2767d47..d93c497 100644--- a/freevideo_engine/cli.py+++ b/freevideo_engine/cli.py@@ -99,6 +99,9 @@ def main():from .media_request import TASKSgenerate.add_argument('--task', choices=TASKS, help='Input task; inferred when --media is provided')generate.add_argument('--media', type=Path, help='Versioned keyframe/reference/LoRA request JSON; local files only')+ generate.add_argument('--audio', type=Path, help='Audio file; may be combined with image/video references in --media')+ generate.add_argument('--audio-use', choices=('reference', 'original'),+ help='Audio use (default: reference); with --media alone, applies to its single audio file')generate.add_argument('--continue-from', type=Path, help='Continue an existing MP4 using its absolute path')generate.add_argument('--first-frame-reference', action='store_true',help='With --continue-from: also give the model the first frame of the whole film as a '@@ -167,6 +170,11 @@ def main():anchor=args.continue_anchor is not None, two_pass=args.two_pass, preview=args.preview,refine_from=args.refine_from)if sys.platform == 'darwin' and args.command in ('plan', 'generate', 'encode', 'doctor'):+ if args.command == 'generate':+ from .media_request import read+ from .original_audio import original, AudioUseError+ if args.audio is not None or args.audio_use == 'original' or original(read(args.media)) is not None:+ raise AudioUseError('audio_original_backend')from .macos_generate import dispatchreturn dispatch(args)if args.command == 'plan':diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.pyindex 0d085fa..4628d13 100644--- a/freevideo_engine/comfy_assets.py+++ b/freevideo_engine/comfy_assets.py@@ -75,7 +75,8 @@ def resolve_assets(serialized, input_directory):root = Path(input_directory).resolve()result = {}for row in rows:- if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth'}:+ if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'depth',+ 'use', 'range', 'segment'}:raise ValueError('Invalid media item')if 'depth' in row:if type(row['depth']) is not bool:@@ -104,6 +105,8 @@ def resolve_assets(serialized, input_directory):if not path.is_file():raise ValueError('Upload or select the missing media file: ' + name)kind = media_kind(name)+ from .original_audio import fields+ audio_fields = fields(dict(row, kind=kind))if role in ('first', 'last'):if kind != 'image':raise ValueError('First and last frames must be images')@@ -111,7 +114,7 @@ def resolve_assets(serialized, input_directory):raise ValueError('Select only one ' + role + ' frame')result[role] = str(path)elif role == 'reference':- reference = {'kind': kind, 'path': str(path)}+ reference = {'kind': kind, 'path': str(path), **audio_fields}if row.get('depth') is True:reference.update(depth=True, early_steps=3)if cast_id(row) is not None:@@ -138,6 +141,8 @@ def resolve_assets(serialized, input_directory):result.setdefault('references', []).append(reference)else:raise ValueError('Media role must be first, last or reference')+ from .original_audio import original+ original(result)# Mixing keyframes and references is checked when the video is made (comfy_media.export):# a part that continues a video uses its keyframes as pictures.return result@@ -147,23 +152,27 @@ def continued_assets(serialized):"""The panel rows a continued part uses: enabled image references in panel order, at most three(continuation.pick_pictures: with two or more characters, one picture each).- Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks."""+ Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks.+ The one exception is a sound used as is: every part plays its own range of it (original_audio_chain.block_prompt)."""from .continuation import pick_picturestry:rows = json.loads(serialized)except (TypeError, ValueError):return '[]'- pictures = []+ pictures, original = [], []for row in rows if isinstance(rows, list) else ():if (isinstance(row, dict) and row.get('enabled', True) is True and row.get('role') == 'reference'and isinstance(row.get('file'), str)):+ if row.get('use') == 'original':+ original.append(dict(file=row['file'], role='reference', use='original'))+ continuetry:image = media_kind(row['file']) == 'image'except ValueError:image = Falseif image:pictures.append(row)- return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)])+ return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)] + original)def cast_id(row):@@ -180,12 +189,19 @@ def _kept_row(row):return kept-def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, **images):+def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, audio_use='reference',+ **images):"""Keep connected IMAGE/AUDIO values native until request input export."""result = dict(assets)if set(images) - set(REFERENCE_INPUTS[1:]):raise TypeError('Unknown reference image input')images = dict(reference=reference, **images)+ from .original_audio import fields, original+ audio_fields = fields(dict(kind='audio', use=audio_use))+ if audio_fields and reference_audio is None:+ raise ValueError('Connect audio before selecting Use as is')+ original(dict(references=[*result.get('references', []), *([dict(kind='audio', **audio_fields)]+ if reference_audio is not None else [])]))connected = {name: value for name, value in (('first', first), ('last', last),*((name, images.get(name)) for name in REFERENCE_INPUTS), ('reference_audio', reference_audio))if value is not None}@@ -202,6 +218,8 @@ def connect_assets(assets, *, first=None, last=None, reference=None, reference_a'For a long image batch, connect a clip to the Reference node\'s video input; a clip counts as one reference.')if connected:result['_connected'] = connected+ if audio_fields:+ result['_audio_use'] = 'original'return result@@ -246,6 +264,8 @@ def output_summary(report, relative_video):# Resource reports keep the generated canvas; result cards show only saved frames.result['geometry'].update(frames=report['output_frames'],seconds=report['output_frames'] / (result['geometry'].get('fps') or 24))+ if video.get('original_audio'):+ result['original_audio'] = {k: v for k, v in video['original_audio'].items() if k != 'path'}checkpoint = encoding.get('checkpoint') or {}sizes = checkpoint.get('bytes'), checkpoint.get('expected_bytes')# Reports are read back from disk; a malformed one gets no notice rather than failing the library.diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.pyindex 7647554..1c192bb 100644--- a/freevideo_engine/comfy_bridge.py+++ b/freevideo_engine/comfy_bridge.py@@ -856,7 +856,7 @@ def installation(source=None, environ=None):def validate_request(prompt, width, height, seconds, seed, *, continue_from=None, frames=None,- two_pass=True, preview=False, refine_from=None, media=None):+ two_pass=True, preview=False, refine_from=None, media=None, original_audio=False):if not isinstance(prompt, str) or not prompt.strip():raise ValueError('Enter a prompt describing the video and its audio.')if type(seed) is not int or not 0 <= seed <= 2**53 - 1:@@ -869,7 +869,8 @@ def validate_request(prompt, width, height, seconds, seed, *, continue_from=Nonecanvas = geometry(width, height, frames=frames, seconds=None if frames is not None else seconds)if continue_from is not None:from .continuation import validate_path, validate_route- validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from, media=media)+ validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from, media=media,+ original_audio=original_audio)validate_path(continue_from)return canvas@@ -1426,6 +1427,24 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,from .media_request import task_forfrom .two_pass import plan, sampling_canvasfrom .generate import hide_tail_geometry+ from .original_audio import prepare_request+ from copy import deepcopy+ original_audio, audio_canvas = prepare_request(deepcopy(extra.get('media') or {}), width, height,+ continuation=continue_from is not None, backend=machine.get('device_backend', 'cuda'))+ if original_audio is not None:+ canvas = audio_canvas+ state['geometry'] = canvas+ extra['original_audio'] = original_audio+ flag = '--frames' if '--frames' in command else '--seconds'+ position = command.index(flag)+ command[position:position + 2] = (['--frames', str(canvas['frames'])] if 'segment' in original_audio else+ ['--seconds', str((original_audio['end_sample'] - original_audio['start_sample']) / original_audio['sample_rate'])])+ # The sound used as is sets the length and is the part's one sound: the engine adds no hidden+ # tail, no voice reference and no first frame of the film beside it (cast pictures stay).+ first_frame_reference = voice_reference = hide_tail = anchored = False+ for flag in ('--first-frame-reference', '--voice-reference', '--hide-tail'):+ if flag in command:+ command.remove(flag)generated_canvas, hidden = hide_tail_geometry(canvas, None if extra.get('conditioning') else prompt,hide_tail, preview=bool(preview or upscale))hide_tail_frames = hidden['frames'] if hidden and hidden['used'] else 0@@ -1439,7 +1458,11 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, MISSING, validate_snapshot, summaryvalidate_route(task='ref2va' if anchored or pictured else task_for(extra.get('media', {})),anchor=anchored or pictured,- media=extra.get('media'))+ media=extra.get('media'), original_audio=original_audio is not None)+ if original_audio is not None and continuation is not None:+ # An encoded ending has no stored origin; the engine checks the part against its own.+ from .original_audio_chain import validate_continuation+ validate_continuation(original_audio, continuation)if continuation is not None:try:continuation = validate_snapshot(continuation, sampled, sampling_plan=planned)@@ -1497,7 +1520,8 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,preparation_started = time.monotonic()try:# Reference audio selects its own tables; match the encoder's choice.- asset_task = ((ANCHOR_ADALN_TASK if anchored or pictured else ADALN_TASK) if continue_from is not None+ asset_task = ((ANCHOR_ADALN_TASK if anchored or pictured else ADALN_TASK)+ if continue_from is not None and original_audio is Noneelse engine_task(extra.get('media', {}), media_base))# Tables follow the step count; the cache is carried for exact level recognition.preparation = prepare_sampling_assets(root, machine, dict(planned, step_cache=step_cache) if step_cache else planned,@@ -1598,7 +1622,11 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,# Dropping the video on the canvas restores this graph, as with ComfyUI's own video nodes.# Written before the result cache records the file's size and hash.from .comfy_metadata import embed_comfy_metadata- state['workflow_in_video'] = embed_comfy_metadata(output, **comfy_metadata)+ timing = {}+ if original_audio is not None:+ from math import lcm+ timing['exact_timescale'] = lcm(original_audio['sample_rate'], 24)+ state['workflow_in_video'] = embed_comfy_metadata(output, **comfy_metadata, **timing)# The MP4 is saved and checked; its uncompressed frames and audio are no longer read.from .comfy_library import release_decoded, release_laterstate['decoded_release'] = release_decoded(run)diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.pyindex 1b99620..fe104d2 100644--- a/freevideo_engine/comfy_chain.py+++ b/freevideo_engine/comfy_chain.py@@ -77,10 +77,15 @@ class ComfyHost:# Each direction is filled once the node offers its input.if name in schema:values[name] = '' if independent else block_inputs[key]- if 'voice_reference' in schema and values.get('continue_from'):+ # A part that plays its range of a sound used as is takes no first frame of the film, no voice+ # reference and no hidden tail: the engine declines all three for it (generate.py).+ original = bool(block_inputs.get('original_audio'))+ if original and 'first_frame_reference' in schema:+ values['first_frame_reference'] = False+ if 'voice_reference' in schema and values.get('continue_from') and not original:values['voice_reference'] = True- if ('hide_tail' in schema and not independent and block_inputs.get('index', 0) < block_inputs.get('count', 0) - 1- and continuation_supported()):+ if ('hide_tail' in schema and not independent and not original+ and block_inputs.get('index', 0) < block_inputs.get('count', 0) - 1 and continuation_supported()):# Only a backend that continues videos makes the extra tail frames; others ignore the option.values['hide_tail'] = Truereturn values@@ -176,6 +181,15 @@ class ComfyHost:blocks = chain_store.leading_blocks(record)# A planned chain shows exactly the requested length: what the last block made beyond it is left out.frames = planned['frames'] if planned and not planned['single'] and len(blocks) == len(record['blocks']) else None+ if record.get('original_audio') is not None:+ # The original selection replaces every part's sound: seam leads and voice levels do not apply.+ from .original_audio_chain import joined_info+ info = joined_info(record, blocks)+ frames = ((info['end_sample'] - info['start_sample']) * 24 + info['sample_rate'] - 1) // info['sample_rate']+ if len(blocks) < len(record['blocks']):+ frames = sum(block['new_frames'] for block in blocks)+ return join_outputs([block['output'] for block in blocks], record['id'], cancelled,+ frames=frames, replace=record.get('output'), original_audio=info)kept = {'leads': record['lead_reader'], 'expected_seams': record.get('expected_seams')} \if record.get('lead_reader') is not None else {}from .chain_level import voiced@@ -357,6 +371,11 @@ def register(host_factory=None):async def create(request):try:value = await body(request)+ from .audio_use_request import graph_audio, prepare_chain+ if isinstance(value.get('prompt'), dict) and value.get('node') in value['prompt']:+ if graph_audio(value['prompt'], value['node']):+ import folder_paths+ value = await asyncio.to_thread(prepare_chain, value, folder_paths.get_input_directory())normalized = chain_store.validate(value)if not fake and os.environ.get('FREEVIDEO_CHAIN_INDEPENDENT') != '1':missing = {key for row in normalized['blocks'] for key in CONTINUE_INPUTS@@ -377,7 +396,7 @@ def register(host_factory=None):except chain_plan.PlanError as failure:return response({'error': str(failure), 'code': failure.code}, 400)except ValueError as failure:- return response({'error': str(failure)}, 400)+ return response({'error': str(failure), **(failure.as_dict() if hasattr(failure, 'as_dict') else {})}, 400)def asking(request):# ?client_id=... marks each chain as this page's own or not.diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.pyindex d989992..5b39686 100644--- a/freevideo_engine/comfy_media.py+++ b/freevideo_engine/comfy_media.py@@ -49,21 +49,24 @@ def continued(exported, notes=None):"""What a continued part keeps of exported inputs: up to three pictures, and the LoRAs.It starts from the footage, so a first or last frame image becomes one more picture after the- reference pictures; pre-encoded conditioning, clips and sound are left out. When keyframes are- used or pictures are left out, notes gets one line saying so."""+ reference pictures; pre-encoded conditioning, clips and sound are left out, except a sound used+ as is, which every part plays its own range of. When keyframes are used or pictures are left out,+ notes gets one line saying so."""from .continuation import PICTURES_MAX, pick_picturesmedia = (exported or {}).get('media') or {}kept = {key: value for key, value in media.items()if key not in ('first', 'last', 'references', 'conditioning_info')}pictures = [row for row in media.get('references', []) if row.get('kind') == 'image']+ original = [row for row in media.get('references', []) if row.get('kind') == 'audio' and row.get('use') == 'original']keyframes = [name for name in ('first', 'last') if media.get(name)]# Characters added by an @ mention keep one picture each when there are several (pick_pictures).chosen = pick_pictures(pictures, lambda row: row.get('cast'))candidates = [_plain(row) for row in chosen] + [{'kind': 'image', 'path': media[name]} for name in keyframes]- if candidates:- kept['references'] = candidates[:PICTURES_MAX]+ if candidates or original:+ kept['references'] = candidates[:PICTURES_MAX] + originallines = [continued_note(len(pictures), keyframes, len(candidates[:PICTURES_MAX]))]- if any(row.get('kind') != 'image' for row in media.get('references', [])) or media.get('conditioning_info') \+ if any(row.get('kind') != 'image' and row not in original for row in media.get('references', [])) \+ or media.get('conditioning_info') \or (exported or {}).get('conditioning'):lines.append('A continued part uses picture references only. Video, audio and pre-encoded inputs are skipped.')for line in filter(None, lines):@@ -100,7 +103,7 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, cif conditioning is not None or first is not None or last is not None or references:raise ValueError('Use the Media panel or direct media/conditioning sockets; disable duplicate inputs')connected = assets.get('_connected', {})- media.update({key: value for key, value in assets.items() if key != '_connected'})+ media.update({key: value for key, value in assets.items() if key not in ('_connected', '_audio_use')})first, last = connected.get('first'), connected.get('last')references = []for name in REFERENCE_INPUTS:@@ -112,6 +115,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, creferences.extend({'kind': 'image', 'value': frame} for frame in frames)if connected.get('reference_audio') is not None:references.append({'kind': 'audio', 'value': connected['reference_audio']})+ if assets.get('_audio_use') == 'original':+ next(row for row in references if row['kind'] == 'audio')['use'] = 'original'if conditioning is not None and (first is not None or last is not None or references):raise ValueError('Preencoded conditioning already includes media; do not connect a second set')keyframes = first is not None or last is not None or any(name in media for name in ('first', 'last'))@@ -145,7 +150,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c_audio(value, path)else:raise ValueError('Unknown reference type')- row = {'kind': kind, 'path': str(path)}+ from .original_audio import fields+ row = {'kind': kind, 'path': str(path), **fields(reference)}if 'early_steps' in reference:row['early_steps'] = reference['early_steps']media['references'].append(row)@@ -162,5 +168,7 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, cif not continuing and media.get('references'):media['references'] = [_plain(row) for row in media['references']]if len(media) > 1:+ from .original_audio import original+ original(media)result['media'] = mediareturn resultdiff --git a/freevideo_engine/comfy_metadata.py b/freevideo_engine/comfy_metadata.pyindex 0abafdf..ca98805 100644--- a/freevideo_engine/comfy_metadata.py+++ b/freevideo_engine/comfy_metadata.py@@ -24,7 +24,7 @@ def _room(path, size):return disk_space.free_bytes(path.parent) >= 2 * size + 512 * 1024 * 1024-def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None):+def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None, exact_timescale=None):"""Rewrite the MP4 at `path` with the graph in its metadata, copying thestreams. The new file replaces the original only after every audio and videopacket matches the original byte for byte and the graph reads back; with@@ -48,7 +48,9 @@ def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None):return False# faststart puts moov, and with it the metadata, before the media data.with av.open(str(path)) as source, av.open(str(temporary), 'w', format='mp4',- options={'movflags': 'use_metadata_tags+faststart'}) as target:+ options={'movflags': 'use_metadata_tags+faststart',+ **({'video_track_timescale': str(exact_timescale), 'movie_timescale': str(exact_timescale)}+ if exact_timescale is not None else {})}) as target:if not source.streams or any(stream.type not in ('video', 'audio') or stream.codec_context is Nonefor stream in source.streams):raise ValueError('Only videos with nothing but audio and video streams are rewritten')@@ -61,6 +63,8 @@ def embed_comfy_metadata(path, *, prompt=None, workflow=None, backup=None):target.mux(packet)if _media(temporary) != _media(path):raise ValueError('The rewritten video does not match the original')+ if exact_timescale is not None and _timing(temporary) != _timing(path):+ raise ValueError('The rewritten original soundtrack changed its timing')with av.open(str(temporary)) as check:if any(check.metadata.get(key) != value for key, value in tags.items()):raise ValueError('The workflow did not read back')@@ -110,6 +114,17 @@ def _media(path):return layout, [(counts[index], digests[index].hexdigest()) for index in sorted(digests)]+def _timing(path):+ import av+ with av.open(str(path)) as reader:+ streams = {s.index: [] for s in reader.streams}+ for p in reader.demux():+ if p.size:+ streams[p.stream.index].append((p.pts * p.time_base if p.pts is not None else None,+ p.dts * p.time_base if p.dts is not None else None, p.duration * p.time_base))+ return streams++def _copy_stream(container, template):try:return container.add_stream_from_template(template=template, opaque=True)diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.pyindex c132629..0f5dff6 100644--- a/freevideo_engine/comfy_nodes.py+++ b/freevideo_engine/comfy_nodes.py@@ -425,7 +425,8 @@ class FreeVideoMedia(io.ComfyNode):io.Audio.Input('reference_audio', display_name='Reference audio', optional=True,tooltip='Voice, music or rhythm reference, up to 15 seconds. Connect a Load Audio ''or other AUDIO output. Describe its role using <Audio 1> in the prompt. '- 'Experimental; does not lock the output soundtrack or lip sync.')],+ 'Experimental; does not guarantee lip sync.'),+ io.Combo.Input('audio_use', options=['reference', 'original'], default='reference', optional=True)],outputs=[Media.Output('media')])@classmethod@@ -444,12 +445,13 @@ class FreeVideoMedia(io.ComfyNode):return float('nan')@classmethod- def execute(cls, assets, first=None, last=None, reference=None, reference_audio=None, **images):+ def execute(cls, assets, first=None, last=None, reference=None, reference_audio=None, audio_use='reference',+ **images):from .comfy_assets import resolve_assets, connect_assetsimport folder_pathsreturn io.NodeOutput(connect_assets(resolve_assets(assets, folder_paths.get_input_directory()),first=first, last=last, reference=reference,- reference_audio=reference_audio, **images))+ reference_audio=reference_audio, audio_use=audio_use, **images))class FreeVideoReference(io.ComfyNode):@@ -460,11 +462,12 @@ class FreeVideoReference(io.ComfyNode):description='Append one image, video or audio reference in order. A clip is cut to the generated length, at most 15 s, and the cut is reported. ''Connect the stack to Generate. VDN reference quality is experimental.',inputs=[io.Image.Input('image', optional=True), io.Video.Input('video', optional=True),- io.Audio.Input('audio', optional=True), References.Input('previous', optional=True)],+ io.Audio.Input('audio', optional=True), References.Input('previous', optional=True),+ io.Combo.Input('use', options=['reference', 'original'], default='reference', optional=True)],outputs=[References.Output('references')])@classmethod- def execute(cls, image=None, video=None, audio=None, previous=None):+ def execute(cls, image=None, video=None, audio=None, previous=None, use='reference'):values = [(kind, value) for kind, value in (('image', image), ('video', video), ('audio', audio)) if value is not None]if len(values) != 1:raise ValueError('Connect exactly one image, video or audio to each Reference node')@@ -472,7 +475,11 @@ class FreeVideoReference(io.ComfyNode):if len(result) >= 32:raise ValueError('At most 32 ordered references; memory is checked against the actual request')kind, value = values[0]- return io.NodeOutput(result + [{'kind': kind, 'value': value}])+ from .original_audio import fields, original+ options = fields(dict(kind=kind, **({'use': use} if use != 'reference' or kind == 'audio' else {})))+ result.append({'kind': kind, 'value': value, **options})+ original(dict(references=result))+ return io.NodeOutput(result)class FreeVideoLoRA(io.ComfyNode):diff --git a/freevideo_engine/comfy_upload.py b/freevideo_engine/comfy_upload.pyindex 175afa8..5a724af 100644--- a/freevideo_engine/comfy_upload.py+++ b/freevideo_engine/comfy_upload.py@@ -117,4 +117,17 @@ def register():return web.json_response({'error': str(error)}, status=400)return web.json_response(result, headers={'Cache-Control': 'no-store'})+ @server.routes.post('/freevideo/media/audio-use')+ async def audio_use(request):+ from .audio_use_request import selection+ from .original_audio import AudioUseError+ try:+ value = await request.json()+ result = await asyncio.to_thread(selection, value, folder_paths.get_input_directory())+ except AudioUseError as error:+ return web.json_response(error.as_dict(), status=400)+ except (OSError, ValueError) as error:+ return web.json_response({'error': str(error)}, status=400)+ return web.json_response(result, headers={'Cache-Control': 'no-store'})+server._freevideo_upload_registered = Truediff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.pyindex d3c285c..f803fac 100644--- a/freevideo_engine/continuation.py+++ b/freevideo_engine/continuation.py@@ -93,11 +93,19 @@ def validate_frames(frames):def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=None,- media=None, backend=None, anchor=False, sound=False):+ media=None, backend=None, anchor=False, sound=False, original_audio=False):"""anchor: the part also takes pictures (an explicit anchor or the cast's pictures);- sound: the engine adds the film's opening sound as the one audio reference."""+ sound: the engine adds the film's opening sound as the one audio reference;+ original_audio: the part's sound is one audio clip used as is, apart from the references."""media = media or {}references = media.get('references')+ if original_audio:+ from .original_audio import original+ kept = original(media) if isinstance(references, list) else None+ if references and kept is None:+ raise ValueError(UNSUPPORTED_MEDIA)+ # The clip used as is is the part's own sound, not a reference; it takes no voice reference.+ references = [row for row in references if row is not kept] if references else referenceskinds = ([row.get('kind') if isinstance(row, dict) else None for row in references]if isinstance(references, list) else None)valid_reference = (kinds is not None and kinds.count('image') + kinds.count('audio') == len(kinds)@@ -106,6 +114,8 @@ def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=Nonvalid_task = task == ('ref2va' if anchor else 't2va')if sound:valid_task = valid_reference and task in (('ref2va', 'ref2va_av') if anchor else ('t2va', 'ref2va', 'ref2va_audio'))+ if original_audio:+ valid_task = not sound and task in (('ref2va', 'ref2va_av') if anchor else ('t2va', 'ref2va', 'ref2va_audio', 'ref2va_av'))if not supported()[0] or backend not in (None, 'cuda', 'mps'):raise ValueError(UNSUPPORTED_DEVICE)if refine_from:@@ -118,6 +128,7 @@ def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=Nonraise ValueError(UNSUPPORTED_MEDIA)+def encoded_references(conditions, task, anchor):"""Check a continued part's encoded references: its pictures, then the engine's sound. True when the sound is there."""rows = conditions or []@@ -480,7 +491,8 @@ def load(request, stage):value = _snapshot_value(request)# Each pass receives writable tensors of its own; sampling cannot mutate# the immutable snapshot used by the next pass or retry.- return dict(audio_context=validate_audio_context(request.get('audio_context', AUDIO_CONTEXT_DEFAULT)),+ return dict(**({'video_only': True} if request.get('video_only') else {}),+ audio_context=validate_audio_context(request.get('audio_context', AUDIO_CONTEXT_DEFAULT)),**({'anchor': request['anchor']} if request.get('anchor') else {}),video=torch.from_numpy(value['video_lo' if stage == 'first' else 'video_hi'].copy()),audio=torch.from_numpy(value['audio'].copy()))@@ -561,6 +573,7 @@ class Prefix:if type(audio_offset) is not int or audio_offset < 0:raise ValueError('Invalid continuation reference row offset')self.audio_offset = audio_offset+ self.video_only = data.get('video_only', False)self.video = data['video'].to(device=device, dtype=torch.float32)self.audio = data['audio'].to(device=device, dtype=torch.float32)if (self.video.ndim != 5 or self.video.shape[:3] != (1, transformer.config.in_channels, LATENT_FRAMES)@@ -568,7 +581,7 @@ class Prefix:raise ValueError(MISSING)self.video_rows = patchify_video_latents(self.video, tuple(transformer.config.patch_size))self.audio_rows = self.audio.permute(0, 2, 1).reshape(-1, 32).contiguous()- if self.audio_context == 'renoise':+ if not self.video_only and self.audio_context == 'renoise':generator = torch.Generator(device).manual_seed((seed + 1) % (1 << 64))self.noise = torch.randn(self.audio_rows.shape, generator=generator, device=device, dtype=torch.float32)from diffusers import MiniMaxH3Scheduler@@ -580,14 +593,16 @@ class Prefix:video, audio = kwargs['hidden_states'][0], kwargs['audio_hidden_states'][0]count, ticks = self.video_rows.shape[0], self.audio.shape[2]generated = audio.shape[0] - self.audio_offset- if video.shape[0] <= self.video_offset + count or generated // 2 < ticks:+ if video.shape[0] <= self.video_offset + count or not self.video_only and generated // 2 < ticks:raise ValueError(MISSING)self.current_video = videoself.restore_video()audio_rows = torch.cat([torch.arange(ticks, device=audio.device) + self.audio_offset + channel * (generated // 2)for channel in range(2)])row_times = kwargs['timestep'][kwargs['timestep_indices']].clone()- if self.audio_context == 'clean':+ if self.video_only:+ pass # The new segment owns the entire clean original-audio lane.+ elif self.audio_context == 'clean':audio[audio_rows] = self.audio_rowsrow_times[kwargs['audio_indices'][audio_rows]] = CLEAN_Telse:@@ -596,7 +611,13 @@ class Prefix:# scale_noise uses H3's clean-at-one convention, not a standard DDPM clock.audio[audio_rows] = self.scheduler.scale_noise(self.audio_rows, audio_t, self.noise)row_times[kwargs['video_indices'][self.video_offset:self.video_offset + count]] = CLEAN_T- kwargs['timestep'], kwargs['timestep_indices'] = torch.unique(row_times, sorted=True, return_inverse=True)+ if self.video_only:+ # Original audio deliberately retains the unused audio clock for+ # the modulation cache. Preserve that bank when pinning picture rows.+ kwargs['timestep'] = torch.unique(torch.cat((kwargs['timestep'], row_times)), sorted=True)+ kwargs['timestep_indices'] = torch.searchsorted(kwargs['timestep'], row_times)+ else:+ kwargs['timestep'], kwargs['timestep_indices'] = torch.unique(row_times, sorted=True, return_inverse=True)return args, kwargsdef restore_video(self):@@ -605,7 +626,8 @@ class Prefix:def finish(self, video, audio):video[:, :, :LATENT_FRAMES] = self.video- audio[:, :, :self.audio.shape[2]] = self.audio+ if not self.video_only:+ audio[:, :, :self.audio.shape[2]] = self.audioreturn video, audiodiff --git a/freevideo_engine/decode.py b/freevideo_engine/decode.pyindex 50832b0..2f31544 100644--- a/freevideo_engine/decode.py+++ b/freevideo_engine/decode.py@@ -26,7 +26,7 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, prepreload=False, pin_weights=False, artifacts_dir=None, phase_callback=None,stream_output=False, stream_weights=False, resident_blocks=0, model_cache=None,linear_compute_cache=False, continuation=None, audio_lead_in_callback=None, output_size=None,- hide_tail_frames=0, ram_budget_bytes=None):+ hide_tail_frames=0, ram_budget_bytes=None, original_audio=None):from .continuation import validate_hide_tailvalidate_hide_tail(hide_tail_frames)def phase(name):@@ -193,6 +193,12 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, preif continuation is not None:from .continuation import OVERLAP_FRAMESframes = frames[OVERLAP_FRAMES:]+ if original_audio is not None and 'segment' not in original_audio:+ from .original_audio import verify+ verify(original_audio)+ count = original_audio['end_sample'] - original_audio['start_sample']+ rate = original_audio['sample_rate']+ frames = frames[:(count * FPS + rate - 1) // rate]artifact_seconds = streamed_timings.get('decoded_artifact_save_seconds', 0.)if artifacts_dir:import numpy as np@@ -200,7 +206,7 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, pretick = time.perf_counter()artifacts_dir = Path(artifacts_dir)artifacts_dir.mkdir(parents=True, exist_ok=True)- if not stream_output or continuation is not None or hide_tail_frames:+ if not stream_output or continuation is not None or hide_tail_frames or original_audio is not None:# A streamed mapping still contains overlap/hidden frames. Replace its# retained artifact after the mapping closes, below.rgb_path = artifacts_dir / ('rgb-new.npy' if stream_output else 'rgb.npy')@@ -211,77 +217,103 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, preimport numpy as npnp.save(directory / 'rgb-new.npy', frames.numpy() if hasattr(frames, 'numpy') else frames,allow_pickle=False)- audio_start = time.perf_counter()- phase('audio_load_decode')- audio_vae = None- if model_cache is not None:- from .resident_models import key, file_identity- audio_key = key([file_identity(path) for path in sorted((Path(base)/'audio_vae').iterdir()) if path.is_file()])- audio_vae = model_cache.take('audio_vae', audio_key)- model_cache.make_room('audio_vae', 1*2**30, model=audio_vae, ram_need=256*2**20)- audio_cache_hit = audio_vae is not None- if audio_vae is None:- from .vae_weights import load_audio_vae- audio_vae, _ = load_audio_vae(base)- if model_cache is not None:- model_cache.put('audio_vae', audio_key, audio_vae)- audio_vae.eval().requires_grad_(False)- audio_mean = torch.tensor(audio_vae.config.latents_mean, device='cuda').view(1, -1, 1)- audio_std = torch.tensor(audio_vae.config.latents_std, device='cuda').view(1, -1, 1)- print(json.dumps({'event': 'decode_phase', 'phase': 'Decoding audio'}), flush=True)- audio = audio_vae.decode(audio_latents * audio_std + audio_mean, return_dict=False)[0]- audio = audio.float().permute(1, 0, 2)[0].cpu()- rate = audio_vae.config.sampling_rate- if continuation is not None:- if audio_lead_in_callback is not None:- from .continuation import capture_audio_lead_in- audio_lead_in_callback(capture_audio_lead_in(audio, continuation, rate))- if hide_tail_frames:- audio = audio[:, :round(kept_frames * 40 / 24) * 800]- if continuation is not None:- audio = audio[:, continuation['audio_latents'] * 800:]- audio_level = {'audio_rms': float(audio.square().mean().sqrt()) if audio.numel() else 0.,- 'audio_peak': float(audio.abs().max()) if audio.numel() else 0.,- 'audio_samples': int(audio.shape[-1])}- del audio_vae, audio_mean, audio_std- gc.collect()- torch.cuda.empty_cache()- audio_seconds = time.perf_counter() - audio_start- if artifacts_dir:- import wave- phase('audio_save')- tick = time.perf_counter()- np.save(artifacts_dir / 'audio.npy', audio.numpy(), allow_pickle=False)- # Preserve exact float audio above; WAV is an independently playable copy.- pcm = (audio.clamp(-1, 1).t().contiguous().numpy() * 32767).round().astype('<i2')- with wave.open(str(artifacts_dir / 'audio.wav'), 'wb') as wav:- wav.setnchannels(audio.shape[0])- wav.setsampwidth(2)- wav.setframerate(rate)- wav.writeframes(pcm.tobytes())- artifact_seconds += time.perf_counter() - tick+ original_receipt = None+ if original_audio is not None:+ # The clip used as is is the sound; the model's audio is not decoded.+ audio, rate, audio_seconds, audio_cache_hit, audio_level = None, original_audio['sample_rate'], 0., False, {}+ else:+ audio_start = time.perf_counter()+ phase('audio_load_decode')+ audio_vae = None+ if model_cache is not None:+ from .resident_models import key, file_identity+ audio_key = key([file_identity(path) for path in sorted((Path(base)/'audio_vae').iterdir()) if path.is_file()])+ audio_vae = model_cache.take('audio_vae', audio_key)+ model_cache.make_room('audio_vae', 1*2**30, model=audio_vae, ram_need=256*2**20)+ audio_cache_hit = audio_vae is not None+ if audio_vae is None:+ from .vae_weights import load_audio_vae+ audio_vae, _ = load_audio_vae(base)+ if model_cache is not None:+ model_cache.put('audio_vae', audio_key, audio_vae)+ audio_vae.eval().requires_grad_(False)+ audio_mean = torch.tensor(audio_vae.config.latents_mean, device='cuda').view(1, -1, 1)+ audio_std = torch.tensor(audio_vae.config.latents_std, device='cuda').view(1, -1, 1)+ print(json.dumps({'event': 'decode_phase', 'phase': 'Decoding audio'}), flush=True)+ audio = audio_vae.decode(audio_latents * audio_std + audio_mean, return_dict=False)[0]+ audio = audio.float().permute(1, 0, 2)[0].cpu()+ rate = audio_vae.config.sampling_rate+ if continuation is not None:+ if audio_lead_in_callback is not None:+ from .continuation import capture_audio_lead_in+ audio_lead_in_callback(capture_audio_lead_in(audio, continuation, rate))+ if hide_tail_frames:+ audio = audio[:, :round(kept_frames * 40 / 24) * 800]+ if continuation is not None:+ audio = audio[:, continuation['audio_latents'] * 800:]+ audio_level = {'audio_rms': float(audio.square().mean().sqrt()) if audio.numel() else 0.,+ 'audio_peak': float(audio.abs().max()) if audio.numel() else 0.,+ 'audio_samples': int(audio.shape[-1])}+ del audio_vae, audio_mean, audio_std+ gc.collect()+ torch.cuda.empty_cache()+ audio_seconds = time.perf_counter() - audio_start+ if artifacts_dir:+ import wave+ phase('audio_save')+ tick = time.perf_counter()+ np.save(artifacts_dir / 'audio.npy', audio.numpy(), allow_pickle=False)+ # Preserve exact float audio above; WAV is an independently playable copy.+ pcm = (audio.clamp(-1, 1).t().contiguous().numpy() * 32767).round().astype('<i2')+ with wave.open(str(artifacts_dir / 'audio.wav'), 'wb') as wav:+ wav.setnchannels(audio.shape[0])+ wav.setsampwidth(2)+ wav.setframerate(rate)+ wav.writeframes(pcm.tobytes())+ artifact_seconds += time.perf_counter() - tickdestination = Path(out_path)destination.parent.mkdir(parents=True, exist_ok=True)temporary = destination.with_name(destination.stem + '.partial' + destination.suffix)encode_started = time.perf_counter()phase('mp4_save')+ picture = framesif output_size is not None:# The model rendered a larger size; only the picture is scaled.from .output_fit import fit_frames- encode_video(fit_frames(frames, *output_size), fps=FPS, output_path=str(temporary), audio=audio,- audio_sample_rate=rate)+ picture = fit_frames(frames, *output_size)+ if original_audio is not None:+ from .original_audio import mux+ # Encode picture only; audio never enters the model decoder or AAC encoder.+ try:+ encode_video(picture, fps=FPS, output_path=str(temporary))+ if 'segment' in original_audio:+ # Keep the complete new picture grid for the ordinary+ # join. Standalone/editor block playback still has its+ # exact source sound; the final join ignores this track.+ info = dict(original_audio, start_sample=original_audio['segment']['start_sample'],+ end_sample=original_audio['segment']['end_sample'], full_source=False)+ original_receipt = mux(temporary, destination, info, trim_video=False)+ original_receipt['segment'] = original_audio['segment']+ else:+ original_receipt = mux(temporary, destination, original_audio)+ if artifacts_dir:+ from .monitoring import save+ save(artifacts_dir / 'original-audio.json', original_receipt)+ finally:+ temporary.unlink(missing_ok=True)else:- encode_video(frames, fps=FPS, output_path=str(temporary), audio=audio, audio_sample_rate=rate)- temporary.replace(destination)+ encode_video(picture, fps=FPS, output_path=str(temporary), audio=audio, audio_sample_rate=rate)+ temporary.replace(destination)frame_count = len(frames)finally:if rgb_mapping is not None:del framesrgb_mapping._mmap.close()retained = Path(artifacts_dir) if artifacts_dir else directory- if (continuation is not None or hide_tail_frames) and (retained / 'rgb-new.npy').is_file():+ if (continuation is not None or hide_tail_frames or original_audio is not None) and (retained / 'rgb-new.npy').is_file():(retained / 'rgb-new.npy').replace(retained / 'rgb.npy')- return {'vae_load_seconds': load_seconds, 'video_decode_seconds': video_decode_seconds,+ return {**({'original_audio': original_receipt} if original_receipt is not None else {}),+ 'vae_load_seconds': load_seconds, 'video_decode_seconds': video_decode_seconds,'vae_linear_compute_cache': linear_compute_cache, 'vae_compute_cache_preparation': compute_cache_metrics,'resident_vae_cache_hit': vae_cache_hit, 'resident_audio_vae_cache_hit': audio_cache_hit,'vae_preload_seconds': preload_seconds,diff --git a/freevideo_engine/decode_resume.py b/freevideo_engine/decode_resume.pyindex c02a964..635497f 100644--- a/freevideo_engine/decode_resume.py+++ b/freevideo_engine/decode_resume.py@@ -150,6 +150,10 @@ def sampling_provenance(request):snapshot_sha256=_digest(tail['snapshot']), source_id=tail['source_id'])result['continuation'] = {key: item for key, item in value.items()if key not in ('snapshot', 'snapshot_data', 'source_video')}+ if request.get('original_audio') is not None:+ from .original_audio import verify, identity+ verify(request['original_audio'])+ result['original_audio'] = identity(request['original_audio'])return resultdiff --git a/freevideo_engine/fixed_track.py b/freevideo_engine/fixed_track.pynew file mode 100644index 0000000..fb1ac13--- /dev/null+++ b/freevideo_engine/fixed_track.py@@ -0,0 +1,24 @@+"""Clean target audio for either sampling pass; no archived-video requirement."""+++def split(conditions, frames, device):+ import torch+ fixed = [row for row in conditions if row.get('use') == 'original']+ if not fixed:+ return conditions, None+ if len(fixed) != 1:+ from .original_audio import AudioUseError+ raise AudioUseError('audio_original_multiple')+ sound = fixed[0]['audio_latent']+ expected = (2 * round(frames / 24 * 40), 32)+ if (not isinstance(sound, torch.Tensor) or tuple(sound.shape) != expected+ or not sound.is_floating_point() or not bool(torch.isfinite(sound).all())):+ raise ValueError('Original audio latent shape or values are invalid')+ return [row for row in conditions if row.get('use') != 'original'], sound.to(device=device, dtype=torch.float32)+++def timesteps(row_times, video_t, audio_t):+ """Keep the unused audio clock in the existing reference modulation bank."""+ import torch+ times = torch.unique(torch.cat((row_times, video_t.float().reshape(1), audio_t.float().reshape(1))), sorted=True)+ return times, torch.searchsorted(times, row_times)diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.pyindex d5815e6..a2cac74 100644--- a/freevideo_engine/generate.py+++ b/freevideo_engine/generate.py@@ -351,6 +351,8 @@ def run(args):phase=planning[-1]['stage'] if planning else 'preflight', resource_planning=planning,error_type=type(error).__name__, exception=exception_details(error),error_message=str(error), resource_error=getattr(error, 'details', None), runtime_code=code_identity())+ if getattr(error, 'code', None):+ record['error_code'] = error.code_runtime_libraries(error, record)try:save(output.with_suffix('.request.json'), record)@@ -418,7 +420,8 @@ def _run(args):raise# A record of another version, other weights or another first-pass size.value = dict(encoded_continuation(continue_from, canvas),- **{key: value[key] for key in ('audio_context', 'source_video', 'anchor', 'pictures', 'sound') if key in value})+ **{key: value[key] for key in ('audio_context', 'source_video', 'anchor', 'pictures', 'sound', 'video_only')+ if key in value})return freeze_continuation(value) if continuation_directory is not None else valueframes = getattr(args, 'frames', None)@@ -434,15 +437,23 @@ def _run(args):if continue_anchor is not None:raise ValueError('Choose --first-frame-reference or --continue-anchor, not both')first_frame = dict(requested=True, used=False)+ from .media_request import read as read_media, task_for, cache_key+ from .original_audio import cli_media, prepare_request+ media = cli_media(args, read_media(getattr(args, 'media', None)))+ original_audio, original_canvas = prepare_request(media, getattr(args, 'width', 1344),+ getattr(args, 'height', 768), continuation=continue_from is not None)if continue_from is not None and frames is not None:validate_frames(frames)- canvas = geometry(getattr(args, 'width', 1344), getattr(args, 'height', 768),+ canvas = original_canvas or geometry(getattr(args, 'width', 1344), getattr(args, 'height', 768),frames=frames, seconds=getattr(args, 'seconds', None))visible_canvas = canvascanvas, hide_tail = hide_tail_geometry(canvas,args.prompt_file.read_text(encoding='utf-8') if getattr(args, 'hide_tail', False) and args.prompt_file else None,- getattr(args, 'hide_tail', False),+ getattr(args, 'hide_tail', False) and original_audio is None,preview=bool(getattr(args, 'preview', False) or getattr(args, 'refine_from', None)))+ if original_audio is not None and getattr(args, 'hide_tail', False):+ # The clip's samples set where the part ends; nothing is generated past them to hide.+ hide_tail = dict(requested=True, used=False, reason='original_audio')hide_tail_frames = hide_tail['frames'] if hide_tail and hide_tail['used'] else 0continuation = Nonefrom .two_pass import validate_steps@@ -457,16 +468,16 @@ def _run(args):validate_steps(requested_steps, refine_steps, two_pass)if requested_steps != 8:canvas['steps'] = requested_steps- from .media_request import read as read_media, task_for, cache_key- media = read_media(getattr(args, 'media', None))if continue_anchor is not None:from .continuation import anchor_mediamedia = anchor_media(media, continue_anchor)# The cast's pictures, sent with a continued part like with a part that starts fresh.+ # A clip used as is is the part's sound, not one of its pictures.pictures = []if continue_from is not None and continue_anchor is None and media.get('references'):from .continuation import continuation_pictures- pictures = continuation_pictures(media)+ pictures = continuation_pictures(dict(media, references=[+ row for row in media['references'] if row.get('use') != 'original']))anchored = continue_anchor is not None or bool(pictures)# Too small to sample cleanly: render larger, scale the picture down. Decided before anything# compares or stores the canvas, so every later check sees the size that is sampled.@@ -476,7 +487,8 @@ def _run(args):if continue_from is not None:validate_route(task=getattr(args, 'task', None) or ('ref2va' if anchored else 't2va'),anchor=anchored, two_pass=two_pass,- preview=preview, refine_from=getattr(args, 'refine_from', None), media=media)+ preview=preview, refine_from=getattr(args, 'refine_from', None), media=media,+ original_audio=original_audio is not None and 'segment' in original_audio)# Reject absent/mismatched context before text encoding or model loading.if getattr(args, 'continuation_snapshot', None):continuation = restore_snapshot(args.continuation_snapshot,@@ -490,7 +502,14 @@ def _run(args):# Nothing was stored for this file: its own ending is encoded when the run starts.continuation = encoded_continuation(continue_from, canvas)continuation = dict(continuation, audio_context=audio_context(), source_video=str(continue_from))- if first_frame is not None and pictures:+ if original_audio is not None:+ from .original_audio_chain import validate_continuation+ validate_continuation(original_audio, continuation)+ continuation.update(video_only=True, audio_context='clean')+ if first_frame is not None and original_audio is not None:+ # The clip used as is is the part's only audio row; no first-frame picture is added beside it.+ first_frame['reason'] = 'original_audio'+ elif first_frame is not None and pictures:# The cast's pictures already hold the people; the film's first frame is not added to them.first_frame['reason'] = 'pictures'elif first_frame is not None:@@ -521,7 +540,9 @@ def _run(args):if pictures:continuation['pictures'] = len(pictures)if voice_reference is not None:- if first_frame is not None and not pictures:+ if original_audio is not None:+ voice_reference['reason'] = 'original_audio'+ elif first_frame is not None and not pictures:voice_reference['reason'] = 'first_frame'elif continue_anchor is not None:voice_reference['reason'] = 'anchor'@@ -593,7 +614,8 @@ def _run(args):manifest = json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8'))if continuation is not None:from .two_pass import plan as continuation_plan- validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored)+ validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored,+ original_audio=original_audio is not None)planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps,scale=upscale_factor(args))if two_pass and not planned['enabled'] and base_steps != 8:@@ -870,6 +892,10 @@ def _run(args):if 'early_reference' in info:canvas['early_reference'] = info['early_reference']profile['engine']['task'] = info['task']+ if original_audio is not None:+ # Include all clean clock constants in live re-planning;+ # the original track adds no reference audio rows.+ args.task = info['task']if not condition.is_file():raise FileNotFoundError(condition)from .two_pass import plan as plan_sampling, ensure_checkpoint@@ -879,6 +905,7 @@ def _run(args):report['sampling_plan'] = sampling_planif continuation is not None:validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored,+ original_audio=original_audio is not None,**(dict(sound=True, media=media) if voice_reference and voice_reference['used'] else {}))continuation = usable_continuation(continuation, canvas,json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')), sampling_plan)@@ -981,6 +1008,9 @@ def _run(args):request['continuation'] = continuationif hide_tail_frames:request['hide_tail_frames'] = hide_tail_frames+ if original_audio is not None:+ request['original_audio'] = original_audio+ report['original_audio'] = original_audioif preview:request['preview'] = Trueif allocator_limit_bytes is not None:@@ -1004,7 +1034,8 @@ def _run(args):if continuation is not None:from .continuation import ADALN_TASK, ANCHOR_ADALN_TASKselected['engine'] = dict(selected['engine'],- adaln_task=ANCHOR_ADALN_TASK if anchored else ADALN_TASK)+ adaln_task=(selected['engine']['task'] if original_audio is not None+ else ANCHOR_ADALN_TASK if anchored else ADALN_TASK))request.update(engine_options=selected['engine'], decoder_options=selected['decoder'],ram_budget_bytes=round(selected['inference_ram_budget_gb'] * 1e9),ram_budget_is_estimate=ram_budget_is_estimate(args),diff --git a/freevideo_engine/media_conditioning.py b/freevideo_engine/media_conditioning.pyindex 2a067f2..770083d 100644--- a/freevideo_engine/media_conditioning.py+++ b/freevideo_engine/media_conditioning.py@@ -100,6 +100,7 @@ def _describe(value, width, height, frames):to_cache([[prompt[None], {'minimax_token_tags': tags}]], task='ref2va')video_rows = audio_rows = seen_rows = 0guides = 0+ original_count = 0for ref in refs:kind = ref.get('kind')if ref.get('guide'):@@ -130,12 +131,20 @@ def _describe(value, width, height, frames):audio = _tensor(ref.get('audio_latent'), (None, 32), 'reference audio rows')if audio.shape[0] % 2:raise ValueError('Reference audio must contain both channels')- audio_rows += audio.shape[0]+ if ref.get('use') == 'original':+ original_count += 1+ if kind != 'audio' or audio.shape[0] != 2 * round(frames / 24 * 40):+ raise ValueError('Original audio latents do not match the generation duration')+ else:+ audio_rows += audio.shape[0]# Guide rows are never shown to the vision-language encoder, so only the# other visual references need jointly encoded vision tokens.if seen_rows and not bool((tags == 0).any()):raise ValueError('Visual references need jointly encoded vision tokens')- task = 'ref2va_av' if video_rows and audio_rows else 'ref2va_audio' if audio_rows else 'ref2va'+ if original_count > 1:+ from .original_audio import AudioUseError+ raise AudioUseError('audio_original_multiple')+ task = 'ref2va_av' if video_rows and (audio_rows or original_count) else 'ref2va_audio' if (audio_rows or original_count) else 'ref2va'value['task'] = taskreturn {'task': task, 'text_tokens': len(tags), 'reference_video_tokens': video_rows,'reference_audio_tokens': audio_rows, 'width': width, 'height': height, 'frames': frames}@@ -177,6 +186,8 @@ def from_comfy(conditioning, width, height, frames):if refs:value['references'] = []for row in refs:+ if row.get('use') == 'original':+ raise ValueError('Use as is needs the source audio file or AUDIO socket, not preencoded reference latents')kind = row['kind']ref = {'kind': 'video' if kind == 'video_audio' else kind}if kind in ('image', 'video', 'video_audio'):diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.pyindex cacccd8..7171ecd 100644--- a/freevideo_engine/media_encoding.py+++ b/freevideo_engine/media_encoding.py@@ -132,6 +132,15 @@ def prepare(media, canvas, directory):result.append({'kind': 'image', 'anchor': anchor, 'pixels': str(target), 'frames': 1})for index, row in enumerate(media.get('references', [])):kind, path = row['kind'], row['path']+ if row.get('use') == 'original':+ from .original_audio import model_audio, inspect+ numbers[kind] += 1+ info = row.get('original_audio') or inspect(row)+ target = directory / ('original-%02d.npy' % index)+ result.append(dict(kind='audio', use='original', original_audio=info,+ audio=model_audio(info, canvas, target)))+ ref_items.append({'type': 'audio'})+ continueref = {'kind': kind}if 'early_steps' in row:ref['early_steps'] = row['early_steps']@@ -356,6 +365,13 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuaudio = torch.from_numpy(np.load(row['audio'], allow_pickle=False)).to(device)latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2)ref['audio_latent'] = ((latent - mean) / std).reshape(-1, 32).contiguous()+ if row.get('use') == 'original':+ ticks = round(canvas['frames'] / 24 * 40)+ sound = ref['audio_latent'].reshape(2, -1, 32)+ if sound.shape[1] < ticks:+ raise ValueError('Original audio encoder returned too few latent frames')+ ref.update(use='original', original_audio=row['original_audio'],+ audio_latent=sound[:, :ticks].reshape(-1, 32).contiguous())del audio, latentdel vaegc.collect()diff --git a/freevideo_engine/media_request.py b/freevideo_engine/media_request.pyindex 56571c7..12eda47 100644--- a/freevideo_engine/media_request.py+++ b/freevideo_engine/media_request.py@@ -55,7 +55,8 @@ def read(path):raise ValueError('Invalid ' + field + ' entry')if field != 'references' and 'early_steps' in entry:raise ValueError('early_steps is only allowed on video references')- allowed = {'path', 'kind', 'early_steps', 'sound', 'guide', 'keep_source_audio'} if field == 'references' else {'path', 'strength', 'alpha'}+ allowed = ({'path', 'kind', 'early_steps', 'sound', 'guide', 'keep_source_audio', 'use', 'range', 'segment'} if field == 'references'+ else {'path', 'strength', 'alpha'})if set(entry) - allowed:raise ValueError('Unknown ' + field + ' fields')row = file_info(entry.get('path'))@@ -80,6 +81,8 @@ def read(path):validate_keep_source_audio(entry, row['path'])if entry['keep_source_audio']:row['keep_source_audio'] = True+ from .original_audio import fields+ row.update(fields(entry))else:if Path(row['path']).suffix.lower() != '.safetensors':raise ValueError('LoRA requires safetensors; pickle adapters are not loaded')@@ -90,6 +93,8 @@ def read(path):raise ValueError('LoRA ' + name + ' must be finite')row[name] = numberresult[field].append(row)+ from .original_audio import original+ original(result)if result['references'] and any(result.get(k) for k in ('first', 'last')):raise ValueError('Reference and first/last keyframe layouts cannot be mixed in one request; ''mark the clip that carries the motion with "guide": true instead')@@ -179,6 +184,11 @@ def cache_key(prompt, media, canvas):if value:identity[key] = [{k: v for k, v in row.items() if k not in ('path', 'keep_source_audio')}for row in value] if isinstance(value, list) else value['sha256']+ if key == 'references':+ from .original_audio import identity as audio_identity+ for row in identity[key]:+ if row.get('original_audio'):+ row['original_audio'] = audio_identity(row['original_audio'])return 'multimodal:' + json.dumps(identity, sort_keys=True, ensure_ascii=False)diff --git a/freevideo_engine/original_audio.py b/freevideo_engine/original_audio.pynew file mode 100644index 0000000..0186c1b--- /dev/null+++ b/freevideo_engine/original_audio.py@@ -0,0 +1,537 @@+"""Original soundtrack contract, integer sample selection and lossless export.++Only the opt-in path imports media libraries. Reference requests retain their+old serialized representation and never pass through audio inspection here.+"""+from fractions import Fraction+from pathlib import Path+import math++VERSION = 'original-clean-v1'+MAX_SECONDS = 30+MESSAGES = {+ 'audio_use_invalid': ('Set the audio to "Reference" or "Use as is".', '音频只能选“参考”或“原样使用”。'),+ 'audio_original_multiple': ('Only one audio file can be set to "Use as is".', '只能有一段音频选“原样使用”。'),+ 'audio_original_format': ('This format cannot be used as is. Try a WAV, FLAC or M4A file.', '这个格式不能原样使用,请换成 WAV、FLAC 或 M4A 文件。'),+ 'audio_original_range': ('The selected range is outside the audio. Select it again.', '所选范围超出了这段音频,请重新选择。'),+ 'audio_original_short': ('The selected audio is too short. Select at least 2 seconds.', '所选音频太短,请至少选 2 秒。'),+ 'audio_original_long': ('This audio is longer than 30 seconds. Generate it in the Create view, or use an audio file of 30 seconds or less.', '这段音频超过 30 秒,请到创作面板里生成,或换一段 30 秒以内的音频。'),+ 'audio_original_limit': ('This audio is longer than 2 hours. Use an audio file of 2 hours or less.', '这段音频超过 2 小时,请换一段 2 小时以内的音频。'),+ 'audio_original_part': ('With "Use as is", each part can be up to 30 seconds.', '用“原样使用”时,每一段最长 30 秒。'),+ 'audio_original_route': ('"Use as is" is not available here yet. Set the audio to "Reference".', '这里还不能用“原样使用”的音频,请改选“参考”。'),+ 'audio_original_backend': ('"Use as is" is not available on this computer yet.', '这台电脑上还不能用“原样使用”。'),+ 'audio_original_slot': ('The selected audio is longer than this part. Make the part longer or select less audio.', '所选音频比这一段长,请把这一段拉长,或少选一些音频。'),+}+++def _too_long(max_seconds):+ # One generation takes 30 seconds; only a background chain is asked for more, up to its own limit.+ return AudioUseError('audio_original_long' if max_seconds <= MAX_SECONDS else 'audio_original_limit')+++class AudioUseError(ValueError):+ def __init__(self, code):+ self.code = code+ self.message_en, self.message_zh = MESSAGES[code]+ super().__init__(f'{code}: {self.message_en} / {self.message_zh}')++ def as_dict(self):+ return dict(code=self.code, message=self.message_en, message_zh=self.message_zh)+++def fields(row):+ """Sparse canonical fields: explicit reference and omission are identical."""+ if not isinstance(row, dict):+ raise AudioUseError('audio_use_invalid')+ use = row.get('use', 'reference')+ if use not in ('reference', 'original') or ('use' in row and row.get('kind', 'audio') != 'audio'):+ raise AudioUseError('audio_use_invalid')+ selected = row.get('range')+ if selected is not None:+ if (use != 'original' or not isinstance(selected, dict)+ or set(selected) != {'start_sample', 'end_sample'}+ or any(type(selected[k]) is not int for k in selected)+ or not 0 <= selected['start_sample'] < selected['end_sample']):+ raise AudioUseError('audio_original_range')+ result = (dict(use='original', **({'range': dict(selected)} if selected is not None else {}))+ if use == 'original' else {})+ if 'segment' in row:+ segment = row['segment']+ keys = {'frames', 'frame_origin', 'start_sample', 'end_sample',+ 'source_start_sample', 'source_end_sample', 'source_sha256'}+ if (use != 'original' or selected is None or not isinstance(segment, dict) or set(segment) != keys+ or any(type(segment[k]) is not int for k in keys - {'source_sha256'})+ or not isinstance(segment['source_sha256'], str) or len(segment['source_sha256']) != 64+ or segment['frames'] < 39 or (segment['frames'] - 5) % 17+ or segment['frame_origin'] < 0+ or not 0 <= segment['source_start_sample'] <= selected['start_sample'] <= segment['start_sample']+ < segment['end_sample'] == selected['end_sample'] <= segment['source_end_sample']):+ raise AudioUseError('audio_original_range')+ result['segment'] = dict(segment)+ return result+++def original(media):+ rows = [row for row in media.get('references', []) if fields(row).get('use') == 'original']+ if len(rows) > 1:+ raise AudioUseError('audio_original_multiple')+ return rows[0] if rows else None+++def cli_media(args, media):+ """Apply CLI shorthand before planning, with no changes for old calls."""+ from .media_request import digest+ path, use = getattr(args, 'audio', None), getattr(args, 'audio_use', None)+ if path is not None:+ path = Path(path).expanduser().resolve(strict=True)+ media = dict(media or {'version': 1, 'references': [], 'loras': []})+ row = dict(path=str(path), sha256=digest(path), bytes=path.stat().st_size, kind='audio')+ row.update(fields(dict(kind='audio', use=use or 'reference')))+ media['references'] = [*media.get('references', []), row]+ elif use is not None:+ rows = [row for row in media.get('references', []) if row['kind'] == 'audio']+ if len(rows) != 1:+ raise AudioUseError('audio_use_invalid')+ rows[0].pop('use', None)+ rows[0].update(fields(dict(rows[0], use=use)))+ original(media)+ if media.get('references') and any(media.get(k) for k in ('first', 'last')):+ raise ValueError('Reference and keyframe layouts cannot be mixed; use separate requests')+ return media+++def prepare_request(media, width, height, *, continuation=False, backend='cuda'):+ row = original(media)+ if row is None:+ return None, None+ segment = row.get('segment')+ if continuation and segment is None:+ raise AudioUseError('audio_original_route')+ if backend != 'cuda':+ raise AudioUseError('audio_original_backend')+ info = inspect(row, max_seconds=(segment['frames'] + 23) // 24) if segment else inspect(row)+ if segment is not None:+ from .geometry import geometry+ from .original_audio_chain import sample_at+ rate = info['sample_rate']+ prefix = 22 if segment['frame_origin'] else 0+ origin, frames = segment['frame_origin'], segment['frames']+ if (frames > geometry(frames=720)['frames'] or info['sha256'] != segment['source_sha256']+ or continuation != bool(origin)+ or info['start_sample'] != segment['source_start_sample'] + sample_at(origin, rate)+ or segment['start_sample'] != segment['source_start_sample'] + sample_at(origin + prefix, rate)+ or segment['end_sample'] != min(segment['source_end_sample'],+ segment['source_start_sample'] + sample_at(origin + frames, rate))):+ raise AudioUseError('audio_original_range')+ info['segment'] = dict(segment)+ canvas = geometry(width, height, frames=frames)+ else:+ canvas = canvas_for(info, width, height)+ # Source identity and normalization version become part of the encoder key.+ row['original_audio'] = info+ return info, canvas+++def _samples(frame):+ """Channel-major native decoded samples, without a resampler or gain."""+ data = frame.to_ndarray()+ return data if frame.format.is_planar else data.reshape(-1, len(frame.layout.channels)).T+++def inspect(row, *, max_seconds=MAX_SECONDS):+ """Read a bounded selection on the source's integer decoded-sample clock.++ An explicit range may start late in a long file; scanning uses one decoded+ frame at a time and stops after that range. An unselected long file fails+ as soon as its length exceeds the single-generation limit.+ """+ import av+ from .media_request import digest+ path = Path(row['path'])+ selection = row.get('range') or {}+ start, end = selection.get('start_sample', 0), selection.get('end_sample')+ fields(row)+ try:+ with av.open(str(path)) as container:+ stream = container.streams.audio[0]+ rate, channels = stream.rate, len(stream.layout.channels)+ if not rate or not 8000 <= rate <= 192000 or channels not in (1, 2):+ raise AudioUseError('audio_original_format')+ if end is not None and end - start > rate * max_seconds:+ raise _too_long(max_seconds)+ count = 0+ for frame in container.decode(audio=0):+ if frame.sample_rate != rate or len(frame.layout.channels) != channels:+ raise AudioUseError('audio_original_format')+ count += frame.samples+ if end is not None and count >= end:+ break+ if end is None and count > rate * max_seconds:+ raise _too_long(max_seconds)+ end = count if end is None else end+ if not start < end <= count:+ raise AudioUseError('audio_original_range')+ if (end - start) * 24 < 39 * rate:+ raise AudioUseError('audio_original_short')+ codec = stream.codec_context.name+ except (IndexError, av.error.FFmpegError, OSError, ValueError) as error:+ if isinstance(error, AudioUseError):+ raise+ raise AudioUseError('audio_original_format') from error+ sha = digest(path)+ if row.get('sha256') and sha != row['sha256']:+ raise ValueError('Original audio changed after inspection')+ return dict(version=VERSION, path=str(path.resolve()), sha256=sha, bytes=path.stat().st_size,+ sample_rate=rate, channels=channels, start_sample=start, end_sample=end,+ codec=codec, full_source=not selection,+ normalization='PyAV-native-to-fltp-32000-v1',+ decoder_version=av.__version__,+ library_versions={k: list(v) for k, v in av.library_versions.items()})+++def identity(info):+ return {key: value for key, value in info.items() if key != 'path'}+++def verify(info):+ from .media_request import digest+ path = Path(info['path'])+ if path.stat().st_size != info['bytes'] or digest(path) != info['sha256']:+ raise ValueError('Original audio changed after inspection; select the audio again')+++def canvas_for(info, width, height, *, max_seconds=MAX_SECONDS):+ from .geometry import geometry+ count, rate = info['end_sample'] - info['start_sample'], info['sample_rate']+ if count * 24 < 39 * rate:+ raise AudioUseError('audio_original_short')+ if count > max_seconds * rate:+ raise _too_long(max_seconds)+ canvas = geometry(width, height, frames=(count * 24 + rate - 1) // rate)+ canvas['requested_seconds'] = float(Fraction(count, rate))+ return canvas+++def _selected_chunks(info):+ """The selection in native samples, one decoded frame at a time."""+ import av+ import numpy as np+ offset = count = 0+ with av.open(info['path']) as container:+ for frame in container.decode(audio=0):+ begin, end = max(0, info['start_sample'] - offset), min(frame.samples, info['end_sample'] - offset)+ if begin < end:+ data = _samples(frame)[:, begin:end]+ if data.shape[0] != info['channels'] or not np.isfinite(data).all():+ raise AudioUseError('audio_original_format')+ count += data.shape[1]+ yield data+ offset += frame.samples+ if offset >= info['end_sample']:+ break+ if not count:+ raise AudioUseError('audio_original_range')+ if count != info['end_sample'] - info['start_sample']:+ raise AudioUseError('audio_original_format')+++def selected_samples(info):+ """The whole selection in memory: for one generation's worth, not for a long chain."""+ import numpy as np+ verify(info)+ return np.concatenate(list(_selected_chunks(info)), axis=1)+++def model_audio(info, canvas, destination):+ """Resample only the conditioning copy; retain original rate and samples."""+ import av+ import numpy as np+ samples = selected_samples(info)+ rate, channels = info['sample_rate'], info['channels']+ layout = 'mono' if channels == 1 else 'stereo'+ dtype = samples.dtype+ # Convert native integers to float using the codec's power-of-two scale.+ if dtype.kind in 'iu':+ samples = ((samples.astype(np.float32) - 128) / 128 if dtype == np.uint8+ else samples.astype(np.float32) / (1 << (dtype.itemsize * 8 - 1)))+ samples = np.ascontiguousarray(samples, dtype=np.float32)+ resampler = av.AudioResampler(format='fltp', layout=layout, rate=32000)+ frame = av.AudioFrame.from_ndarray(samples, format='fltp', layout=layout)+ frame.sample_rate, frame.pts, frame.time_base = rate, 0, Fraction(1, rate)+ parts = [item.to_ndarray() for item in resampler.resample(frame) + resampler.resample(None)]+ data = np.concatenate(parts, axis=1)+ if channels == 1:+ data = np.repeat(data, 2, axis=0)+ # Match the sampler's 40 Hz audio latent grid, including its final rounding.+ ticks = round(canvas['frames'] / 24 * 40)+ target = max(math.ceil(canvas['frames'] * 32000 / 24), ticks * 800)+ data = np.pad(data, ((0, 0), (0, max(0, target - data.shape[1]))))+ np.save(destination, data, allow_pickle=False)+ return str(destination)+++def av_errors():+ import av+ return av.FFmpegError+++_FORMATS = {'pcm_f32le': 'flt', 'pcm_f64le': 'dbl', 'pcm_s32le': 's32', 'pcm_s16le': 's16',+ 'alac': 's16p', 'flac': 's16', 'aac': 'fltp'}+++def _audio_stream(container, codec, rate, channels, itemsize):+ stream = container.add_stream(codec, rate=rate, layout='mono' if channels == 1 else 'stereo')+ fmt = _FORMATS[codec]+ stream.codec_context.format = fmt+ if codec == 'flac' and itemsize == 4:+ fmt = stream.codec_context.format = 's32'+ return stream, fmt+++def _encoded(stream, fmt, chunks, rate):+ """Packets for channel-first chunks of any length, encoded as they arrive."""+ import av+ import numpy as np+ size = stream.codec_context.frame_size or 4096+ layout = stream.codec_context.layout.name+ position, held = 0, None++ def frame(chunk):+ nonlocal position+ chunk = np.ascontiguousarray(chunk)+ array = chunk if fmt.endswith('p') else chunk.T.reshape(1, -1)+ value = av.AudioFrame.from_ndarray(np.ascontiguousarray(array), format=fmt, layout=layout)+ value.sample_rate, value.pts, value.time_base = rate, position, Fraction(1, rate)+ position += chunk.shape[1]+ return stream.encode(value)++ for chunk in chunks:+ held = chunk if held is None else np.concatenate((held, chunk), axis=1)+ while held.shape[1] >= size:+ yield from frame(held[:, :size])+ held = held[:, size:]+ if held is not None and held.shape[1]:+ yield from frame(held)+ yield from stream.encode(None)+++def encode_audio(container, codec, samples, rate):+ """Encode channel-first samples into an open PyAV container; return the packets.++ Packets are returned rather than muxed so callers can interleave them with+ copied video. Integer formats keep their exact values; float formats keep+ their bits.+ """+ stream, fmt = _audio_stream(container, codec, rate, samples.shape[0], samples.dtype.itemsize)+ return stream, list(_encoded(stream, fmt, [samples], rate))+++def _mux_streams(picture, target, rate, copy_from=None, encode=None):+ """Write picture's video plus one audio track, copied or losslessly encoded.++ Both tracks are read and written as they go: a long chain's video and+ soundtrack never sit in memory.+ """+ import av+ from math import lcm+ from .comfy_metadata import _copy_stream+ scale = str(lcm(rate, 24))+ clock = Fraction(1, lcm(rate, 24))+ with av.open(str(picture)) as reader, av.open(str(target), 'w', format='mp4',+ options={'video_track_timescale': scale, 'movie_timescale': scale}) as writer:+ source = reader.streams.video[0]+ video = _copy_stream(writer, source)+ video.time_base = clock++ def pictures():+ for packet in reader.demux(source):+ if packet.dts is None or packet.pts is None:+ continue+ base = packet.time_base+ packet.pts, packet.dts = round(packet.pts * base / clock), round(packet.dts * base / clock)+ packet.duration = round((packet.duration or 0) * base / clock)+ packet.time_base, packet.stream = clock, video+ yield packet++ if copy_from is not None:+ with av.open(str(copy_from)) as audio_reader:+ audio_source = audio_reader.streams.audio[0]+ audio = writer.add_stream_from_template(audio_source)++ def sounds():+ for packet in audio_reader.demux(audio_source):+ if packet.dts is None:+ continue+ packet.stream = audio+ yield packet++ _interleave(writer, pictures(), sounds())+ else:+ codec, chunks, channels, itemsize = encode+ stream, fmt = _audio_stream(writer, codec, rate, channels, itemsize)+ _interleave(writer, pictures(), _encoded(stream, fmt, chunks, rate))+++def _interleave(writer, pictures, sounds):+ # Each track is already in decode order, so merging them equals sorting them all.+ from heapq import merge+ for packet in merge(pictures, sounds, key=lambda item: (item.dts * item.time_base, item.stream.type != 'video')):+ writer.mux(packet)+++def _normalized(data):+ """Numeric sample amplitudes that compare exactly across PCM storage sizes."""+ import numpy as np+ if data.dtype.kind in 'iu':+ return ((data.astype(np.float64) - 128) / 128 if data.dtype == np.uint8+ else data.astype(np.float64) / (1 << (data.dtype.itemsize * 8 - 1)))+ return data+++def _rate(path):+ import av+ with av.open(str(path)) as container:+ return container.streams.audio[0].rate+++def _decoded_chunks(path):+ import av+ with av.open(str(path)) as container:+ for frame in container.decode(audio=0):+ yield _normalized(_samples(frame))+++def _decoded(path):+ """A whole track in memory: for tests and one generation's worth."""+ import numpy as np+ return _rate(path), np.concatenate(list(_decoded_chunks(path)), axis=1)+++def _same_samples(left, right):+ """Whether two chunked tracks hold the same samples, whatever their chunk sizes."""+ import numpy as np+ a = b = None+ while True:+ while a is None or not a.shape[1]:+ a = next(left, None)+ if a is None:+ break+ while b is None or not b.shape[1]:+ b = next(right, None)+ if b is None:+ break+ if a is None or b is None:+ return a is None and b is None+ count = min(a.shape[1], b.shape[1])+ if a.shape[0] != b.shape[0] or not np.array_equal(a[:, :count], b[:, :count]):+ return False+ a, b = a[:, count:], b[:, count:]+++def _trim_video(source, target, count, rate, frames=None):+ """Remux video on a sample-accurate clock, adjusting the last kept frame.++ A chain caller may name its verified frame count to retain a rounded+ sample boundary up to half a source sample past the picture grid.+ """+ import av+ from .comfy_metadata import _copy_stream+ from math import lcm+ duration = Fraction(count, rate)+ with av.open(str(source)) as reader, av.open(str(target), 'w', format='mp4',+ options={'video_track_timescale': str(lcm(rate, 24)), 'movie_timescale': str(lcm(rate, 24))}) as writer:+ stream = reader.streams.video[0]+ output = _copy_stream(writer, stream)+ clock = Fraction(1, lcm(rate, 24))+ output.time_base = clock+ pending = None+ for packet in reader.demux(stream):+ if packet.dts is None or packet.pts is None:+ continue+ base = packet.time_base+ when = packet.pts * base+ if when >= duration:+ continue+ length = min(Fraction(packet.duration or 1, 1) * base, duration - when)+ packet.pts = round(when / clock)+ packet.dts = round(packet.dts * base / clock)+ packet.duration = round(length / clock)+ packet.time_base, packet.stream = clock, output+ if frames is None:+ writer.mux(packet)+ else:+ if pending is not None:+ writer.mux(pending)+ pending = packet+ if pending is not None:+ # MP4 derives intermediate sample durations from consecutive DTS.+ # The last decoded packet may display before the final frame, so+ # close the decode-order duration table, not just the highest PTS.+ pending.duration = round((duration - Fraction(frames - 1, 24)) / clock)+ writer.mux(pending)+++def mux(video, output, info, *, trim_video=True, video_frames=None):+ """Copy compatible full tracks; otherwise preserve native PCM in MP4.++ Decode and compare the saved track before making it visible. Never fall+ back to lossy audio. Unsupported lossless muxers fail with an actionable+ error instead of claiming preservation of a quantized soundtrack.+ The selection is read, written and compared a frame at a time, so a+ two-hour soundtrack costs no more memory than a short one.+ """+ import numpy as np+ import tempfile+ from itertools import chain+ verify(info)+ destination = Path(output)+ rate, count = info['sample_rate'], info['end_sample'] - info['start_sample']++ def saved(path):+ return _rate(path) == rate and _same_samples(_decoded_chunks(path), map(_normalized, _selected_chunks(info)))++ with tempfile.TemporaryDirectory(prefix='original-audio-', dir=destination.parent) as work:+ work = Path(work)+ temporary = work / 'mux.mp4'+ picture = work / 'picture.mp4'+ if trim_video:+ _trim_video(video, picture, count, rate, frames=video_frames)+ else:+ picture = Path(video)+ copied = False+ if info['full_source'] and info['codec'] in ('aac', 'alac', 'mp3', 'flac'):+ try:+ _mux_streams(picture, temporary, rate, copy_from=info['path'])+ copied = saved(temporary)+ except AudioUseError:+ raise+ except (OSError, ValueError, av_errors()):+ copied = False+ if not copied:+ chunks = _selected_chunks(info)+ first = next(chunks)+ if first.dtype.kind == 'f':+ codec, kind = ('pcm_f64le', first.dtype) if first.dtype.itemsize == 8 else ('pcm_f32le', np.dtype('<f4'))+ convert = lambda data: data.astype(kind, copy=False)+ elif first.dtype.itemsize <= 2:+ # FLAC, not ALAC: browsers play FLAC in MP4, so the app's own player has sound.+ codec, kind = 'flac', np.dtype('<i2')+ convert = ((lambda data: ((data.astype(np.int16) - 128) * 256).astype(kind)) if first.dtype == np.uint8+ else (lambda data: data.astype(kind, copy=False)))+ else:+ codec, kind = 'pcm_s32le', np.dtype('<i4')+ convert = lambda data: data.astype(kind, copy=False)+ try:+ _mux_streams(picture, temporary, rate,+ encode=(codec, map(convert, chain([first], chunks)), info['channels'], kind.itemsize))+ except (OSError, ValueError, av_errors()):+ raise AudioUseError('audio_original_format') from None+ if not saved(temporary):+ raise ValueError('Original audio export did not preserve every sample; no output was saved')+ verify(info)+ temporary.replace(destination)+ return dict(use='original', mode='stream_copy' if copied else 'lossless_pcm',+ sample_identical=True, sample_rate=rate, channels=info['channels'],+ samples_per_channel=count, source_sha256=info['sha256'],+ start_sample=info['start_sample'], end_sample=info['end_sample'],+ duration_seconds=count / rate)diff --git a/freevideo_engine/original_audio_chain.py b/freevideo_engine/original_audio_chain.pynew file mode 100644index 0000000..97867ee--- /dev/null+++ b/freevideo_engine/original_audio_chain.py@@ -0,0 +1,155 @@+"""Original soundtrack ranges on one global integer clock, independent of chain IO."""+from copy import deepcopy+import json++from . import chain_plan+from .original_audio import AudioUseError, VERSION, fields, verify+++def sample_at(frame, rate):+ """Nearest sample, ties to even, without floating point or accumulated rounding."""+ whole, remainder = divmod(frame * rate, chain_plan.FPS)+ return whole + (remainder * 2 > chain_plan.FPS or+ remainder * 2 == chain_plan.FPS and whole % 2 == 1)+++def plan(info, **options):+ count = info['end_sample'] - info['start_sample']+ rate = info['sample_rate']+ if count > chain_plan.MAX_SECONDS * rate:+ raise chain_plan.PlanError('too_long', 'A video is at most 2 hours long')+ # Keep the existing split and short-tail rules. Only the last block's+ # visible length changes: it ends on the selected final source sample.+ output_frames = (count * chain_plan.FPS + rate - 1) // rate+ from .geometry import geometry+ aligned = geometry(frames=output_frames)['frames']+ limit = options.get('max_frames')+ if limit is None:+ seconds = options.get('single_limit_seconds')+ seconds = chain_plan.BLOCK_SECONDS if seconds is None else seconds+ chain_plan._number(seconds, 'The single generation limit')+ limit = chain_plan._first_frames(seconds)+ chain_plan._number(limit, 'The frame limit', whole=True)+ limit = min(limit, chain_plan._first_frames(30))+ options = dict(options, max_frames=limit)+ exact_seconds = count / rate+ length = (output_frames - .5) / 24 if count <= 30 * rate and aligned <= limit else (count + rate - 1) // rate+ result = chain_plan.plan(length, **options)+ if result['single']:+ result['blocks'] = [dict(seconds=exact_seconds, frames=aligned, new_frames=aligned)]+ result.update(seconds=exact_seconds, frames=output_frames,+ hide_frames=sum(row['new_frames'] for row in result['blocks']) - output_frames)+ return result+++def ranges(info, blocks):+ """New ranges partition the selection; conditioning also includes the fixed picture prefix."""+ rate, start, end = info['sample_rate'], info['start_sample'], info['end_sample']+ shown, result = 0, []+ for index, block in enumerate(blocks):+ frames = block.get('frames', block.get('inputs', {}).get('frames'))+ origin = shown - chain_plan.OVERLAP if index else 0+ new = frames - chain_plan.OVERLAP if index else frames+ begin = start + sample_at(shown, rate)+ finish = min(end, start + sample_at(shown + new, rate))+ if not start <= begin < finish <= end:+ raise AudioUseError('audio_original_slot')+ result.append(dict(frames=frames, frame_origin=origin,+ start_sample=begin, end_sample=finish,+ source_start_sample=start, source_end_sample=end,+ source_sha256=info['sha256'],+ range=dict(start_sample=start + sample_at(origin, rate), end_sample=finish)))+ shown += new+ if result[-1]['end_sample'] != end:+ raise AudioUseError('audio_original_slot')+ return result+++def selection_info(selected):+ """Shape validation does not require a still-present source when loading receipts."""+ if (not isinstance(selected, dict) or set(selected) != {'file', 'use', 'info'}+ or selected['use'] != 'original' or not isinstance(selected['file'], str)):+ raise AudioUseError('audio_original_route')+ info = selected['info']+ if (not isinstance(info, dict) or any(type(info.get(key)) is not int for key in+ ('sample_rate', 'channels', 'start_sample', 'end_sample', 'bytes'))+ or not 8000 <= info['sample_rate'] <= 192000 or info['channels'] not in (1, 2)+ or not 0 <= info['start_sample'] < info['end_sample']+ or info['end_sample'] - info['start_sample'] > chain_plan.MAX_SECONDS * info['sample_rate']+ or info.get('version') != VERSION or type(info.get('full_source')) is not bool+ or not isinstance(info.get('sha256'), str) or len(info['sha256']) != 64+ or any(c not in '0123456789abcdef' for c in info['sha256'])+ or not isinstance(info.get('path'), str) or info['bytes'] <= 0):+ raise AudioUseError('audio_original_range')+ return info+++def bind(record, selected):+ """Validate a durable forward chain and derive (never trust) its per-block selections."""+ from .audio_use_request import graph_audio+ info = selection_info(selected)+ rows = graph_audio(record['snapshot']['prompt'], record['node'])+ if len(rows) != 1 or rows[0].get('file') != selected['file']:+ raise AudioUseError('audio_original_route')+ wanted = rows[0].get('range')+ if wanted and wanted != {k: info[k] for k in ('start_sample', 'end_sample')}:+ raise AudioUseError('audio_original_range')+ blocks = record['blocks']+ inputs = record['snapshot']['prompt'][record['node']]['inputs']+ if len(blocks) > 1 and (inputs.get('two_pass', True) is not True+ or inputs.get('preview') or inputs.get('refine_from')):+ raise AudioUseError('audio_original_route')+ for index, block in enumerate(blocks):+ if block['before'] is not None or block['after'] != (blocks[index - 1]['id'] if index else None):+ raise AudioUseError('audio_original_route')+ if block['inputs']['frames'] > chain_plan._first_frames(30):+ raise AudioUseError('audio_original_part')+ spans = ranges(info, blocks)+ record['original_audio'] = deepcopy(selected)+ for block, span in zip(blocks, spans):+ block['original_audio'] = span+++def block_prompt(prompt, node, selected, span):+ """Only replace the enabled original asset in the executing prompt copy."""+ from .chain_runner import node_prompt+ verify(selected['info'])+ found = False+ for key in node_prompt(prompt, node):+ item = prompt[key]+ if not isinstance(item, dict) or item.get('class_type') != 'FreeVideoMedia':+ continue+ raw = item['inputs'].get('assets', '[]')+ if not isinstance(raw, str):+ continue+ assets = json.loads(raw)+ changed = False+ for asset in assets:+ if asset.get('enabled', True) and asset.get('use') == 'original':+ if asset['file'] != selected['file']:+ raise AudioUseError('audio_original_route')+ asset['range'] = deepcopy(span['range'])+ asset['segment'] = {k: v for k, v in span.items() if k != 'range'}+ fields(dict(asset, kind='audio'))+ found = changed = True+ if changed:+ item['inputs']['assets'] = json.dumps(assets, ensure_ascii=False)+ if not found:+ raise AudioUseError('audio_original_route')+++def joined_info(record, blocks):+ selected = record.get('original_audio')+ if selected is None:+ return None+ info = deepcopy(selected['info'])+ end = blocks[-1]['original_audio']['end_sample']+ if end != info['end_sample']:+ info.update(end_sample=end, full_source=False)+ return info+++def validate_continuation(info, request):+ span = info.get('segment')+ if not span or span['frame_origin'] != request['frame_origin']:+ raise AudioUseError('audio_original_route')diff --git a/freevideo_engine/reference_sampler.py b/freevideo_engine/reference_sampler.pyindex 30c2fee..0ebb9e3 100644--- a/freevideo_engine/reference_sampler.py+++ b/freevideo_engine/reference_sampler.py@@ -43,6 +43,8 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nurefs = conditionsif not refs:raise ValueError('Ref2VA requires references')+ from .fixed_track import split, timesteps as fixed_timesteps+ refs, fixed_audio = split(refs, num_frames, device)condition_latents = [row['latent'].to(device) for row in refs if row['kind'] in ('image', 'video')]audio_conditions = [row['audio_latent'].to(device) for row in refs if row.get('audio_latent') is not None]references = [SimpleNamespace(kind=row['kind'], has_audio=row.get('audio_latent') is not None,@@ -103,8 +105,9 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nuif video_rows.shape[0] != video_indices.numel():raise ValueError(f"{video_rows.shape[0]} video rows (with conditioning) != "f"{video_indices.numel()} layout rows")- audio_rows = torch.randn((num_audio_latents * AUDIO_CHANNELS, 32),- generator=generator, device=device, dtype=torch.float32)+ audio_rows = (fixed_audio.clone() if fixed_audio is not None else+ torch.randn((num_audio_latents * AUDIO_CHANNELS, 32),+ generator=generator, device=device, dtype=torch.float32))if audio_conditions:audio_rows = torch.cat(audio_conditions + [audio_rows])@@ -164,15 +167,21 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nurow_timesteps = torch.full((seq_len,), float(t), dtype=torch.float32, device=device)if num_condition_rows:row_timesteps[video_indices[:num_condition_rows]] = max(float(t), KEYFRAME_NOISE_AUG)- row_timesteps[audio_indices[num_audio_condition_rows:]] = float(audio_t)+ row_timesteps[audio_indices[num_audio_condition_rows:]] = 1.0 if fixed_audio is not None else float(audio_t)row_timesteps[audio_indices[:num_audio_condition_rows]] = 1.0timestep, timestep_indices = torch.unique(row_timesteps, sorted=True, return_inverse=True)+ if fixed_audio is not None:+ audio_rows[num_audio_condition_rows:] = fixed_audio+ timestep, timestep_indices = fixed_timesteps(row_timesteps, t, audio_t)if timestep_task is not None:from .adaln import modality_timesteps# The cached table covers the initial superset of modalities. Keep# unused clock entries so removing the last visual/audio reference# does not change the table or the remaining rows' clock indices.timestep = modality_timesteps(t[None], audio_t[None], timestep_task)[0]+ if fixed_audio is not None:+ # Original audio rows stay clean (t = 1) at every step, so the superset stays the same per step.+ timestep = torch.unique(torch.cat((timestep, timestep.new_ones(1))), sorted=True)timestep_indices = torch.searchsorted(timestep, row_timesteps)noise_pred, audio_noise_pred = transformer(hidden_states=video_rows[None],@@ -191,9 +200,12 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nuvideo_rows[num_condition_rows:] = scheduler.step(noise_pred[0, num_condition_rows:].float(), t, video_rows[num_condition_rows:],return_dict=False)[0]- audio_rows[num_audio_condition_rows:] = audio_scheduler.step(- audio_noise_pred[0, num_audio_condition_rows:].float(), audio_t,- audio_rows[num_audio_condition_rows:], return_dict=False)[0]+ if fixed_audio is None:+ audio_rows[num_audio_condition_rows:] = audio_scheduler.step(+ audio_noise_pred[0, num_audio_condition_rows:].float(), audio_t,+ audio_rows[num_audio_condition_rows:], return_dict=False)[0]+ else:+ audio_rows[num_audio_condition_rows:] = fixed_audioif step_seconds is not None:synchronize()diff --git a/freevideo_engine/refine.py b/freevideo_engine/refine.pyindex f535d3f..094cd50 100644--- a/freevideo_engine/refine.py+++ b/freevideo_engine/refine.py@@ -56,9 +56,12 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nuraise ValueError(f'Initial {name} latent must be finite floating point')audio_conditions = []+ fixed_audio = Noneif conditions and isinstance(conditions[0], dict):from types import SimpleNamespacerefs = conditions+ from .fixed_track import split, timesteps as fixed_timesteps+ refs, fixed_audio = split(refs, frames, device)condition_latents = [row['latent'].to(device) for row in refs if row['kind'] in ('image', 'video')]audio_conditions = [row['audio_latent'].to(device) for row in refs if row.get('audio_latent') is not None]references = [SimpleNamespace(kind=row['kind'], has_audio=row.get('audio_latent') is not None,@@ -114,23 +117,31 @@ def generate_latents(transformer, prompt_embeds, text_token_tags, num_frames, nuraise ValueError('Refinement video conditioning disagrees with the packed layout')del fixed_videoaudio_clean = source_audio.permute(0, 2, 1).reshape(-1, 32).contiguous()- audio_noise = torch.randn(audio_clean.shape, generator=generator, device=device, dtype=torch.float32)+ if fixed_audio is not None and not torch.equal(audio_clean, fixed_audio):+ raise ValueError('First-pass original audio differs from its clean conditioning')+ audio_noise = (None if fixed_audio is not None else+ torch.randn(audio_clean.shape, generator=generator, device=device, dtype=torch.float32))del source_video, video_noisefor video_t, audio_t in zip(video_scheduler.timesteps[start:], audio_scheduler.timesteps[start:]):tick = time.perf_counter()- audio_rows = audio_scheduler.scale_noise(audio_clean, audio_t, audio_noise)+ audio_rows = (fixed_audio.clone() if fixed_audio is not None else+ audio_scheduler.scale_noise(audio_clean, audio_t, audio_noise))if audio_conditions:audio_rows = torch.cat(audio_conditions + [audio_rows])if audio_rows.shape[0] != audio_ids.numel():raise ValueError('Refinement audio conditioning disagrees with the packed layout')row_times = torch.full((positions.shape[0],), float(video_t), dtype=torch.float32, device=device)row_times[video_ids[:condition_rows]] = max(float(video_t), KEYFRAME_NOISE_AUG)- row_times[audio_ids[audio_condition_rows:]] = float(audio_t)+ row_times[audio_ids[audio_condition_rows:]] = 1.0 if fixed_audio is not None else float(audio_t)row_times[audio_ids[:audio_condition_rows]] = 1.times, time_ids = torch.unique(row_times, sorted=True, return_inverse=True)+ if fixed_audio is not None:+ times, time_ids = fixed_timesteps(row_times, video_t, audio_t)if timestep_task is not None:from .adaln import modality_timestepstimes = modality_timesteps(video_t[None], audio_t[None], timestep_task)[0]+ if fixed_audio is not None:+ times = torch.unique(torch.cat((times, times.new_ones(1))), sorted=True)time_ids = torch.searchsorted(times, row_times)prediction, _ = transformer(hidden_states=video_rows[None], audio_hidden_states=audio_rows[None],encoder_hidden_states=prompt_embeds[None], timestep=times, timestep_indices=time_ids,diff --git a/freevideo_engine/result_cache.py b/freevideo_engine/result_cache.pyindex fe81d25..d02d068 100644--- a/freevideo_engine/result_cache.py+++ b/freevideo_engine/result_cache.py@@ -242,6 +242,12 @@ def request_key(prompt, seed, canvas, sampling_plan, extra, machine, resources,for name in ('references', 'loras'):media[name] = [{**{k: v for k, v in row.items() if k != 'path'},'sha256': file(row['path'], content=True)} for row in media.get(name, [])]+ for row in media.get('references', []):+ if row.get('use') == 'reference':+ row.pop('use')+ if row.get('original_audio'):+ from .original_audio import identity+ row['original_audio'] = identity(row['original_audio'])conditioning = file(extra['conditioning'], content=True) if extra.get('conditioning') else Noneif inspection:inspection.phases['inputs'] = time.monotonic() - input_started@@ -253,7 +259,9 @@ def request_key(prompt, seed, canvas, sampling_plan, extra, machine, resources,compute_env = {k: v for k, v in environment.items() if k not in ignored and k.startswith(prefixes)}return _key(dict(schema=2, prompt=prompt, seed=seed, canvas=canvas, sampling_plan=sampling_plan,media=media, conditioning=conditioning, machine=machine, settings=settings,- models=models, code=code, packages=packages, environment=compute_env))+ models=models, code=code, packages=packages, environment=compute_env,+ **({'original_audio': {k: v for k, v in extra['original_audio'].items() if k != 'path'}}+ if extra.get('original_audio') else {})))except TimeoutError:raiseexcept (OSError, ValueError, TypeError, KeyError, AttributeError, sqlite3.Error):diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.pyindex 3c21b35..badd9f2 100644--- a/freevideo_engine/runtime.py+++ b/freevideo_engine/runtime.py@@ -556,7 +556,9 @@ class Engine:if continuation is not None:from .continuation import validate_route# Audio references are admitted below, after inspecting the encoded modalities.- if self.task not in ('ref2va_audio', 'ref2va_av'):+ if continuation.get('video_only'):+ validate_route(task=self.task, anchor=bool(continuation.get('anchor')), original_audio=True)+ elif self.task not in ('ref2va_audio', 'ref2va_av'):validate_route(task=self.task, anchor=bool(continuation.get('anchor')))if tuple(continuation['video'].shape[-2:]) != (height // 16, width // 16):from .continuation import SIZE_MISMATCH@@ -623,10 +625,19 @@ class Engine:sampling_kwargs.update(condition_schedule=condition_schedule, condition_trace=early_receipt['steps'])conditions = value.get('references') or Noneif continuation is not None:- from .continuation import encoded_references- if encoded_references(conditions, self.task, bool(continuation.get('anchor'))):- validate_route(task=self.task, sound=True, anchor=bool(continuation.get('anchor')),- media=dict(references=[dict(kind=row['kind']) for row in conditions]))+ from .continuation import UNSUPPORTED_MEDIA, encoded_references+ from .fixed_track import split+ # A clip used as is fills the part's own sound; only the other rows are references.+ references, fixed = split(conditions or [], frames, self.device_backend.device)+ if (fixed is None) == bool(continuation.get('video_only')):+ raise ValueError(UNSUPPORTED_MEDIA)+ del fixed+ if not continuation.get('video_only') or references:+ if encoded_references(references, self.task, bool(continuation.get('anchor'))):+ if continuation.get('video_only'):+ raise ValueError(UNSUPPORTED_MEDIA)+ validate_route(task=self.task, sound=True, anchor=bool(continuation.get('anchor')),+ media=dict(references=[dict(kind=row['kind']) for row in references]))# Ref2VA packs independent-size references before generated# video rows in both passes. Their latents do not need the# keyframe resize used by first/last-frame conditioning.diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.pyindex ba03bf6..bb53ce9 100644--- a/freevideo_engine/worker.py+++ b/freevideo_engine/worker.py@@ -100,6 +100,18 @@ def generate(request, resident=None):if request['sampling_plan'] != expected:raise ValueError('Sampling plan does not match the requested canvas and task')metrics['sampling_plan'] = expected+ original_audio = request.get('original_audio')+ if original_audio is not None:+ from .original_audio import verify, AudioUseError+ verify(original_audio)+ if request.get('continuation') is not None and 'segment' not in original_audio:+ raise AudioUseError('audio_original_route')+ if request['engine_options'].get('task') not in ('ref2va_audio', 'ref2va_av'):+ raise ValueError('Original audio requires a clean audio reference layout')+ if hide_tail_frames:+ # The clip's samples set the part's length; no frames are hidden after it.+ raise AudioUseError('audio_original_route')+ metrics['original_audio'] = original_audiocontinuation = request.get('continuation')if continuation is not None:from .continuation import (validate_route, summary, ADALN_TASK, ANCHOR_ADALN_TASK, UNSUPPORTED_DEVICE,@@ -112,14 +124,20 @@ def generate(request, resident=None):**(dict(sound=True, media=dict(references=[continuation['sound']]))if continuation.get('sound') else {}),two_pass=bool(request.get('sampling_plan', {}).get('requested')),- preview=request.get('preview', False), refine_from=request.get('resume_first_pass'))+ preview=request.get('preview', False), refine_from=request.get('resume_first_pass'),+ original_audio=original_audio is not None and 'segment' in original_audio)if isinstance(getattr(torch.version, 'hip', None), str):raise ValueError(UNSUPPORTED_DEVICE)continuation = dict(continuation,audio_context=validate_audio_context(continuation.get('audio_context', AUDIO_CONTEXT_DEFAULT)))+ if original_audio is not None:+ from .original_audio_chain import validate_continuation+ validate_continuation(original_audio, continuation)+ continuation.update(video_only=True, audio_context='clean')request['continuation'] = continuationrequest['engine_options'] = dict(request['engine_options'],- adaln_task=ANCHOR_ADALN_TASK if continuation.get('anchor') else ADALN_TASK)+ adaln_task=(request['engine_options']['task'] if original_audio is not None+ else ANCHOR_ADALN_TASK if continuation.get('anchor') else ADALN_TASK))metrics['continuation'] = summary(continuation, hide_tail_frames=hide_tail_frames)metrics['audio_context'] = continuation['audio_context']metrics['device_memory'] = device_memory_report(request['gpu_budget_bytes'], request.get('allocator_limit_bytes'),@@ -208,6 +226,12 @@ def generate(request, resident=None):# Older official keyframe caches have no explicit task field.value.setdefault('task', options['task'])actual = describe(value, canvas['width'], canvas['height'], canvas['frames'])+ from .original_audio import identity+ originals = [row for row in value.get('references', []) if row.get('use') == 'original']+ if originals or original_audio is not None:+ if (len(originals) != 1 or original_audio is None+ or identity(originals[0].get('original_audio', {})) != identity(original_audio)):+ raise ValueError('Encoded original audio does not match the retained source')if actual['task'] != options['task'] or any(canvas.get(name, actual[name]) != actual[name]for name in ('reference_video_tokens', 'reference_audio_tokens')):raise ValueError('Encoded media disagrees with the admitted request layout')@@ -219,9 +243,11 @@ def generate(request, resident=None):# too: its first and final canvases are the same one, so one tail# serves both stages. An explicit single-pass request keeps nothing.# Whatever media conditioned the request, the sampler returns only the- # generated picture and sound rows, so the tail is the same.+ # generated picture and sound rows, so the tail is the same. A clip used+ # as is continues only as a part of its own chain.plan_flags = request.get('sampling_plan') or {}continuable = (bool(plan_flags.get('enabled') or plan_flags.get('requested'))+ and (original_audio is None or 'segment' in original_audio)and not request.get('preview'))first_tail = continuation_data = Nonedef tail_failure(error):@@ -656,9 +682,10 @@ def generate(request, resident=None):continuation_data['metadata']['source_id'] = continuation['source_id']metrics.update(decode_to_file(latents, audio, request['output'], base=base, artifacts_dir=artifacts,**({'hide_tail_frames': hide_tail_frames} if hide_tail_frames else {}),+ **({'original_audio': original_audio} if original_audio is not None else {}),**({'continuation': continuation} if continuation is not None else {}),**({'audio_lead_in_callback': keep_audio_lead_in}- if continuation is not None and continuation_data is not None else {}),+ if continuation is not None and continuation_data is not None and original_audio is None else {}),model_cache=resident, phase_callback=decode_phase, output_size=output_size,ram_budget_bytes=request.get('ram_budget_bytes'), **decoder))if continuable and continuation_data is not None:diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjsindex 4db68af..b1b8c19 100644--- a/scripts/test_freevideo_entry.mjs+++ b/scripts/test_freevideo_entry.mjs@@ -98,6 +98,8 @@ test('main browser entry registers both views and preserves node hooks', async tattachReferencePicker() {}, referenceItems: () => [], syncReferencePrompt() {},shareButton: () => new Element('button'), reportFileName: () => 'FreeVideo-report-20261008-101530.json',rememberPromptDraft() {}, savePromptDraft: async () => false, openImageEditor: async () => null, icon: () => '',+ AUDIO_USE: {reference: 'reference', original: 'original'}, usesOriginal: row => row?.use === 'original', checkOriginal: async () => ({ok: true}),+ checkedLength: () => null, failureText: () => '', multipleText: () => '', createUseSwitch: () => globalThis.document.createElement('div'),};const previous = new Map();for (const [name, value] of Object.entries({@@ -146,7 +148,8 @@ test('main browser entry registers both views and preserves node hooks', async t.replace("import { swapMissingText } from './replace_person.js';", 'const swapMissingText = () => "";').replace("import { agentInterfaceState } from './agent_connect.js';", "const agentInterfaceState = async () => 'off';").replace("import { installNativeErrorGuard } from './native_errors.js';", 'const installNativeErrorGuard = () => {};')- .replace("import { installTranslateGuard } from './translate_guard.js';", 'const installTranslateGuard = () => {};');+ .replace("import { installTranslateGuard } from './translate_guard.js';", 'const installTranslateGuard = () => {};')+ .replace("import { AUDIO_USE, usesOriginal, checkOriginal, checkedLength, failureText, multipleText, createUseSwitch } from './audio_original.js';", 'const {AUDIO_USE,usesOriginal,checkOriginal,checkedLength,failureText,multipleText,createUseSwitch} = globalThis.__freevideoEntryTest;');await import('data:text/javascript;base64,' + Buffer.from(source).toString('base64'));const extension = extensions.find(row => row.name === 'FreeVideo.UnifiedMedia');assert.ok(extension, 'the main extension must register');diff --git a/web/audio_original.css b/web/audio_original.cssnew file mode 100644index 0000000..be4b686--- /dev/null+++ b/web/audio_original.css@@ -0,0 +1,10 @@+.fv-audio-use{display:inline-flex;gap:0;margin-left:auto}+.fv-audio-use button{min-height:var(--fv-h-sm,32px);border-radius:0;padding:5px 10px;white-space:nowrap}+.fv-audio-use button+button{margin-left:-1px}+.fv-audio-use button:first-child{border-radius:var(--fv-r-sm,8px) 0 0 var(--fv-r-sm,8px)}+.fv-audio-use button:last-child{border-radius:0 var(--fv-r-sm,8px) var(--fv-r-sm,8px) 0}+.fv-audio-use button[aria-pressed=true]{background:var(--fv-selected);color:var(--fv-accent);border-color:var(--fv-accent);position:relative}+.fv-audio-use button:disabled{cursor:default}+.fv-audio-lock{display:inline-flex;margin-left:6px;vertical-align:-3px;color:var(--fv-muted)}+.fv-audio-lock svg{width:13px;height:13px}+.fv-audio-lock[hidden]{display:none}diff --git a/web/audio_original.js b/web/audio_original.jsnew file mode 100644index 0000000..bb5f613--- /dev/null+++ b/web/audio_original.js@@ -0,0 +1,94 @@+// "Reference | Use as is" for an audio card, and the length the chosen audio fixes.+// The server's preview check is the only source of that length; the client keeps+// the reply in memory so the media panel and the Studio ask once per clip.++const css = document.createElement('link');+css.rel = 'stylesheet'; css.href = new URL('./audio_original.css', import.meta.url).href; document.head.append(css);++export const AUDIO_USE = Object.freeze({reference: 'reference', original: 'original'});+export const usesOriginal = row => row?.use === AUDIO_USE.original;+export const activeOriginal = rows => rows.find(row => row.enabled !== false && usesOriginal(row)) || null;++const rangeKey = range => range ? `${range.start_sample}-${range.end_sample}` : 'all';+const keyOf = row => `${row.file}|${rangeKey(row.range)}`;+const settled = new Map(), pending = new Map();++// The reply for this clip and range, if already checked.+export const checkedLength = row => settled.get(keyOf(row)) || null;+export const forgetChecks = () => { settled.clear(); pending.clear(); };++// {ok: true, ...reply} or {ok: false, code, message, message_zh}. A network failure+// has no server sentence, so it comes back with code 'unreachable'.+export function checkOriginal(api, row) {+ const key = keyOf(row);+ if (settled.has(key)) return Promise.resolve({ok: true, ...settled.get(key)});+ if (pending.has(key)) return pending.get(key);+ const body = {file: row.file, use: AUDIO_USE.original};+ if (row.range) body.range = row.range;+ const request = (async () => {+ try {+ const response = await api.fetchApi('/freevideo/media/audio-use', {method: 'POST',+ headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body)});+ const reply = await response.json();+ if (!response.ok || reply.use !== AUDIO_USE.original || !(reply.seconds > 0)) {+ return {ok: false, code: reply.code || 'unreachable', message: reply.message, message_zh: reply.message_zh};+ }+ settled.set(key, reply);+ return {ok: true, ...reply};+ } catch { return {ok: false, code: 'unreachable'}; }+ finally { pending.delete(key); }+ })();+ pending.set(key, request);+ return request;+}++export function failureText(result, t) {+ if (result.message && result.message_zh) return t(result.message, result.message_zh);+ return t('This audio could not be used as is.', '无法原样使用这段音频。');+}++// Where the engine refuses Use as is (continuing a video, a first-pass preview), the Studio+// names that state here and the option is greyed out on every audio card until it ends.+const blockers = new Set();+export const originalBlocked = () => blockers.size > 0;+export function blockOriginal(reason, on) {+ const before = blockers.size > 0;+ if (on) blockers.add(reason); else blockers.delete(reason);+ if (before !== blockers.size > 0) window.dispatchEvent(new CustomEvent('freevideo-audio-blocked'));+}++export const multipleText = t => t('Choose "Use as is" for only one audio file.', '一次只能有一段音频选择“原样使用”。');++// The two-way switch on an audio card. `onPick` receives 'reference' or 'original'.+export function createUseSwitch({t, use, onPick}) {+ const group = document.createElement('div');+ group.className = 'fv-audio-use'; group.setAttribute('role', 'group');+ group.setAttribute('aria-label', t('Audio use', '音频用途'));+ const buttons = {};+ for (const [value, label] of [[AUDIO_USE.reference, t('Reference', '参考')], [AUDIO_USE.original, t('Use as is', '原样使用')]]) {+ const b = document.createElement('button'); b.type = 'button'; b.textContent = label; b.dataset.use = value;+ b.setAttribute('aria-pressed', String(value === use));+ b.onclick = () => { if (b.getAttribute('aria-pressed') !== 'true') onPick(value); };+ buttons[value] = b; group.append(b);+ }+ let chosen = use, working = false;+ const apply = () => {+ for (const [name, b] of Object.entries(buttons)) {+ b.setAttribute('aria-pressed', String(name === chosen));+ b.disabled = working || (name === AUDIO_USE.original && originalBlocked());+ }+ };+ group.set = (value, busy = false) => { chosen = value; working = busy; apply(); };+ const refresh = () => { if (group.isConnected) apply(); else window.removeEventListener('freevideo-audio-blocked', refresh); };+ window.addEventListener('freevideo-audio-blocked', refresh);+ apply();+ return group;+}++// The lock beside the duration label while the audio decides the length.+export function lockMark() {+ const mark = document.createElement('span'); mark.className = 'fv-audio-lock'; mark.hidden = true; mark.setAttribute('aria-hidden', 'true');+ mark.innerHTML = '<svg viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round">'+ + '<rect x="3.5" y="7" width="9" height="6.5" rx="1.5"/><path d="M5.5 7V5a2.5 2.5 0 0 1 5 0v2"/></svg>';+ return mark;+}diff --git a/web/freevideo.js b/web/freevideo.jsindex 0c5d931..ba255d7 100644--- a/web/freevideo.js+++ b/web/freevideo.js@@ -32,6 +32,7 @@ import { uploadMedia } from './media_upload.js';import { swapState, requestRows, swapMissing, withPrefix } from './replace_state.js';import { swapMissingText } from './replace_person.js';import { issueButton } from './issue_button.js';+import { AUDIO_USE, usesOriginal, checkOriginal, checkedLength, failureText, multipleText, createUseSwitch } from './audio_original.js';const languageOverride = typeof location !== 'undefined'? new URLSearchParams(location.search).get('freevideo_lang') : null;@@ -258,6 +259,22 @@ function mediaPanel(node, mount = null) {rows[at] = {...rest, file: source}; update(rows);message(text('Restored the original image. Edit it again to reapply the changes.', '已恢复原图;再次编辑可以重新应用修改。'));}+ // Reference | Use as is: the choice is checked against the file before it is kept.+ function useSwitch(row) {+ const group = createUseSwitch({t: text, use: usesOriginal(row) ? AUDIO_USE.original : AUDIO_USE.reference, onPick: async value => {+ const current = read(), at = current.findIndex(r => r.file === row.file);+ if (at < 0) return;+ if (value === AUDIO_USE.reference) { const {use: _use, range: _range, ...rest} = current[at]; current[at] = rest; update(current); return; }+ if (current.some((other, i) => i !== at && other.enabled !== false && usesOriginal(other))) { message(multipleText(text), true); return; }+ group.set(AUDIO_USE.original, true);+ const result = await checkOriginal(api, {...current[at], use: AUDIO_USE.original});+ const latest = read(), now = latest.findIndex(r => r.file === row.file);+ if (now < 0) return;+ if (!result.ok) { group.set(AUDIO_USE.reference); message(failureText(result, text), true); return; }+ latest[now] = {...latest[now], use: AUDIO_USE.original}; update(latest);+ }});+ return group;+ }function render() {for (const preview of cards.querySelectorAll("video,audio")) {preview.pause(); preview.removeAttribute("src"); preview.load();@@ -296,7 +313,12 @@ function mediaPanel(node, mount = null) {const card = el("div", undefined, "fv-card"); card.dataset.enabled = String(row.enabled !== false);const top = el("div", undefined, "fv-row");const enabled = el("input"); enabled.type = "checkbox"; enabled.checked = row.enabled !== false;- enabled.setAttribute("aria-label", text("Use media", "使用素材")); enabled.onchange = () => { rows[index].enabled = enabled.checked; update(rows); };+ enabled.setAttribute("aria-label", text("Use media", "使用素材")); enabled.onchange = () => {+ if (enabled.checked && usesOriginal(rows[index]) && rows.some((other, i) => i !== index && other.enabled !== false && usesOriginal(other))) {+ enabled.checked = false; message(multipleText(text), true); return;+ }+ rows[index].enabled = enabled.checked; update(rows);+ };const label = el("span", row.cast?.name || (row.source || row.file).split("/").pop(), "fv-file"); label.title = row.file;const select = el("select"); select.setAttribute("aria-label", text("Media role", "素材用途"));const kind = mediaKind(row.file);@@ -306,7 +328,8 @@ function mediaPanel(node, mount = null) {select.value = row.role; select.onchange = () => { rows[index].role = select.value; update(rows); };top.append(enabled, label);if (row.source) top.append(el('span', text('Edited', '已编辑'), 'fv-edited'));- top.append(select); card.append(top);+ if (kind === "audio") top.append(useSwitch(row)); else top.append(select);+ card.append(top);const preview = el(kind === "image" ? "img" : kind); preview.className = "fv-preview";preview.src = viewURL(row.file);if (kind === "image") {@@ -373,6 +396,15 @@ function mediaPanel(node, mount = null) {button(text("Remove", "移除"), text("Remove from this request; keep the uploaded file", "从本次请求移除,保留已上传文件"), () => { rows.splice(index, 1); update(rows); }));card.append(controls); cards.append(card);});+ // A saved choice is checked again; a clip the server now refuses goes back to Reference.+ const saved = rows.find(row => row.enabled !== false && usesOriginal(row));+ if (saved && !checkedLength(saved)) checkOriginal(api, saved).then(result => {+ if (result.ok || result.code === 'unreachable') return;+ const latest = read(), at = latest.findIndex(row => row.file === saved.file && usesOriginal(row));+ if (at < 0) return;+ const {use: _use, range: _range, ...rest} = latest[at]; latest[at] = rest; update(latest);+ message(failureText(result, text), true);+ });}let uploading = false;serialized.serializeValue = () => {@@ -384,7 +416,7 @@ function mediaPanel(node, mount = null) {return JSON.stringify(requestRows(swapState(node)));}// The run takes only what it uses (and which character a picture belongs to); the editing data of a picture stays in the workflow.- try { return JSON.stringify(read().map(({file, role, enabled, cast, sound, depth}) => ({file, role, enabled, ...(sound === false && mediaKindOrNull(file) === 'video' ? {sound: false} : {}), ...(depth === true && role === 'reference' && mediaKindOrNull(file) === 'video' ? {depth: true} : {}), ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id}} : {})}))); } catch { return serialized.value; }+ try { return JSON.stringify(read().map(({file, role, enabled, cast, sound, depth, use, range}) => ({file, role, enabled, ...(use === AUDIO_USE.original ? {use, ...(range && typeof range === 'object' ? {range} : {})} : {}), ...(sound === false && mediaKindOrNull(file) === 'video' ? {sound: false} : {}), ...(depth === true && role === 'reference' && mediaKindOrNull(file) === 'video' ? {depth: true} : {}), ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id}} : {})}))); } catch { return serialized.value; }};async function upload(files) {if (node.isUploading || !files.length) return;diff --git a/web/health.js b/web/health.jsindex 577ed0b..fc54131 100644--- a/web/health.js+++ b/web/health.js@@ -13,7 +13,7 @@ const t = (en, zh) => cn ? zh : en;// launcher.js removes this flag during setup; read it before that.const launchVisit = query.get('freevideo') === 'launch';// Every script this folder ships; a test keeps the list in step with web/*.js.-export const MODULES = ['chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js',+export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js','editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js','editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js','progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js',diff --git a/web/studio.js b/web/studio.jsindex ed4f837..9d64d53 100644--- a/web/studio.js+++ b/web/studio.js@@ -47,6 +47,7 @@ import { firstFrameRow } from './first_frame_row.js';import { createReplacePerson, swapMissingText } from './replace_person.js';import { swapState, swapMissing, effectiveRows, prepareSwapLora, releaseSwapLora, SWAP_LORA } from './replace_state.js';import { createStudioChain, chainPosition, chainParts, chainSplit, composePartText, readChainResult, sumPartEstimates, singleRunSnapshot } from './studio_chain.js?v=20261019-words';+import { activeOriginal, checkOriginal, lockMark, blockOriginal } from './audio_original.js';import { waitingText } from './chain_support.js';import { createExtendState, extendText, extendSourceProblem, extendSourceState, readExtendRecord, extendSnapshot, mountExtendThumbnail, videoUnavailableText } from './studio_extend.js';import { topDialog, isolateDialogKeys, listenDialogKey, routeDialogKey } from './dialog_keys.js';@@ -869,7 +870,8 @@ export function openStudio(node, {extendFrom = null} = {}) {const legacyOption = [...mp.options].find(o => Number(o.value) === 1);mp.disabled = dimensionsLinked;const seconds = el('input'); seconds.type = 'number'; seconds.required = true; seconds.min = '1.625'; seconds.step = 'any'; seconds.value = savedSeconds(node, value(node, 'seconds')); seconds.disabled = linked(node, 'seconds'); seconds.setAttribute('aria-label', t('Duration in seconds', '时长(秒)')); seconds.dataset.fvGuide = 'seconds';- const fields = el('div', null, 'fv-fields'); fields.append(field(t('Resolution', '清晰度'), resolutionSelect), field(t('Duration · seconds', '时长 · 秒'), seconds)); canvas.append(fields);+ const audioLock = lockMark(), durationField = field(t('Duration · seconds', '时长 · 秒'), seconds); durationField.firstChild.append(audioLock);+ const fields = el('div', null, 'fv-fields'); fields.append(field(t('Resolution', '清晰度'), resolutionSelect), durationField); canvas.append(fields);const canvasNote = el('div', null, 'fv-canvas-note'), dimensions = el('span'), duration = el('span'); canvasNote.append(dimensions, duration); canvas.append(canvasNote);const rangeMount = el('div'); canvas.append(rangeMount);let capabilityView;@@ -1064,13 +1066,14 @@ export function openStudio(node, {extendFrom = null} = {}) {firstFrame.row.hidden = true; fields.after(firstFrame.row);firstFrame.input.checked = value(node, 'first_frame_reference') === true;firstFrame.input.onchange = () => { set(node, 'first_frame_reference', firstFrame.input.checked); firstFrame.input.checked = value(node, 'first_frame_reference') === true; };- let manyParts = false;+ let manyParts = false, originalOn = false;function updateFirstFrame() {// While a new plan is being worked out the row keeps its place.const plan = lengthControl.plan; if (plan || !lengthControl.needsPlan) manyParts = plan?.count > 1;firstFrame.row.hidden = !widget(node, 'first_frame_reference') || !(manyParts || extension.source);- firstFrame.input.checked = value(node, 'first_frame_reference') === true;- firstFrame.setDisabled(Boolean(capturing || queueState?.submitting || queueState?.running || chainController?.active));+ // Audio used as is fixes the picture to the sound, so this switch is off while it applies.+ firstFrame.input.checked = !originalOn && value(node, 'first_frame_reference') === true;+ firstFrame.setDisabled(Boolean(originalOn || capturing || queueState?.submitting || queueState?.running || chainController?.active));}const samplingFields = el('div', null, 'fv-fields'); samplingFields.style.marginTop = '14px';const baseSteps = el('input'), refineSteps = el('input');@@ -1123,6 +1126,7 @@ export function openStudio(node, {extendFrom = null} = {}) {if (runButtonReady) updateRunButton();}previewFirst.disabled = !twoPass.checked || linked(node, 'preview') || !widget(node, 'preview');+ blockOriginal('preview', previewFirst.checked && !previewFirst.disabled);syncPassMode();baseSteps.disabled = baseLinked || !widget(node, 'base_steps') || Boolean(extension.source);refineSteps.disabled = !twoPass.checked || linked(node, 'refine_steps') || !widget(node, 'refine_steps') || Boolean(extension.source);@@ -1982,7 +1986,33 @@ export function openStudio(node, {extendFrom = null} = {}) {const clearAddedNote = () => { if (status.textContent === addedNote) status.textContent = ''; };seconds.addEventListener('input', clearAddedNote); prompt.addEventListener('input', clearAddedNote);seconds.onchange = () => { if (lengthControl.commit({report: false})) updateCanvas(); };- const mediaChanged = async e => { if (String(e.detail) !== String(mediaNode?.id)) return; refreshEffortEstimate(true); if (selected !== 'input') return; try { await inputRatio(); } catch (error) { status.textContent = error.message; } };+ // Audio used as is fixes the duration; long selections use the background chain.+ const SECONDS_BEFORE_AUDIO = 'freevideo_seconds_before_audio';+ let audioRevision = 0;+ async function syncOriginalAudio() {+ const revision = ++audioRevision;+ const row = activeOriginal(mediaNode ? JSON.parse(value(mediaNode, 'assets') || '[]') : []);+ const result = row ? await checkOriginal(api, row) : null;+ if (disposed || revision !== audioRevision || result?.code === 'unreachable') return;+ originalOn = Boolean(result?.ok); updateFirstFrame();+ if (result?.ok) {+ lengthControl.forceSingle(result.seconds <= 30);+ if (linked(node, 'seconds')) return;+ node.properties ||= {};+ if (!Object.hasOwn(node.properties, SECONDS_BEFORE_AUDIO)) { node.properties[SECONDS_BEFORE_AUDIO] = Number(seconds.value); node.graph?.change(); }+ const length = Math.ceil(result.seconds * 1000) / 1000;+ seconds.disabled = false; audioLock.hidden = false;+ if (Number(seconds.value) !== length) { seconds.value = length; if (lengthControl.commit({exact: true})) updateCanvas(); }+ seconds.disabled = true;+ return;+ }+ lengthControl.forceSingle(false); audioLock.hidden = true; seconds.disabled = linked(node, 'seconds');+ if (node.properties && Object.hasOwn(node.properties, SECONDS_BEFORE_AUDIO)) {+ seconds.value = node.properties[SECONDS_BEFORE_AUDIO]; delete node.properties[SECONDS_BEFORE_AUDIO]; node.graph?.change();+ if (lengthControl.commit()) updateCanvas();+ }+ }+ const mediaChanged = async e => { if (String(e.detail) !== String(mediaNode?.id)) return; syncOriginalAudio(); refreshEffortEstimate(true); if (selected !== 'input') return; try { await inputRatio(); } catch (error) { status.textContent = error.message; } };const lorasChanged = e => { if (String(e.detail) === String(loraNode?.id)) refreshEffortEstimate(true); };window.addEventListener('freevideo-loras', lorasChanged);cleanup.push(() => window.removeEventListener('freevideo-loras', lorasChanged));@@ -1990,7 +2020,7 @@ export function openStudio(node, {extendFrom = null} = {}) {const prewarmChanged = e => { if (String(e.detail.node) === String(node.id)) prewarm.textContent = e.detail.label; };window.addEventListener('freevideo-prewarm', prewarmChanged);cleanup.push(() => window.removeEventListener('freevideo-prewarm', prewarmChanged));- window.addEventListener('freevideo-media', mediaChanged); window.addEventListener('freevideo-result', resultChanged);+ window.addEventListener('freevideo-media', mediaChanged); window.addEventListener('freevideo-result', resultChanged); syncOriginalAudio();cleanup.push(() => { window.removeEventListener('freevideo-media', mediaChanged); window.removeEventListener('freevideo-result', resultChanged); });const listen = (type, handler) => { api.addEventListener(type, handler); cleanup.push(() => api.removeEventListener(type, handler)); };listen('status', () => { syncQueue().catch(() => {}); });@@ -2045,7 +2075,7 @@ export function openStudio(node, {extendFrom = null} = {}) {cleanup.push(isolateDialogKeys(dialog, event => routeDialogKey(event, dialog)));dialog.addEventListener('cancel', e => { e.preventDefault(); closeDialog(dialog); });- dialog.onclose = () => { disposed = true; for (const f of cleanup) f(); stage.querySelector('video')?.pause(); dialog.remove(); if (current?.dialog === dialog) { current = null; viewChanged('nodes', node); } };+ dialog.onclose = () => { disposed = true; blockOriginal('extend', false); blockOriginal('preview', false); for (const f of cleanup) f(); stage.querySelector('video')?.pause(); dialog.remove(); if (current?.dialog === dialog) { current = null; viewChanged('nodes', node); } };capabilityView = createCapabilityView({node, api, t, getState: () => ({ratio,custom: selected === 'custom', steps: Number(baseSteps.value), twoPass: twoPass.checked,refine: Number(refineSteps.value), stepCache: stepCacheNow(), upscale: upscaleNow(), disabled: Boolean(extension.source) || dimensionsLinked || seconds.disabled}),@@ -2060,7 +2090,7 @@ export function openStudio(node, {extendFrom = null} = {}) {function syncExtension() {releaseExtendThumbnail(); extendMount.replaceChildren();const source = extension.source;- extendMount.hidden = !source; mediaMount.classList.toggle('fv-extend-active', Boolean(source));+ extendMount.hidden = !source; mediaMount.classList.toggle('fv-extend-active', Boolean(source)); blockOriginal('extend', Boolean(source));if (source) runMode = 'single';canvasLock.hidden = !source;mp.disabled = width.disabled = height.disabled = Boolean(source) || dimensionsLinked;diff --git a/web/studio_length.js b/web/studio_length.jsindex dfd0b41..5c58d77 100644--- a/web/studio_length.js+++ b/web/studio_length.js@@ -76,7 +76,8 @@ export function createStudioLength({api, node, seconds, advanced, fields, t, fielet plan = null, error = null, absent = false, disposed = false, pending = null, edited = false, blockEdited = false, plannedSeconds = null;const remember = (key, value) => { node.properties ||= {}; node.properties[key] = value; node.graph?.change(); node.setDirtyCanvas?.(true, true); };const total = () => Number(seconds.value);- const effectiveSingle = () => node.properties?.[LENGTH_PROPERTIES.single] === true && total() <= (planLimits?.singleMaximum ?? 15);+ let forced = false;+ const effectiveSingle = () => forced || (node.properties?.[LENGTH_PROPERTIES.single] === true && total() <= (planLimits?.singleMaximum ?? 15));function options() {// Pin short fallback plans that would otherwise qualify for a single run.return {seconds: total(), single: effectiveSingle(), singleLimit: planLimits ? effectiveSingle() ? planLimits.singleMaximum : planLimits.maximum : null,@@ -102,7 +103,7 @@ export function createStudioLength({api, node, seconds, advanced, fields, t, fieconst maximum = planLimits?.maximum, forcedMaximum = planLimits?.singleMaximum;const clamped = positive(maximum) && node.properties?.[LENGTH_PROPERTIES.block] > maximum;const cannot = total() > (forcedMaximum ?? 15);- single.disabled = absent || seconds.disabled || cannot; single.checked = !absent && effectiveSingle();+ single.disabled = absent || seconds.disabled || cannot || forced; single.checked = !absent && effectiveSingle();if (absent) seconds.max = '60';const reason = cannot ? positive(forcedMaximum)? t(`Up to ${Math.floor(forcedMaximum)}\u00a0s in one run at this size.`, `当前画面大小单次最长 ${Math.floor(forcedMaximum)}\u00a0秒。`)@@ -167,10 +168,10 @@ export function createStudioLength({api, node, seconds, advanced, fields, t, fie};const input = () => { submitted = false; planLimits = limits; edited = false; invalidate(); schedule(); };seconds.addEventListener('input', input);- function commit({report = true} = {}) {+ function commit({report = true, exact = false} = {}) {const valid = report ? seconds.reportValidity() : seconds.checkValidity();if (!seconds.value || !valid) return false;- const value = total() > 30 ? Math.round(total()) : total(); seconds.value = value;+ const value = !exact && total() > 30 ? Math.round(total()) : total(); seconds.value = value;if (value > 60) remember(LENGTH_PROPERTIES.seconds, value);else if (node.properties && Object.hasOwn(node.properties, LENGTH_PROPERTIES.seconds)) { delete node.properties[LENGTH_PROPERTIES.seconds]; node.graph?.change(); }setSeconds(Math.min(60, value));@@ -185,6 +186,8 @@ export function createStudioLength({api, node, seconds, advanced, fields, t, fienoParts = true; writeHint(); invalidate(); schedule();}).catch(() => {});return {+ // Short original audio keeps its single run; long selections can use parts.+ forceSingle(on) { if (forced === on) return; forced = on; submitted = false; invalidate(); schedule(); },commit, options, get plan() { return plan; }, get absent() { return absent; },get needsPlan() { return !absent && !effectiveSingle() && (total() > singleLimitSeconds(planLimits?.maximum)|| Object.hasOwn(node.properties || {}, LENGTH_PROPERTIES.block)); },