Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Voice change with a script: a sentence cut by a pause is said once (joined only when each line...
Bowen Xue7b2675a129ce
diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.pyindex 173a34e..9cc6533 100644--- a/freevideo_engine/agent/voice.py+++ b/freevideo_engine/agent/voice.py@@ -340,14 +340,15 @@ def run_voice_lines(jobs, row):words = [word for line in heard['lines'] for word in line['words']]rows = sentences.lines(words, audio, rate)known = spec.get('script') or []- entries = script.match(rows, known) if known else [None] * len(rows)+ rows, entries = script.merge(rows, known) if known else (rows, [None] * len(rows))for line, entry in zip(rows, entries):if entry is not None:# the text is known: it is what the line says, and the recogniser only gives the timesline['words'] = script.retime(line['words'], entry['text'], line['start'], line['end']) or line['words']line['text'] = sentences._join(line['words'])slices = [audio[int(line['start'] * rate):int(line['end'] * rate)] for line in rows]- numbers = speakers.label(slices, speakers.embedder())+ why = []+ numbers = speakers.label(slices, speakers.embedder(why=why), why)if known:numbers = speakers.by_role(numbers, [None if entry is None or not (entry.get('speaker') or '').strip()else script.roles(known)[entry['speaker'].strip()] for entry in entries])@@ -359,6 +360,7 @@ def run_voice_lines(jobs, row):kind = 'dialogue' if ctx['separated'] or sources else 'a1'return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'],language=heard.get('language'), speech=bool(lines), lines=lines, source=kind,+ **(dict(speakers_not_told_apart=why[0]) if why and lines else {}),recognition=remember(spec, ctx, spec.get('language', 'auto'), lines) if lines else None)finally:_drop_folder(jobs, sources)diff --git a/freevideo_engine/voice/script.py b/freevideo_engine/voice/script.pyindex 7547f49..7eec2db 100644--- a/freevideo_engine/voice/script.py+++ b/freevideo_engine/voice/script.py@@ -8,9 +8,14 @@ import refrom difflib import SequenceMatcherFLOOR = .55 # least likeness of a recognised line to a script line for them to be the same line+BALANCE = .6 # a recognised line is the script line only if neither is much longer than the other+SPAN = 4 # most recognised lines (a pause cuts one sentence in pieces) that one script line may cover+COVER = .85 # the lines a script line covers must say at least this much of it+GAP = 2. # longest silence, seconds, inside one script line cut by a pause+LONGEST = 29. # longest line, seconds (as in sentences.LONGEST)MINIMUM = .06 # no word is given less time than this, seconds_CJK = '-ヿ㐀-鿿가-'-_PIECE = re.compile('[%s]|[^\\W_%s]+(?:\'[^\\W_%s]+)?' % (_CJK, _CJK, _CJK))+_PIECE = re.compile('[%s]|(?:[^\\W\\d_%s]\\.){2,}(?:[^\\W\\d_%s](?!\\w))?|[^\\W_%s]+(?:[\'’.,-][^\\W_%s]+)*%%?' % (_CJK, _CJK, _CJK, _CJK, _CJK))def letters(text):@@ -25,22 +30,69 @@ def pieces(text):def match(lines, script):"""For each recognised line, the script entry it is (or None), in order; an entry is used once.- `lines` are [{text}], `script` [{text, speaker?}]."""+ `lines` are [{text}], `script` [{text, speaker?}]. Of equally alike entries the first unused one is taken."""+ return [entry for _, entry in _assign(lines, script)]+++def _assign(lines, script):+ """[(number of lines covered, entry)] for each line: the first of a covered run holds the run, the rest (0, None)."""wanted = [letters(entry['text']) for entry in script]- used, found = set(), []- for line in lines:- heard = letters(line['text'])- best, likeness = None, FLOOR- for index, text in enumerate(wanted):- if index in used or not heard or not text:- continue- ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()- if ratio >= likeness:- best, likeness = index, ratio+ used, result, at = set(), [], 0+ while at < len(lines):+ best, likeness, span = None, FLOOR, 1+ for count in range(1, min(SPAN, len(lines) - at) + 1):+ group = lines[at:at + count]+ heard = letters(' '.join(line['text'] for line in group))+ for index, text in enumerate(wanted):+ if index in used or not heard or not text or min(len(heard), len(text)) < BALANCE * max(len(heard), len(text)):+ continue+ if count > 1 and not _one_line(group, text):+ continue+ ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()+ if ratio >= FLOOR and (best is None or ratio > likeness):+ best, likeness, span = index, ratio, countif best is not None:used.add(best)- found.append(None if best is None else script[best])- return found+ result.append((span, None if best is None else script[best]))+ result.extend([(0, None)] * (span - 1))+ at += span+ return result+++def _one_line(group, text):+ """Whether the recognised `group` of lines is one script line (letters `text`) cut by pauses: each line is the part+ of it that stands in its place, together they say all of it, and no silence or length is more than one line has."""+ parts = [letters(line['text']) for line in group]+ if sum(map(len, parts)) < COVER * len(text):+ return False+ here = 0+ for part in parts:+ if SequenceMatcher(None, part, text[here:here + len(part)], autojunk=False).ratio() < FLOOR:+ return False+ here += len(part)+ if all('start' in line and 'end' in line for line in group):+ return (group[-1]['end'] - group[0]['start'] <= LONGEST+ and all(after['start'] - before['end'] <= GAP for before, after in zip(group, group[1:])))+ return True+++def merge(lines, script):+ """(lines, entries) where the recognised lines that one script line covers are joined into one line.++ A sentence cut by a pause is two recognised lines; joined, it is said once. `lines` are [{text, words, start, end}]."""+ rows, entries, at = [], [], 0+ for span, entry in _assign(lines, script):+ if span == 0:+ continue+ group = lines[at:at + span]+ at += span+ row = dict(group[0])+ if span > 1:+ row.update(words=[word for line in group for word in line['words']], end=group[-1]['end'],+ text=' '.join(line['text'] for line in group))+ rows.append(row)+ entries.append(entry)+ return rows, entriesdef retime(words, text, start, end):diff --git a/freevideo_engine/voice/speakers.py b/freevideo_engine/voice/speakers.pyindex b8767be..0e48df5 100644--- a/freevideo_engine/voice/speakers.py+++ b/freevideo_engine/voice/speakers.py@@ -14,17 +14,22 @@ SHORTEST = .8 # a shorter line has too little voice to be told apart; itRATE = 16000-def embedder(root=None):- """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here."""+def embedder(root=None, why=None):+ """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here.++ The reason it is None is appended to the list `why` (a short code, never a path)."""+ why = why if why is not None else []try:from ..dubbing import packfile = pack.directory(root) / 'campplus.onnx'if not file.is_file():+ why.append('model_missing')return Nonetry:import onnxruntimeexcept ImportError:if not pack.activate(pack.runtime_path(root)):+ why.append('runtime_missing')return Noneimport onnxruntimeimport torch@@ -34,6 +39,7 @@ def embedder(root=None):session = onnxruntime.InferenceSession(str(file), sess_options=options, providers=['CPUExecutionProvider'])except Exception as error: # a missing optional part must never fail the lines_LOG.info('Speakers are not told apart: %s', error)+ why.append('load_failed:%s' % type(error).__name__)return Nonename = session.get_inputs()[0].name@@ -77,8 +83,10 @@ def cluster(vectors, same=SAME):return numbers-def label(slices, embed):- """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder)."""+def label(slices, embed, why=None):+ """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder).++ When `why` (a list) already holds a reason it stays; a line that could not be told apart adds 'line_failed:<error>'."""if embed is None:return [0] * len(slices)vectors = []@@ -87,6 +95,8 @@ def label(slices, embed):vectors.append(embed(samples) if len(samples) >= SHORTEST * RATE else None)except Exception as error:_LOG.info('A line could not be told apart: %s', error)+ if why is not None and not why:+ why.append('line_failed:%s' % type(error).__name__)vectors.append(None)return cluster(vectors)