Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Voice change: lines are numbered by who says them, known text replaces the recognised text, and...
Bowen Xued10c005b9bd3
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.pyindex f4dcf9a..5fb613e 100644--- a/freevideo_engine/agent/schema.py+++ b/freevideo_engine/agent/schema.py@@ -366,8 +366,11 @@ VOICE_LINE = obj(dict(id=string(16, pattern=r'^l[0-9]{3,5}$'), start=number(0, 3('id', 'start', 'end', 'text', 'original_text'))VOICE_REFERENCE = obj(dict(clip_id=string(64), start=number(0, 36000), end=number(0, 36000), text=string(1000)),('clip_id', 'start', 'text'), description='A stretch of a sound clip whose voice every line takes.')+VOICE_SCRIPT = array(obj(dict(text=string(2000), speaker=string(64, description='Who says it; the same name is the same number in the result.')),+ ('text',)), 1, 400,+ description='What the selection is known to say, in order. A recognised line that is one of these says it as written, and the lines are numbered by who says them.')VOICE_LINES_FIELDS = dict(- selection=VOICE_SELECTION, language=VOICE_LANGUAGE,+ selection=VOICE_SELECTION, language=VOICE_LANGUAGE, script=VOICE_SCRIPT,separate_first={'type': 'boolean', 'default': False,'description': 'Separate the sound first and recognise on the dialogue alone.'})VOICE_FIELDS = dict(diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.pyindex ef3200a..468e495 100644--- a/freevideo_engine/agent/spec.py+++ b/freevideo_engine/agent/spec.py@@ -302,7 +302,8 @@ def normalize(value, *, port=None):lay=value.get('lay', True))elif kind == 'voice_lines':result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),- language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)))+ language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)),+ **({'script': [dict(entry) for entry in value['script']]} if value.get('script') else {}))elif kind == 'voice':result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'),diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.pyindex fc3602d..96e9ff8 100644--- a/freevideo_engine/agent/voice.py+++ b/freevideo_engine/agent/voice.py@@ -304,7 +304,7 @@ def run_voice_lines(jobs, row):"""The lines of a selection: where each one is (from the voice) and what it says (from the recogniser)."""from .. import subtitle_earsfrom .. import subtitle_sherpa # noqa: F401 registers the recogniser- from ..voice import sentences+ from ..voice import script, sentences, speakersspec = row['spec']ctx = context(jobs, spec)sources = None@@ -338,10 +338,22 @@ def run_voice_lines(jobs, row):'The selection could not be recognised (%s)' % error.reason) from errorwords = [word for line in heard['lines'] for word in line['words']]rows = sentences.lines(words, audio, rate)+ known = spec.get('script') or []+ entries = script.match(rows, known) if known else [None] * len(rows)+ for line, entry in zip(rows, entries):+ if entry is not None:+ # the text is known: it is what the line says, and the recogniser only gives the times+ line['words'] = script.retime(line['words'], entry['text'], line['start'], line['end']) or line['words']+ line['text'] = sentences._join(line['words'])+ slices = [audio[int(line['start'] * rate):int(line['end'] * rate)] for line in rows]+ numbers = speakers.label(slices, speakers.embedder())+ if known:+ numbers = speakers.by_role(numbers, [None if entry is None or not (entry.get('speaker') or '').strip()+ else script.roles(known)[entry['speaker'].strip()] for entry in entries])lines = []for index, line in enumerate(rows, 1):lines.append(dict(id='l%03d' % index, start=round(line['start'], 3), end=round(line['end'], 3), text=line['text'],- speaker=0, words=line['words'],+ speaker=numbers[index - 1], words=line['words'],rate=round(max(len(line['words']), 1) / max(line['end'] - line['start'], .1), 2)))kind = 'dialogue' if ctx['separated'] or sources else 'a1'return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'],@@ -574,6 +586,10 @@ def run_voice_replace(jobs, row, *, poll=.25):reference, own = _reference(jobs, spec, voice, every, original_wav, base)seed = _seed(spec, todo)pace = Pace(work)+ # a text that cannot be said in the room its line has, even at the fastest tempo, is known before anything is made+ expected = {found['id']: found['overrun'] for found in estimate(every, ctx['end'])['lines'] if found['overrun']}+ if expected:+ _phase(row, 'preparing', too_long=sorted(expected))def item(line, length, offset):window = windows[line['id']]@@ -626,6 +642,8 @@ def run_voice_replace(jobs, row, *, poll=.25):if made is None:result['code'] = 'cancelled'continue+ if line['id'] in expected:+ result['expected_overrun'] = expected[line['id']]result.update(available=round((window['deadline'] if line['id'] in second_round else window['last']) - window['onset'], 3),speed=made.get('speed'), stretch=made.get('stretch'), gain_db=made.get('gain_db'),overrun=round(float(made.get('overrun') or 0.), 3), warning=made.get('warning'))diff --git a/freevideo_engine/voice/script.py b/freevideo_engine/voice/script.pynew file mode 100644index 0000000..7547f49--- /dev/null+++ b/freevideo_engine/voice/script.py@@ -0,0 +1,97 @@+"""A script given with a selection: what each line is to say, and who says it.++The recogniser misspells and misses (a name, a number, a word shorter than a third of a second). When the text of a line+is known, it replaces the recognised text and keeps the recogniser only for the times: the letters of the known text are+laid on the letters that were heard, and the ones that were not heard are spread between their neighbours.+"""+import re+from difflib import SequenceMatcher++FLOOR = .55 # least likeness of a recognised line to a script line for them to be the same line+MINIMUM = .06 # no word is given less time than this, seconds+_CJK = '-ヿ㐀-鿿가-'+_PIECE = re.compile('[%s]|[^\\W_%s]+(?:\'[^\\W_%s]+)?' % (_CJK, _CJK, _CJK))+++def letters(text):+ return ''.join(char for char in text.lower() if char.isalnum())+++def pieces(text):+ """The words of a text (a character each for Chinese, Japanese and Korean)."""+ return [found.group(0) for found in _PIECE.finditer(text)]+++def match(lines, script):+ """For each recognised line, the script entry it is (or None), in order; an entry is used once.++ `lines` are [{text}], `script` [{text, speaker?}]."""+ wanted = [letters(entry['text']) for entry in script]+ used, found = set(), []+ for line in lines:+ heard = letters(line['text'])+ best, likeness = None, FLOOR+ for index, text in enumerate(wanted):+ if index in used or not heard or not text:+ continue+ ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()+ if ratio >= likeness:+ best, likeness = index, ratio+ if best is not None:+ used.add(best)+ found.append(None if best is None else script[best])+ return found+++def retime(words, text, start, end):+ """The words of `text` with times taken from the recognised `words` ([{text, start, end}]) of the same line.++ Letters that were heard keep their time; the others share what lies between the heard ones, in proportion to their+ count, inside [start, end]."""+ heard, spans = [], []+ for word in words:+ chars = letters(word['text'])+ for index, char in enumerate(chars):+ share = (word['end'] - word['start']) / max(len(chars), 1)+ heard.append(char)+ spans.append((word['start'] + index * share, word['start'] + (index + 1) * share))+ given = pieces(text)+ chars = []+ for index, piece in enumerate(given):+ chars.extend((char, index) for char in letters(piece))+ times = [None] * len(chars)+ for block in SequenceMatcher(None, ''.join(heard), ''.join(char for char, _ in chars), autojunk=False).get_matching_blocks():+ for offset in range(block.size):+ times[block.b + offset] = spans[block.a + offset]+ anchors = [-1] + [index for index, found in enumerate(times) if found] + [len(chars)]+ for before, after in zip(anchors, anchors[1:]):+ if after - before <= 1:+ continue+ first = times[before][1] if before >= 0 else start+ last = times[after][0] if after < len(chars) else end+ last = max(last, first)+ for offset in range(before + 1, after):+ step = (last - first) / (after - before - 1)+ times[offset] = (first + (offset - before - 1) * step, first + (offset - before) * step)+ result = []+ for index, piece in enumerate(given):+ own = [times[at] for at, (_, word) in enumerate(chars) if word == index]+ if not own:+ continue+ result.append(dict(text=piece, start=round(min(span[0] for span in own), 3), end=round(max(span[1] for span in own), 3)))+ previous = start+ for word in result:+ word['start'] = round(max(word['start'], previous), 3)+ word['end'] = round(max(word['end'], word['start'] + MINIMUM), 3)+ previous = word['start']+ return result+++def roles(script):+ """Role name -> number, in order of first appearance."""+ numbers = {}+ for entry in script:+ name = (entry.get('speaker') or '').strip()+ if name and name not in numbers:+ numbers[name] = len(numbers)+ return numbersdiff --git a/freevideo_engine/voice/speakers.py b/freevideo_engine/voice/speakers.pynew file mode 100644index 0000000..b8767be--- /dev/null+++ b/freevideo_engine/voice/speakers.py@@ -0,0 +1,115 @@+"""Who says each line: lines that sound like one person get one number.++A speaker embedding of the line (the model of the dubbing pack) is compared with the average of each person met so+far. Without the model or its runtime every line is speaker 0, as before.+"""+import logging+from collections import Counter++import numpy as np++_LOG = logging.getLogger(__name__)+SAME = .8 # cosine likeness from which two lines are one person (measured on film dialogue, see the card)+SHORTEST = .8 # a shorter line has too little voice to be told apart; it follows the line before it+RATE = 16000+++def embedder(root=None):+ """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here."""+ try:+ from ..dubbing import pack+ file = pack.directory(root) / 'campplus.onnx'+ if not file.is_file():+ return None+ try:+ import onnxruntime+ except ImportError:+ if not pack.activate(pack.runtime_path(root)):+ return None+ import onnxruntime+ import torch+ import torchaudio.compliance.kaldi as kaldi+ options = onnxruntime.SessionOptions()+ options.intra_op_num_threads = 1+ session = onnxruntime.InferenceSession(str(file), sess_options=options, providers=['CPUExecutionProvider'])+ except Exception as error: # a missing optional part must never fail the lines+ _LOG.info('Speakers are not told apart: %s', error)+ return None+ name = session.get_inputs()[0].name++ def embed(samples):+ feat = kaldi.fbank(torch.from_numpy(np.ascontiguousarray(samples, dtype=np.float32))[None], num_mel_bins=80,+ dither=0, sample_frequency=RATE)+ feat = feat - feat.mean(dim=0, keepdim=True)+ return np.asarray(session.run(None, {name: feat.unsqueeze(0).numpy()})[0]).flatten()+ return embed+++def cluster(vectors, same=SAME):+ """Numbers for vectors ([array | None]) in order: a vector joins the person it is most like, if alike enough, else is a new one.++ None (a line too short to tell) takes the number of the line before it, or of the first one that has a vector."""+ centres, counts, numbers = [], [], []+ for vector in vectors:+ if vector is None:+ numbers.append(None)+ continue+ vector = np.asarray(vector, dtype=np.float64)+ vector = vector / (np.linalg.norm(vector) or 1.)+ best, likeness = None, same+ for index, centre in enumerate(centres):+ value = float(np.dot(vector, centre / (np.linalg.norm(centre) or 1.)))+ if value >= likeness:+ best, likeness = index, value+ if best is None:+ centres.append(vector.copy())+ counts.append(1)+ best = len(centres) - 1+ else:+ centres[best] = (centres[best] * counts[best] + vector) / (counts[best] + 1)+ counts[best] += 1+ numbers.append(best)+ known = [number for number in numbers if number is not None]+ fill = known[0] if known else 0+ for index, number in enumerate(numbers):+ if number is None:+ numbers[index] = numbers[index - 1] if index else fill+ return numbers+++def label(slices, embed):+ """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder)."""+ if embed is None:+ return [0] * len(slices)+ vectors = []+ for samples in slices:+ try:+ vectors.append(embed(samples) if len(samples) >= SHORTEST * RATE else None)+ except Exception as error:+ _LOG.info('A line could not be told apart: %s', error)+ vectors.append(None)+ return cluster(vectors)+++def by_role(numbers, roles):+ """Speaker numbers where the lines that match a script say who they are.++ `numbers`: the cluster of each line; `roles`: the role number of each line or None. A cluster takes the role most of+ its matched lines have; a line without a cluster match keeps the role of the nearest line that has one. Without any+ role the clusters stand as they are."""+ if all(role is None for role in roles):+ return list(numbers)+ votes = {}+ for number, role in zip(numbers, roles):+ if role is not None:+ votes.setdefault(number, Counter())[role] += 1+ result = []+ for number, role in zip(numbers, roles):+ if role is None and number in votes:+ role = votes[number].most_common(1)[0][0]+ result.append(role)+ known = [index for index, role in enumerate(result) if role is not None]+ for index, role in enumerate(result):+ if role is None:+ result[index] = result[min(known, key=lambda at: abs(at - index))] if known else 0+ return result