Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Voice change: lines are numbered by who says them, known text replaces the recognised text, and...

Bowen Xued10c005b9bd3
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index f4dcf9a..5fb613e 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -366,8 +366,11 @@ VOICE_LINE = obj(dict(id=string(16, pattern=r'^l[0-9]{3,5}$'), start=number(0, 3
('id', 'start', 'end', 'text', 'original_text'))
VOICE_REFERENCE = obj(dict(clip_id=string(64), start=number(0, 36000), end=number(0, 36000), text=string(1000)),
('clip_id', 'start', 'text'), description='A stretch of a sound clip whose voice every line takes.')
+VOICE_SCRIPT = array(obj(dict(text=string(2000), speaker=string(64, description='Who says it; the same name is the same number in the result.')),
+ ('text',)), 1, 400,
+ description='What the selection is known to say, in order. A recognised line that is one of these says it as written, and the lines are numbered by who says them.')
VOICE_LINES_FIELDS = dict(
- selection=VOICE_SELECTION, language=VOICE_LANGUAGE,
+ selection=VOICE_SELECTION, language=VOICE_LANGUAGE, script=VOICE_SCRIPT,
separate_first={'type': 'boolean', 'default': False,
'description': 'Separate the sound first and recognise on the dialogue alone.'})
VOICE_FIELDS = dict(
diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py
index ef3200a..468e495 100644
--- a/freevideo_engine/agent/spec.py
+++ b/freevideo_engine/agent/spec.py
@@ -302,7 +302,8 @@ def normalize(value, *, port=None):
lay=value.get('lay', True))
elif kind == 'voice_lines':
result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
- language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)))
+ language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)),
+ **({'script': [dict(entry) for entry in value['script']]} if value.get('script') else {}))
elif kind == 'voice':
result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'),
diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py
index fc3602d..96e9ff8 100644
--- a/freevideo_engine/agent/voice.py
+++ b/freevideo_engine/agent/voice.py
@@ -304,7 +304,7 @@ def run_voice_lines(jobs, row):
"""The lines of a selection: where each one is (from the voice) and what it says (from the recogniser)."""
from .. import subtitle_ears
from .. import subtitle_sherpa # noqa: F401 registers the recogniser
- from ..voice import sentences
+ from ..voice import script, sentences, speakers
spec = row['spec']
ctx = context(jobs, spec)
sources = None
@@ -338,10 +338,22 @@ def run_voice_lines(jobs, row):
'The selection could not be recognised (%s)' % error.reason) from error
words = [word for line in heard['lines'] for word in line['words']]
rows = sentences.lines(words, audio, rate)
+ known = spec.get('script') or []
+ entries = script.match(rows, known) if known else [None] * len(rows)
+ for line, entry in zip(rows, entries):
+ if entry is not None:
+ # the text is known: it is what the line says, and the recogniser only gives the times
+ line['words'] = script.retime(line['words'], entry['text'], line['start'], line['end']) or line['words']
+ line['text'] = sentences._join(line['words'])
+ slices = [audio[int(line['start'] * rate):int(line['end'] * rate)] for line in rows]
+ numbers = speakers.label(slices, speakers.embedder())
+ if known:
+ numbers = speakers.by_role(numbers, [None if entry is None or not (entry.get('speaker') or '').strip()
+ else script.roles(known)[entry['speaker'].strip()] for entry in entries])
lines = []
for index, line in enumerate(rows, 1):
lines.append(dict(id='l%03d' % index, start=round(line['start'], 3), end=round(line['end'], 3), text=line['text'],
- speaker=0, words=line['words'],
+ speaker=numbers[index - 1], words=line['words'],
rate=round(max(len(line['words']), 1) / max(line['end'] - line['start'], .1), 2)))
kind = 'dialogue' if ctx['separated'] or sources else 'a1'
return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'],
@@ -574,6 +586,10 @@ def run_voice_replace(jobs, row, *, poll=.25):
reference, own = _reference(jobs, spec, voice, every, original_wav, base)
seed = _seed(spec, todo)
pace = Pace(work)
+ # a text that cannot be said in the room its line has, even at the fastest tempo, is known before anything is made
+ expected = {found['id']: found['overrun'] for found in estimate(every, ctx['end'])['lines'] if found['overrun']}
+ if expected:
+ _phase(row, 'preparing', too_long=sorted(expected))
def item(line, length, offset):
window = windows[line['id']]
@@ -626,6 +642,8 @@ def run_voice_replace(jobs, row, *, poll=.25):
if made is None:
result['code'] = 'cancelled'
continue
+ if line['id'] in expected:
+ result['expected_overrun'] = expected[line['id']]
result.update(available=round((window['deadline'] if line['id'] in second_round else window['last']) - window['onset'], 3),
speed=made.get('speed'), stretch=made.get('stretch'), gain_db=made.get('gain_db'),
overrun=round(float(made.get('overrun') or 0.), 3), warning=made.get('warning'))
diff --git a/freevideo_engine/voice/script.py b/freevideo_engine/voice/script.py
new file mode 100644
index 0000000..7547f49
--- /dev/null
+++ b/freevideo_engine/voice/script.py
@@ -0,0 +1,97 @@
+"""A script given with a selection: what each line is to say, and who says it.
+
+The recogniser misspells and misses (a name, a number, a word shorter than a third of a second). When the text of a line
+is known, it replaces the recognised text and keeps the recogniser only for the times: the letters of the known text are
+laid on the letters that were heard, and the ones that were not heard are spread between their neighbours.
+"""
+import re
+from difflib import SequenceMatcher
+
+FLOOR = .55 # least likeness of a recognised line to a script line for them to be the same line
+MINIMUM = .06 # no word is given less time than this, seconds
+_CJK = '぀-ヿ㐀-鿿가-힯'
+_PIECE = re.compile('[%s]|[^\\W_%s]+(?:\'[^\\W_%s]+)?' % (_CJK, _CJK, _CJK))
+
+
+def letters(text):
+ return ''.join(char for char in text.lower() if char.isalnum())
+
+
+def pieces(text):
+ """The words of a text (a character each for Chinese, Japanese and Korean)."""
+ return [found.group(0) for found in _PIECE.finditer(text)]
+
+
+def match(lines, script):
+ """For each recognised line, the script entry it is (or None), in order; an entry is used once.
+
+ `lines` are [{text}], `script` [{text, speaker?}]."""
+ wanted = [letters(entry['text']) for entry in script]
+ used, found = set(), []
+ for line in lines:
+ heard = letters(line['text'])
+ best, likeness = None, FLOOR
+ for index, text in enumerate(wanted):
+ if index in used or not heard or not text:
+ continue
+ ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()
+ if ratio >= likeness:
+ best, likeness = index, ratio
+ if best is not None:
+ used.add(best)
+ found.append(None if best is None else script[best])
+ return found
+
+
+def retime(words, text, start, end):
+ """The words of `text` with times taken from the recognised `words` ([{text, start, end}]) of the same line.
+
+ Letters that were heard keep their time; the others share what lies between the heard ones, in proportion to their
+ count, inside [start, end]."""
+ heard, spans = [], []
+ for word in words:
+ chars = letters(word['text'])
+ for index, char in enumerate(chars):
+ share = (word['end'] - word['start']) / max(len(chars), 1)
+ heard.append(char)
+ spans.append((word['start'] + index * share, word['start'] + (index + 1) * share))
+ given = pieces(text)
+ chars = []
+ for index, piece in enumerate(given):
+ chars.extend((char, index) for char in letters(piece))
+ times = [None] * len(chars)
+ for block in SequenceMatcher(None, ''.join(heard), ''.join(char for char, _ in chars), autojunk=False).get_matching_blocks():
+ for offset in range(block.size):
+ times[block.b + offset] = spans[block.a + offset]
+ anchors = [-1] + [index for index, found in enumerate(times) if found] + [len(chars)]
+ for before, after in zip(anchors, anchors[1:]):
+ if after - before <= 1:
+ continue
+ first = times[before][1] if before >= 0 else start
+ last = times[after][0] if after < len(chars) else end
+ last = max(last, first)
+ for offset in range(before + 1, after):
+ step = (last - first) / (after - before - 1)
+ times[offset] = (first + (offset - before - 1) * step, first + (offset - before) * step)
+ result = []
+ for index, piece in enumerate(given):
+ own = [times[at] for at, (_, word) in enumerate(chars) if word == index]
+ if not own:
+ continue
+ result.append(dict(text=piece, start=round(min(span[0] for span in own), 3), end=round(max(span[1] for span in own), 3)))
+ previous = start
+ for word in result:
+ word['start'] = round(max(word['start'], previous), 3)
+ word['end'] = round(max(word['end'], word['start'] + MINIMUM), 3)
+ previous = word['start']
+ return result
+
+
+def roles(script):
+ """Role name -> number, in order of first appearance."""
+ numbers = {}
+ for entry in script:
+ name = (entry.get('speaker') or '').strip()
+ if name and name not in numbers:
+ numbers[name] = len(numbers)
+ return numbers
diff --git a/freevideo_engine/voice/speakers.py b/freevideo_engine/voice/speakers.py
new file mode 100644
index 0000000..b8767be
--- /dev/null
+++ b/freevideo_engine/voice/speakers.py
@@ -0,0 +1,115 @@
+"""Who says each line: lines that sound like one person get one number.
+
+A speaker embedding of the line (the model of the dubbing pack) is compared with the average of each person met so
+far. Without the model or its runtime every line is speaker 0, as before.
+"""
+import logging
+from collections import Counter
+
+import numpy as np
+
+_LOG = logging.getLogger(__name__)
+SAME = .8 # cosine likeness from which two lines are one person (measured on film dialogue, see the card)
+SHORTEST = .8 # a shorter line has too little voice to be told apart; it follows the line before it
+RATE = 16000
+
+
+def embedder(root=None):
+ """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here."""
+ try:
+ from ..dubbing import pack
+ file = pack.directory(root) / 'campplus.onnx'
+ if not file.is_file():
+ return None
+ try:
+ import onnxruntime
+ except ImportError:
+ if not pack.activate(pack.runtime_path(root)):
+ return None
+ import onnxruntime
+ import torch
+ import torchaudio.compliance.kaldi as kaldi
+ options = onnxruntime.SessionOptions()
+ options.intra_op_num_threads = 1
+ session = onnxruntime.InferenceSession(str(file), sess_options=options, providers=['CPUExecutionProvider'])
+ except Exception as error: # a missing optional part must never fail the lines
+ _LOG.info('Speakers are not told apart: %s', error)
+ return None
+ name = session.get_inputs()[0].name
+
+ def embed(samples):
+ feat = kaldi.fbank(torch.from_numpy(np.ascontiguousarray(samples, dtype=np.float32))[None], num_mel_bins=80,
+ dither=0, sample_frequency=RATE)
+ feat = feat - feat.mean(dim=0, keepdim=True)
+ return np.asarray(session.run(None, {name: feat.unsqueeze(0).numpy()})[0]).flatten()
+ return embed
+
+
+def cluster(vectors, same=SAME):
+ """Numbers for vectors ([array | None]) in order: a vector joins the person it is most like, if alike enough, else is a new one.
+
+ None (a line too short to tell) takes the number of the line before it, or of the first one that has a vector."""
+ centres, counts, numbers = [], [], []
+ for vector in vectors:
+ if vector is None:
+ numbers.append(None)
+ continue
+ vector = np.asarray(vector, dtype=np.float64)
+ vector = vector / (np.linalg.norm(vector) or 1.)
+ best, likeness = None, same
+ for index, centre in enumerate(centres):
+ value = float(np.dot(vector, centre / (np.linalg.norm(centre) or 1.)))
+ if value >= likeness:
+ best, likeness = index, value
+ if best is None:
+ centres.append(vector.copy())
+ counts.append(1)
+ best = len(centres) - 1
+ else:
+ centres[best] = (centres[best] * counts[best] + vector) / (counts[best] + 1)
+ counts[best] += 1
+ numbers.append(best)
+ known = [number for number in numbers if number is not None]
+ fill = known[0] if known else 0
+ for index, number in enumerate(numbers):
+ if number is None:
+ numbers[index] = numbers[index - 1] if index else fill
+ return numbers
+
+
+def label(slices, embed):
+ """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder)."""
+ if embed is None:
+ return [0] * len(slices)
+ vectors = []
+ for samples in slices:
+ try:
+ vectors.append(embed(samples) if len(samples) >= SHORTEST * RATE else None)
+ except Exception as error:
+ _LOG.info('A line could not be told apart: %s', error)
+ vectors.append(None)
+ return cluster(vectors)
+
+
+def by_role(numbers, roles):
+ """Speaker numbers where the lines that match a script say who they are.
+
+ `numbers`: the cluster of each line; `roles`: the role number of each line or None. A cluster takes the role most of
+ its matched lines have; a line without a cluster match keeps the role of the nearest line that has one. Without any
+ role the clusters stand as they are."""
+ if all(role is None for role in roles):
+ return list(numbers)
+ votes = {}
+ for number, role in zip(numbers, roles):
+ if role is not None:
+ votes.setdefault(number, Counter())[role] += 1
+ result = []
+ for number, role in zip(numbers, roles):
+ if role is None and number in votes:
+ role = votes[number].most_common(1)[0][0]
+ result.append(role)
+ known = [index for index, role in enumerate(result) if role is not None]
+ for index, role in enumerate(result):
+ if role is None:
+ result[index] = result[min(known, key=lambda at: abs(at - index))] if known else 0
+ return result