Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Voice change with a script: a sentence cut by a pause is said once (joined only when each line...

Bowen Xue7b2675a129ce
diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py
index 173a34e..9cc6533 100644
--- a/freevideo_engine/agent/voice.py
+++ b/freevideo_engine/agent/voice.py
@@ -340,14 +340,15 @@ def run_voice_lines(jobs, row):
words = [word for line in heard['lines'] for word in line['words']]
rows = sentences.lines(words, audio, rate)
known = spec.get('script') or []
- entries = script.match(rows, known) if known else [None] * len(rows)
+ rows, entries = script.merge(rows, known) if known else (rows, [None] * len(rows))
for line, entry in zip(rows, entries):
if entry is not None:
# the text is known: it is what the line says, and the recogniser only gives the times
line['words'] = script.retime(line['words'], entry['text'], line['start'], line['end']) or line['words']
line['text'] = sentences._join(line['words'])
slices = [audio[int(line['start'] * rate):int(line['end'] * rate)] for line in rows]
- numbers = speakers.label(slices, speakers.embedder())
+ why = []
+ numbers = speakers.label(slices, speakers.embedder(why=why), why)
if known:
numbers = speakers.by_role(numbers, [None if entry is None or not (entry.get('speaker') or '').strip()
else script.roles(known)[entry['speaker'].strip()] for entry in entries])
@@ -359,6 +360,7 @@ def run_voice_lines(jobs, row):
kind = 'dialogue' if ctx['separated'] or sources else 'a1'
return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'],
language=heard.get('language'), speech=bool(lines), lines=lines, source=kind,
+ **(dict(speakers_not_told_apart=why[0]) if why and lines else {}),
recognition=remember(spec, ctx, spec.get('language', 'auto'), lines) if lines else None)
finally:
_drop_folder(jobs, sources)
diff --git a/freevideo_engine/voice/script.py b/freevideo_engine/voice/script.py
index 7547f49..7eec2db 100644
--- a/freevideo_engine/voice/script.py
+++ b/freevideo_engine/voice/script.py
@@ -8,9 +8,14 @@ import re
from difflib import SequenceMatcher
FLOOR = .55 # least likeness of a recognised line to a script line for them to be the same line
+BALANCE = .6 # a recognised line is the script line only if neither is much longer than the other
+SPAN = 4 # most recognised lines (a pause cuts one sentence in pieces) that one script line may cover
+COVER = .85 # the lines a script line covers must say at least this much of it
+GAP = 2. # longest silence, seconds, inside one script line cut by a pause
+LONGEST = 29. # longest line, seconds (as in sentences.LONGEST)
MINIMUM = .06 # no word is given less time than this, seconds
_CJK = '぀-ヿ㐀-鿿가-힯'
-_PIECE = re.compile('[%s]|[^\\W_%s]+(?:\'[^\\W_%s]+)?' % (_CJK, _CJK, _CJK))
+_PIECE = re.compile('[%s]|(?:[^\\W\\d_%s]\\.){2,}(?:[^\\W\\d_%s](?!\\w))?|[^\\W_%s]+(?:[\'’.,-][^\\W_%s]+)*%%?' % (_CJK, _CJK, _CJK, _CJK, _CJK))
def letters(text):
@@ -25,22 +30,69 @@ def pieces(text):
def match(lines, script):
"""For each recognised line, the script entry it is (or None), in order; an entry is used once.
- `lines` are [{text}], `script` [{text, speaker?}]."""
+ `lines` are [{text}], `script` [{text, speaker?}]. Of equally alike entries the first unused one is taken."""
+ return [entry for _, entry in _assign(lines, script)]
+
+
+def _assign(lines, script):
+ """[(number of lines covered, entry)] for each line: the first of a covered run holds the run, the rest (0, None)."""
wanted = [letters(entry['text']) for entry in script]
- used, found = set(), []
- for line in lines:
- heard = letters(line['text'])
- best, likeness = None, FLOOR
- for index, text in enumerate(wanted):
- if index in used or not heard or not text:
- continue
- ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()
- if ratio >= likeness:
- best, likeness = index, ratio
+ used, result, at = set(), [], 0
+ while at < len(lines):
+ best, likeness, span = None, FLOOR, 1
+ for count in range(1, min(SPAN, len(lines) - at) + 1):
+ group = lines[at:at + count]
+ heard = letters(' '.join(line['text'] for line in group))
+ for index, text in enumerate(wanted):
+ if index in used or not heard or not text or min(len(heard), len(text)) < BALANCE * max(len(heard), len(text)):
+ continue
+ if count > 1 and not _one_line(group, text):
+ continue
+ ratio = SequenceMatcher(None, heard, text, autojunk=False).ratio()
+ if ratio >= FLOOR and (best is None or ratio > likeness):
+ best, likeness, span = index, ratio, count
if best is not None:
used.add(best)
- found.append(None if best is None else script[best])
- return found
+ result.append((span, None if best is None else script[best]))
+ result.extend([(0, None)] * (span - 1))
+ at += span
+ return result
+
+
+def _one_line(group, text):
+ """Whether the recognised `group` of lines is one script line (letters `text`) cut by pauses: each line is the part
+ of it that stands in its place, together they say all of it, and no silence or length is more than one line has."""
+ parts = [letters(line['text']) for line in group]
+ if sum(map(len, parts)) < COVER * len(text):
+ return False
+ here = 0
+ for part in parts:
+ if SequenceMatcher(None, part, text[here:here + len(part)], autojunk=False).ratio() < FLOOR:
+ return False
+ here += len(part)
+ if all('start' in line and 'end' in line for line in group):
+ return (group[-1]['end'] - group[0]['start'] <= LONGEST
+ and all(after['start'] - before['end'] <= GAP for before, after in zip(group, group[1:])))
+ return True
+
+
+def merge(lines, script):
+ """(lines, entries) where the recognised lines that one script line covers are joined into one line.
+
+ A sentence cut by a pause is two recognised lines; joined, it is said once. `lines` are [{text, words, start, end}]."""
+ rows, entries, at = [], [], 0
+ for span, entry in _assign(lines, script):
+ if span == 0:
+ continue
+ group = lines[at:at + span]
+ at += span
+ row = dict(group[0])
+ if span > 1:
+ row.update(words=[word for line in group for word in line['words']], end=group[-1]['end'],
+ text=' '.join(line['text'] for line in group))
+ rows.append(row)
+ entries.append(entry)
+ return rows, entries
def retime(words, text, start, end):
diff --git a/freevideo_engine/voice/speakers.py b/freevideo_engine/voice/speakers.py
index b8767be..0e48df5 100644
--- a/freevideo_engine/voice/speakers.py
+++ b/freevideo_engine/voice/speakers.py
@@ -14,17 +14,22 @@ SHORTEST = .8 # a shorter line has too little voice to be told apart; it
RATE = 16000
-def embedder(root=None):
- """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here."""
+def embedder(root=None, why=None):
+ """A function from mono 16 kHz samples to a vector, or None when the model or its runtime is not here.
+
+ The reason it is None is appended to the list `why` (a short code, never a path)."""
+ why = why if why is not None else []
try:
from ..dubbing import pack
file = pack.directory(root) / 'campplus.onnx'
if not file.is_file():
+ why.append('model_missing')
return None
try:
import onnxruntime
except ImportError:
if not pack.activate(pack.runtime_path(root)):
+ why.append('runtime_missing')
return None
import onnxruntime
import torch
@@ -34,6 +39,7 @@ def embedder(root=None):
session = onnxruntime.InferenceSession(str(file), sess_options=options, providers=['CPUExecutionProvider'])
except Exception as error: # a missing optional part must never fail the lines
_LOG.info('Speakers are not told apart: %s', error)
+ why.append('load_failed:%s' % type(error).__name__)
return None
name = session.get_inputs()[0].name
@@ -77,8 +83,10 @@ def cluster(vectors, same=SAME):
return numbers
-def label(slices, embed):
- """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder)."""
+def label(slices, embed, why=None):
+ """Speaker numbers for the mono 16 kHz `slices` of the lines, in order (all 0 without an embedder).
+
+ When `why` (a list) already holds a reason it stays; a line that could not be told apart adds 'line_failed:<error>'."""
if embed is None:
return [0] * len(slices)
vectors = []
@@ -87,6 +95,8 @@ def label(slices, embed):
vectors.append(embed(samples) if len(samples) >= SHORTEST * RATE else None)
except Exception as error:
_LOG.info('A line could not be told apart: %s', error)
+ if why is not None and not why:
+ why.append('line_failed:%s' % type(error).__name__)
vectors.append(None)
return cluster(vectors)