Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Cut detection, per-segment checks and stitching of same-shot takes.
Pure CPU helpers: `detect_cuts` finds the picture cuts of a take, `segment_qa` applies the take checks to one
segment, `seam` measures how a join between two takes sounds and `stitch` picks the best take for every
segment. Thresholds are those of take_qa.
"""
from functools import lru_cache
from pathlib import Path
import numpy as np
from . import take_qa
from .take_qa import _db, _runs
CUT_DIFF = 25.0 # 0-255 mean change between consecutive frames of a hard cut
CUT_SPIKE = 3.0 # a cut stands out this much from the frame changes around it
MATCH_DIFF = 12.0 # weakest change accepted as an expected cut
SEAM_LEVEL_DB = 20.0 # level step at which the sound penalty is full
SEAM_CLICK = 0.5 # sample jump at which the click penalty is full
SEAM_SPEECH = 0.4 # penalty added for a join inside speech
SEAM_QUIET = 0.25 # weight of level and click at a join in silence
@lru_cache(maxsize=4)
def _decoded(path, stamp):
return take_qa._pictures(path)
@lru_cache(maxsize=4)
def _heard(path, stamp):
return take_qa._samples(path)
def _stamp(path):
stat = Path(path).stat()
return stat.st_mtime_ns, stat.st_size
def _pictures(path):
"""take_qa's frames, decoded once while the same file is checked part by part."""
return _decoded(str(path), _stamp(path))
def _samples(path):
return _heard(str(path), _stamp(path))
def _changes(small):
return [0.0] + [float(np.abs(small[i] - small[i - 1]).mean()) for i in range(1, len(small))]
def _spike(diffs, index, lo, hi, floor):
"""Whether frame `index` is a cut: a large change that stands out from its neighbours in lo..hi."""
around = [diffs[i] for i in range(lo, hi) if i != index]
return diffs[index] >= floor and diffs[index] >= CUT_SPIKE * (float(np.median(around)) if around else 0.0)
def cuts_in(small, times):
"""Seconds of the hard picture cuts in decoded frames, ascending, excluding 0 and the end."""
diffs = _changes(small)
return [round(times[i], 3) for i in range(1, len(times))
if (diffs[i] >= CUT_DIFF and diffs[i] >= diffs[i - 1] and diffs[i] > (diffs[i + 1] if i + 1 < len(diffs) else -1)
and _spike(diffs, i, max(1, i - 3), min(len(diffs), i + 4), CUT_DIFF))]
def detect_cuts(path, expected=None, *, tolerance=0.6):
"""Seconds of the hard picture cuts, ascending, excluding 0 and the end. With `expected`, each expected cut
takes the strongest frame change within `tolerance` s; one with no such change is None, not guessed."""
_, small, times = _pictures(path)
if not expected:
return cuts_in(small, times)
diffs = _changes(small)
found = []
for wanted in expected:
near = [i for i in range(1, len(times)) if abs(times[i] - wanted) <= tolerance]
best = max(near, key=lambda i: (diffs[i], -abs(times[i] - wanted)), default=None)
if best is not None and _spike(diffs, best, max(1, min(near) - 1), min(len(diffs), max(near) + 2), MATCH_DIFF):
found.append(times[best])
else:
found.append(None)
# one frame cannot be two cuts: the cut nearer its expected time keeps it, the other is not found
for index, at in enumerate(found):
for other in range(index):
if at is not None and found[other] == at:
loser = index if abs(at - expected[index]) >= abs(at - expected[other]) else other
found[loser] = None
if loser == index:
break
return found
def segment_qa(path, start, end):
"""Black picture, frozen picture and a long silence in the middle of speech, inside [start, end)."""
problems, notes = [], []
means, small, times = _pictures(path)
keep = [i for i, t in enumerate(times) if start <= t < end]
if keep:
means, small, times = [means[i] for i in keep], [small[i] for i in keep], [times[i] for i in keep]
black = _runs([m < take_qa.BLACK_LUMA and float(g.std()) < take_qa.BLACK_SPREAD for m, g in zip(means, small)],
times, take_qa.BLACK_SECONDS)
flags = [False] * len(small)
for index in range(1, len(small)):
if float(np.abs(small[index] - small[index - 1]).mean()) <= take_qa.FREEZE_DIFF:
flags[index] = flags[index - 1] = True
frozen = _runs(flags, times, take_qa.FREEZE_SECONDS)
if black:
problems.append('black picture at %s' % ', '.join('%.1f-%.1f s' % s for s in black))
if frozen:
problems.append('frozen picture at %s' % ', '.join('%.1f-%.1f s' % s for s in frozen))
samples = _samples(path)[int(start * take_qa.SAMPLE_RATE):int(end * take_qa.SAMPLE_RATE)]
step = int(take_qa.SAMPLE_RATE * take_qa.WINDOW)
count = samples.size // step
if count:
levels = [_db(np.sqrt(np.mean(samples[i * step:(i + 1) * step] ** 2))) for i in range(count)]
moments = [start + i * take_qa.WINDOW for i in range(count)]
if any(level > take_qa.SPEECH_DB for level in levels):
gaps = [(a, b) for a, b in _runs([level < take_qa.GAP_DB for level in levels], moments, take_qa.GAP_SECONDS)
if a > start + take_qa.GAP_EDGE and b < moments[-1] - take_qa.GAP_EDGE]
if gaps:
problems.append('silent gap at %s' % ', '.join('%.1f-%.1f s' % g for g in gaps))
return dict(problems=problems, notes=notes, ok=not problems)
def _side(samples, at, span, before):
rate = take_qa.SAMPLE_RATE
n = max(1, int(span * rate))
cut = min(samples.size, max(0, int(round(at * rate))))
return samples[max(0, cut - n):cut] if before else samples[cut:cut + n]
def _seam(a, a_end, b, b_start, span):
left, right = _side(a, a_end, span, True), _side(b, b_start, span, False)
if not left.size or not right.size:
return dict(level_db=0.0, click=0.0, in_speech=False, penalty=0.0)
levels = [_db(np.sqrt(np.mean(x ** 2))) for x in (left, right)]
level_db = abs(levels[0] - levels[1])
click = float(abs(right[0] - left[-1]))
in_speech = any(level > take_qa.SPEECH_DB for level in levels)
base = 0.6 * min(1.0, level_db / SEAM_LEVEL_DB) + 0.4 * min(1.0, click / SEAM_CLICK)
penalty = base + SEAM_SPEECH if in_speech else base * SEAM_QUIET
return dict(level_db=round(level_db, 2), click=round(click, 4), in_speech=in_speech,
penalty=round(min(1.0, penalty), 4))
def seam(a_path, a_end, b_path, b_start, *, span=0.1):
"""How a join sounds: level step over `span` s each side, sample jump, whether it falls in speech, penalty 0..1."""
return _seam(_samples(a_path), a_end, _samples(b_path), b_start, span)
def _whole(candidates):
best = min(candidates, key=lambda c: (-sum(c.get('scores') or [0]), c['take']))
return dict(pieces=[dict(take=best['take'], start=0.0, end=best['seconds'])],
score=round(float(sum(best.get('scores') or [0])), 6), fallback=True)
def stitch(candidates, *, seam_weight=0.5):
"""Choose a take for every segment, maximizing segment scores minus seam_weight * 10 * seam penalties.
A join between different takes inside speech is not allowed. Falls back to the best whole take when cut counts
differ, a segment has no usable candidate or no allowed sequence of takes exists."""
if not candidates:
return dict(pieces=[], score=0.0, fallback=True)
candidates = sorted(candidates, key=lambda c: c['take'])
count = len(candidates[0]['cuts']) + 1
if any(len(c['cuts']) + 1 != count or len(c['scores']) != count or len(c['qa']) != count for c in candidates):
return _whole(candidates)
bounds = {c['take']: [0.0] + [float(t) for t in c['cuts']] + [float(c['seconds'])] for c in candidates}
if any(b <= a for row in bounds.values() for a, b in zip(row, row[1:])):
return _whole(candidates) # a cut at or before the previous one leaves an empty piece
usable = [[c for c in candidates if c['qa'][k]] for k in range(count)]
if not all(usable):
return _whole(candidates)
audio, joins = {}, {}
def penalty(a, b, k):
"""The seam penalty, or None when the join is not allowed: a join between takes inside speech, or unreadable."""
key = (a['take'], b['take'], k)
if key not in joins:
try:
for c in (a, b):
if c['take'] not in audio:
audio[c['take']] = _samples(c['path'])
row = _seam(audio[a['take']], bounds[a['take']][k + 1], audio[b['take']], bounds[b['take']][k + 1], 0.1)
joins[key] = None if row['in_speech'] else row['penalty']
except Exception:
joins[key] = None
return joins[key]
# state: candidate take -> (score, takes so far); ties keep the smaller take sequence
state = {c['take']: (float(c['scores'][0]), [c['take']]) for c in usable[0]}
for k in range(1, count):
step = {}
for c in usable[k]:
options = []
for p in usable[k - 1]:
if p['take'] not in state:
continue
score, path = state[p['take']]
cost = 0.0
if p['take'] != c['take']:
seam_penalty = penalty(p, c, k - 1)
if seam_penalty is None:
continue
cost = seam_weight * 10 * seam_penalty
options.append((round(score + float(c['scores'][k]) - cost, 9), path + [c['take']]))
if options:
step[c['take']] = min(options, key=lambda o: (-o[0], o[1]))
if not step:
return _whole(candidates)
state = step
score, takes = min(state.values(), key=lambda o: (-o[0], o[1]))
pieces = []
for k, take in enumerate(takes):
start, end = bounds[take][k], bounds[take][k + 1]
if pieces and pieces[-1]['take'] == take:
pieces[-1]['end'] = end
else:
pieces.append(dict(take=take, start=start, end=end))
return dict(pieces=pieces, score=round(score, 6), fallback=False)