Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

fit: the 0.4 engine

Leonxlnx0f47d1dbd4cf
diff --git a/earshot/forge/fit/__init__.py b/earshot/forge/fit/__init__.py
index 7889ab5..c0c6141 100644
--- a/earshot/forge/fit/__init__.py
+++ b/earshot/forge/fit/__init__.py
@@ -1,7 +1,7 @@
"""earshot fit: translate a real recording into code (analysis by synthesis), then vary it.
from earshot.forge import fit
- m = fit.fit('click.wav') # kind='auto' | 'modal' | 'spectral' | 'sines' | 'hybrid'
+ m = fit.fit('click.wav') # kind='auto' | 'modal' | 'spectral' | 'sines' | 'gesture' | 'hybrid'
y = fit.render(m, seed=0) # the resynthesis (seeds change only noise / jitter); knobs vary it
from earshot.forge.fit.quality import quality
q = quality('click.wav', y) # mel dB, envelope r, tone dB, onset error, ... (clap=True: CLAP cos)
@@ -16,6 +16,8 @@ The fitters (one module each, plain-JSON models, deterministic per seed, pure nu
modal.py struck / ringing sounds -> strikes of damped sinusoids (ESPRIT + least squares) + band noise
sines.py tonal sounds -> tracked partials (McAulay-Quatieri / SMS) + strike atoms + 1/3-octave noise
spectral.py noisy sounds -> a time-varying 48-band noise gesture + stereo + texture + tonal ridges
+ gesture.py designed gestures (whooshes, swells, risers) -> tones (a sines / modal fit's partials) + air (noise)
+ space.py a stereo recording's decorrelated tail -> a seeded diffuse side layer (the chosen fit's post stage)
auto.py the engine: character -> candidates -> fit, render, score, keep the best; hybrids; morph
quality.py the metrics every fit is judged by (and the side-by-side spectrogram picture)
codegen.py model -> a forge recipe module (TARGET / SPACE / DEFAULTS / MODEL / render(p, r))
@@ -68,10 +70,11 @@ def _write_wav(path, y):
def run_fit(path, kind='auto', layers=None, output=None, out_dir=None, render_path=None, report=False, clap=False,
- describe=False, seed=0, target=None, fast=False, verbose=True, log=print, takes=False):
+ describe=False, seed=0, target=None, fast=False, verbose=True, log=print, takes=False, bed=True):
"""the `earshot fit <file>` pipeline as a function: fit, write the recipe / WAV / PNG / model JSON, measure.
- takes=True scores the candidates on seeds 0, 1, 2 (auto.TAKE_SEEDS), not seed 0 alone. Returns the result dict
- (paths, quality, floor, chosen kind, timings)."""
+ takes=True scores the candidates on seeds 0, 1, 2 (auto.TAKE_SEEDS), not seed 0 alone. bed=False fits a
+ recording with a bed under it as it is (auto.fit strips the bed by default). Returns the result dict (paths,
+ quality, floor, chosen kind, timings)."""
import time
import numpy as np
@@ -88,7 +91,7 @@ def run_fit(path, kind='auto', layers=None, output=None, out_dir=None, render_pa
t0 = time.time()
X = Q.load(path)
m = auto.fit(X, kind=kind, layers=layers, clap=clap, fast=fast, name=os.path.basename(path),
- verbose=verbose and kind == 'auto', seeds=auto.TAKE_SEEDS if takes else (0,))
+ verbose=verbose and kind == 'auto', seeds=auto.TAKE_SEEDS if takes else (0,), bed=bed)
t_fit = time.time() - t0
codegen.write_recipe(P['recipe'], m, name=P['name'], source=path, target=target)
rec = codegen.load_recipe(P['recipe'])
@@ -165,7 +168,7 @@ def add_cli(sub):
p.add_argument('file', nargs='?', help='the recording to fit')
p.add_argument('-o', '--output', help='recipe path (.py); the WAV / PNG / model JSON go next to it')
p.add_argument('--dir', help='output folder when -o is not given (default: current folder)')
- p.add_argument('--kind', default='auto', choices=('auto', 'modal', 'spectral', 'sines', 'hybrid'))
+ p.add_argument('--kind', default='auto', choices=('auto', 'modal', 'spectral', 'sines', 'gesture', 'hybrid'))
p.add_argument('--layers', help='hybrid layers, e.g. sines,modal,spectral (default modal,spectral)')
p.add_argument('--render', help='resynthesis WAV path (default next to the recipe)')
p.add_argument('--report', action='store_true',
@@ -177,6 +180,9 @@ def add_cli(sub):
p.add_argument('--fast', action='store_true', help='cheaper fitter settings (previews)')
p.add_argument('--takes', action='store_true',
help='a fit meant to make takes: score every candidate on seeds 0, 1, 2 (not seed 0 alone)')
+ p.add_argument('--keep-bed', action='store_true',
+ help='fit a recording with a bed under it (music in its pre-roll) as it is; by default the bed is '
+ 'stripped and the sound alone is fitted')
p.add_argument('--morph', nargs=2, metavar=('A', 'B'),
help='fit two recordings with one kind and write a morph recipe')
p.add_argument('--amount', type=float, default=0.5, help='with --morph: 0 = A, 1 = B (default 0.5)')
@@ -219,7 +225,7 @@ def add_cli(sub):
return 2
layers = tuple(a.layers.split(',')) if a.layers else None
r = run_fit(a.file, a.kind, layers, a.output, a.dir, a.render, a.report, a.clap, a.describe, a.seed, a.target,
- a.fast, verbose=not a.quiet and not a.json, takes=a.takes)
+ a.fast, verbose=not a.quiet and not a.json, takes=a.takes, bed=not a.keep_bed)
lines = [f"{os.path.basename(a.file)} -> {r['kind']} model" + (f" (auto chose {r['chosen']})"
if a.kind == 'auto' else '')
+ f" in {r['seconds']['fit']:.1f} s",
diff --git a/earshot/forge/fit/auto.py b/earshot/forge/fit/auto.py
index 30dff92..f3231a1 100644
--- a/earshot/forge/fit/auto.py
+++ b/earshot/forge/fit/auto.py
@@ -20,20 +20,37 @@ Kinds (each a fitter in this package, all analysis by synthesis, all plain JSON
'morph' moves w), so a variation sits between two recordings instead of copying one.
kind='auto' (fit):
+ 0. a recording with a bed under it (bed_of: a steady pre-roll 12-40 dB under the sound, a film's music) is
+ stripped (strip_bed) and the sound alone is fitted, from its onset; post['bed'] keeps the bed's spectrum and
+ render() puts a steady noise with it back under the sound (bed=False: the sound alone, for a kit or a bank).
+ Analysis by synthesis decides whether it was a bed (BED_CHECK): the winning candidate's kind is fitted on the
+ recording as it is too, and the bed is dropped when that scores lower on the same seeds (a whoosh's own steady
+ approach is not a bed).
1. character() measures attack, decay, crest, tonal share (energy in spectral peaks 12 dB over the local median),
- partial persistence, the glide of the strongest partial and flatness, and names a class (glide, ringing,
- struck, gesture, mixed); plan() orders the candidates by class. Under 1 s every candidate is fitted (each fit
- takes 0.5-4 s); longer sounds get the class's four.
+ partial persistence, the glide of the strongest partial (2 semitones over 15 ms count) and flatness, and names
+ a class (glide, ringing, struck, gesture, mixed); plan() orders the candidates by class. Under 1 s every
+ candidate is fitted (each fit takes 0.5-4 s); longer sounds get the class's four.
2. every candidate is fitted, then its post stage (fit_post) is measured from two renders: a smooth third-octave
EQ onto the original's balance, the level, and the attack: the original's 0.25 ms envelope before the onset
(renders may not stand above it: band noise and window smear leak energy before a strike) and over the first
- 6-25 ms after a sharp onset (renders follow its contour within +-9 dB). render() always applies it.
+ 6-25 ms after a sharp onset (renders follow its contour within +-9 dB; for a noise model high-passed, and kept
+ only when it lowers the score); then the gate: a cut-only gain curve
+ where the renders stand over the original after the onset (gated pulses, hard stops, merged runs: modes and
+ partials cannot stop), kept when it lowers the score (with CLAP: and costs none). render() always applies the
+ post stage, then a fade-in and a peak limit at -1 dBFS (a gain).
3. each candidate is rendered (seed 0) and scored against the original (quality.score: mel dB, tone, envelope,
onset, excess / missing energy, + CLAP with clap=True); the lowest score wins. Hybrids reuse the single-kind
fits as their first layer. The model gets a 'fit_engine' entry: the character, every candidate's metrics, the
choice. seeds=TAKE_SEEDS (`earshot fit --takes`) scores the mean over seeds 0, 1, 2 instead: a candidate with
noise in it can win on one lucky seed-0 draw and then make weaker takes (on 10 of the 12 validation fits the
- chosen candidate's seed-0 score sat 0.1-2.4 under its mean over seeds 0-7).
+ chosen candidate's seed-0 score sat 0.1-2.4 under its mean over seeds 0-7). With seed 0 alone the best three
+ are scored on two more seeds (CONFIRM, CONFIRM_SEEDS) and the choice is made on the mean; a candidate whose CLAP
+ is CLAP_MARGIN higher on seed 0 and on that mean, at most CLAP_TOL worse on the score and CLAP_ENV_TOL on the
+ envelope, is chosen instead (the model-free terms must not decide a near-tie against a clearly closer sound;
+ each confirmed row keeps 'score2' and 'clap2', the one chosen so 'chosen_by': 'clap'). A modal / hybrid
+ winner gets 'vary_scale' 0.5 (TAKE_VARY): its takes at vary 0.3 jitter as 0.15 did. A stereo recording with a
+ decorrelated tail gives the chosen model fit.space's diffuse side layer (post['space']: a seeded room in the
+ side, so the mono sum and every mono measure stay as they were).
Pure numpy / scipy (CLAP only with clap=True). Deterministic: render(model, seed, knobs) is a pure function.
"""
import copy
@@ -47,7 +64,7 @@ from . import quality as Q
SR = 48000
TAKE_DC_HZ = 8.0 # a take (render(take=True)) is high-passed here: no DC offset, no infrasound
TAKE_SEEDS = (0, 1, 2) # fit(seeds=TAKE_SEEDS): score candidates on these seeds (a fit meant to make takes)
-KINDS = ('modal', 'spectral', 'sines', 'hybrid')
+KINDS = ('modal', 'spectral', 'sines', 'hybrid', 'gesture')
ROLE = {'sines': 'tone', 'modal': 'body', 'spectral': 'air'}
HYBRIDS = {'modal+spectral': ('modal', 'spectral'), 'sines+spectral': ('sines', 'spectral'),
'sines+modal+spectral': ('sines', 'modal', 'spectral')}
@@ -63,7 +80,7 @@ KNOBS = {
'noise': ((-6.0, 4.0), 0.0, 'dB on the noise bands'),
'width': ((0.5, 1.5), 1.0, 'x the side signal'),
'room': ((0.0, 0.25), 0.0, 'wet of a small stochastic room'),
- 'vary': ((0.0, 0.6), 0.0, 'seeded jitter of modes, timing and noise: new takes of the same object'),
+ 'vary': ((0.0, 0.6), 0.0, 'seeded jitter of the modes and the noise: new takes of the same object'),
'spacing': ((0.8, 1.25, 'log'), 1.0, 'x the time between strikes'),
},
'sines': {
@@ -77,6 +94,7 @@ KNOBS = {
'transient': ((0.5, 2.0, 'log'), 1.0, 'gain on the strike noise (first 15 ms)'),
'vary': ((0.0, 0.6), 0.0, 'seeded per-partial detune / level / decay jitter'),
'inharm': ((-0.03, 0.03), 0.0, 'stretch of the partial ratios'),
+ 'width': ((0.5, 1.5), 1.0, 'x the fitted side'),
},
'spectral': {
'length': ((0.8, 1.25, 'log'), 1.0, 'time-stretch of the gesture'),
@@ -93,7 +111,7 @@ KNOBS = {
'decay': ((0.7, 1.43, 'log'), 1.0, 'decay factor of the tonal / body layers'),
'bright': ((-3.0, 3.0), 0.0, 'dB / octave on every layer'),
'vary': ((0.0, 0.6), 0.0, 'seeded jitter of the tonal / body layers'),
- 'width': ((0.5, 1.5), 1.0, 'stereo width of the air layer'),
+ 'width': ((0.5, 1.5), 1.0, 'stereo width (x every layer\'s side)'),
'tone_db': ((-6.0, 6.0), 0.0, 'level of the tonal (sines) layer'),
'body_db': ((-6.0, 6.0), 0.0, 'level of the body (modal) layer'),
'air_db': ((-6.0, 6.0), 0.0, 'level of the air (spectral) layer'),
@@ -102,16 +120,16 @@ KNOBS = {
def _fitters():
- from . import modal, sines, spectral
- return {'modal': modal, 'sines': sines, 'spectral': spectral}
+ from . import gesture, modal, sines, spectral
+ return {'modal': modal, 'sines': sines, 'spectral': spectral, 'gesture': gesture}
def kind_of(model):
- """'modal' | 'sines' | 'spectral' | 'hybrid' | 'morph'"""
+ """'modal' | 'sines' | 'spectral' | 'gesture' | 'hybrid' | 'morph'"""
k = model.get('kind') if isinstance(model, dict) else None
if k == 'spectral_noise':
return 'spectral'
- if k not in ('modal', 'sines', 'hybrid', 'morph'):
+ if k not in ('modal', 'sines', 'gesture', 'hybrid', 'morph'):
raise ValueError(f'not a fit model (kind {k!r})')
return k
@@ -127,6 +145,9 @@ def knobs_of(model):
roles = {L['role'] for L in model['layers']}
return {n: v for n, v in KNOBS['hybrid'].items()
if not n.endswith('_db') or n[:-3] in roles}
+ if k == 'gesture': # fit/gesture.py's own table (the same format)
+ kn = _fitters()['gesture'].KNOBS
+ return {n: v for n, v in kn.items() if model.get('tones') or n not in ('tone_db', 'vary')}
return KNOBS[k]
@@ -138,6 +159,8 @@ def channels_of(x_or_model):
return max(channels_of(m['a']), channels_of(m['b']))
if k == 'spectral':
return 2 if m.get('stereo') else int(m.get('channels', 1))
+ if k == 'gesture':
+ return int(m.get('channels', 1))
if k == 'sines':
return int(m.get('channels', 2 if m.get('width', -120.0) > -50.0 else 1))
return int(m.get('channels', 1))
@@ -146,6 +169,58 @@ def channels_of(x_or_model):
# ================================================================================================ character
+GLIDE_OCT = 2.0 / 12 # character(): a glide is the strongest partial moving this much (2 semitones) ...
+GLIDE_TRACK_MS = 15.0 # ... along a track this long (before 0.4: 0.3 octave over 25 ms on 43 ms frames, which
+ # never fired on the films' 2-6 semitone pops and dings)
+GLIDE_NFFT, GLIDE_HOP = 1024, 128 # the glide's own analysis: 21 ms frames every 2.7 ms
+
+
+def _glide(m):
+ """(octaves, track ms): the strongest partial's frequency over the frames where it stands out (a peak 15 dB over
+ the local median, within 15 dB of its loudest), along the longest track that moves < 2 semitones per frame; the
+ glide is the median of the track's last fifth minus its first fifth (0 for a track under 4 frames)"""
+ from scipy.ndimage import maximum_filter1d, median_filter
+ n, nfft, hop = len(m), GLIDE_NFFT, GLIDE_HOP
+ T = max(1, (n + hop - 1) // hop)
+ xp = np.zeros(T * hop + nfft)
+ xp[nfft // 2:nfft // 2 + n] = m
+ fr = np.lib.stride_tricks.sliding_window_view(xp, nfft)[::hop][:T]
+ P = np.abs(np.fft.rfft(fr * np.hanning(nfft + 2)[1:-1], axis=1)) ** 2
+ f = np.fft.rfftfreq(nfft, 1 / SR)
+ band = (f >= 60) & (f <= 16000)
+ P, fb = P[:, band], f[band]
+ fe = P.sum(1)
+ if fe.max() <= 0:
+ return 0.0, 0.0
+ L = 10 * np.log10(P + 1e-20)
+ med = median_filter(L, size=(1, 11), mode='nearest')
+ peaks = (L >= maximum_filter1d(L, 3, axis=1)) & (L - med > 15) & (L > L.max(1, keepdims=True) - 50)
+ tr = []
+ for t in np.nonzero(fe > fe.max() * 10 ** -3.5)[0]:
+ if not peaks[t].any():
+ continue
+ j = int(np.argmax(np.where(peaks[t], L[t], -1e9)))
+ a, b, c = L[t, max(j - 1, 0)], L[t, j], L[t, min(j + 1, len(fb) - 1)]
+ d = 0.5 * (a - c) / (a - 2 * b + c) if (a - 2 * b + c) < 0 else 0.0
+ tr.append((t * hop / SR, np.log2(max(fb[j] + d * (fb[1] - fb[0]), 20.0)), b))
+ if len(tr) < 4:
+ return 0.0, 0.0
+ tt, lf, lv = map(np.asarray, zip(*tr))
+ strong = lv > lv.max() - 15
+ tt, lf = tt[strong], lf[strong]
+ if len(tt) < 4:
+ return 0.0, 0.0
+ cut = np.nonzero((np.abs(np.diff(lf)) >= 1 / 6) | (np.diff(tt) > 2.5 * hop / SR))[0] + 1 # track breaks
+ segs = np.split(np.arange(len(tt)), cut)
+ s = max(segs, key=len)
+ st, sl = tt[s], lf[s]
+ track_ms = float((st[-1] - st[0]) * 1000) if len(st) > 1 else 0.0
+ if len(st) < 4:
+ return 0.0, round(track_ms, 1)
+ q = max(1, len(st) // 5)
+ return float(np.median(sl[-q:]) - np.median(sl[:q])), track_ms
+
+
def character(x):
"""measurements that decide which fitter suits a sound (mono sum): duration, attack / t40 (ms), crest (dB),
tonal share (0..1, energy in peaks 12 dB over the local median), persistence (0..1, peaks that continue into
@@ -175,7 +250,6 @@ def character(x):
f = np.fft.rfftfreq(nfft, 1 / SR)
band = (f >= 60) & (f <= 16000)
P = P[:, band]
- fb = f[band]
fe = P.sum(1)
af = fe > fe.max() * 10 ** -3.5
L = 10 * np.log10(P + 1e-20)
@@ -197,33 +271,7 @@ def character(x):
num += float((pw * mask[b][peaks[a]]).sum())
den += float(pw.sum())
persist = num / den if den > 0 else 0.0
- # glide: the strongest partial's frequency over the frames where it stands out
- tr = []
- for t in idx:
- if not peaks[t].any():
- continue
- j = int(np.argmax(np.where(peaks[t], L[t], -1e9)))
- if L[t, j] - med[t, j] < 15:
- continue
- a, b, c = L[t, max(j - 1, 0)], L[t, j], L[t, min(j + 1, len(fb) - 1)]
- d = 0.5 * (a - c) / (a - 2 * b + c) if (a - 2 * b + c) < 0 else 0.0
- tr.append((t * hop / SR, np.log2(max(fb[j] + d * (fb[1] - fb[0]), 20.0)), b))
- glide = 0.0
- track_ms = 0.0
- if len(tr) >= 4:
- tt, lf, lv = map(np.asarray, zip(*tr))
- strong = lv > lv.max() - 15
- tt, lf = tt[strong], lf[strong]
- if len(tt) >= 4:
- jumps = np.abs(np.diff(lf))
- run = np.concatenate([[True], jumps < 1 / 6]) # a continuous track: < 2 semitones per frame
- k0 = int(np.argmax(run))
- seg = slice(k0, k0 + max(1, int(np.argmin(run[k0:])) if not run[k0:].all() else len(run) - k0))
- st, sl = tt[seg], lf[seg]
- track_ms = float((st[-1] - st[0]) * 1000) if len(st) > 1 else 0.0
- if len(st) >= 4:
- q = max(1, len(st) // 5)
- glide = float(np.median(sl[-q:]) - np.median(sl[:q]))
+ glide, track_ms = _glide(m)
from .modal import onsets
strikes = len(onsets(m))
c = {'dur_s': round(n / SR, 4), 'attack_ms': float(pk - on), 'rise_db': round(rise, 1), 't40_ms': t40,
@@ -234,7 +282,8 @@ def character(x):
c['impulsive'] = c['attack_ms'] <= 40.0 and (c['t40_ms'] <= 400.0 or c['crest_db'] >= 18.0)
c['tonal'] = c['tonal_share'] >= 0.45 and c['persistence'] >= 0.5
c['noisy'] = c['tonal_share'] < 0.25
- c['gliding'] = abs(c['glide_oct']) >= 0.3 and c['track_ms'] >= 25 and c['tonal_share'] >= 0.25
+ c['gliding'] = (abs(c['glide_oct']) >= GLIDE_OCT and c['track_ms'] >= GLIDE_TRACK_MS
+ and c['tonal_share'] >= 0.25)
if c['gliding']:
c['class'] = 'glide'
elif c['tonal_share'] >= 0.8 and c['persistence'] >= 0.9 and c['t40_ms'] > 300:
@@ -248,28 +297,161 @@ def character(x):
return c
-PLANS = { # class -> candidates, expected best first
+PLANS = { # class -> candidates, expected best first (a sound under 1 s gets every candidate in ALL after these)
'glide': ['sines', 'hybrid:sines+modal+spectral', 'hybrid:sines+spectral', 'modal'],
'ringing': ['sines', 'modal', 'hybrid:sines+spectral', 'spectral'],
'struck': ['modal', 'hybrid:sines+modal+spectral', 'hybrid:modal+spectral', 'sines'],
'gesture': ['spectral', 'hybrid:sines+spectral', 'modal', 'sines'],
'mixed': ['modal', 'spectral', 'hybrid:modal+spectral', 'sines'],
}
-ALL = ('modal', 'sines', 'spectral', 'hybrid:modal+spectral', 'hybrid:sines+spectral', 'hybrid:sines+modal+spectral')
+LONG_PLANS = { # a sound of 1 s or more: the class's four only (0.4: the gesture model in place of the spectral one,
+ 'glide': ['sines', 'gesture', 'hybrid:sines+modal+spectral', 'hybrid:sines+spectral'], # which won none of the
+ 'ringing': ['sines', 'gesture', 'modal', 'hybrid:sines+spectral'], # films' whooshes, swells,
+ 'struck': ['modal', 'sines', 'hybrid:modal+spectral', 'spectral'], # risers, camera moves)
+ 'gesture': ['sines', 'gesture', 'modal', 'hybrid:sines+spectral'],
+ 'mixed': ['modal', 'sines', 'gesture', 'hybrid:modal+spectral'],
+}
+ALL = ('modal', 'sines', 'gesture', 'spectral', 'hybrid:modal+spectral', 'hybrid:sines+spectral',
+ 'hybrid:sines+modal+spectral') # 'gesture' reuses the sines candidate's fit: it always comes after 'sines'
def plan(c, max_candidates=None):
"""ordered candidate kinds for a character (first = the expected best); hybrids as 'hybrid:<a>+<b>[+<c>]'.
The measured quality decides among them; the class orders them and prunes the slow fits of long sounds: under
- 1 s every candidate is tried (each fit takes 0.5-4 s), longer sounds the class's four (a long struck sound:
- modal, sines, modal + spectral, spectral; the three-layer hybrid alone took 147 s on a 4.5 s impact)."""
- order = list(PLANS[c['class']])
- if c['dur_s'] >= 1.0 and c['class'] == 'struck':
- order = ['modal', 'sines', 'hybrid:modal+spectral', 'spectral']
+ 1 s every candidate is tried (each fit takes 0.5-4 s; the gesture 3-60 s), longer sounds the class's four
+ (LONG_PLANS: a long struck sound: modal, sines, modal + spectral, spectral; the three-layer hybrid alone took
+ 147 s on a 4.5 s impact). A long gesture, mixed, glide or ringing sound gets the gesture model (fit/gesture.py:
+ tones + air) second, in place of the spectral one (0.4). 'gesture' always comes after 'sines' (it takes the sines
+ candidate's partials as its tones)."""
+ long_ = c['dur_s'] >= 1.0
+ order = list((LONG_PLANS if long_ else PLANS)[c['class']])
order += [k for k in ALL if k not in order]
if max_candidates is None:
- max_candidates = len(ALL) if c['dur_s'] < 1.0 else 4
- return order[:max_candidates]
+ max_candidates = 4 if long_ else len(ALL)
+ order = order[:max_candidates]
+ if 'gesture' in order and 'sines' in order and order.index('gesture') < order.index('sines'):
+ order.remove('gesture')
+ order.insert(order.index('sines') + 1, 'gesture')
+ return order
+
+
+# ================================================================================================ bed
+BED_DB = -40.0 # a pre-roll this loud re the loudest 50 ms (or louder) is a bed under the sound (the film's
+ # music, a room): fit() strips it; a quieter one is the recording's own floor and stays
+BED_MIN_S = 0.03 # the pre-roll used (3 ms after the clip's start to 2 ms before the first onset): this long
+BED_PRE_MAX_S = 0.12 # ... and the onset within this (a later one is a slow sound's peak, not its start)
+BED_MAX_DB = -12.0 # ... but not louder than this (a pre-roll that loud is the sound's own start, or a bed
+ # the sound cannot be told from)
+BED_STEADY_DB = 6.0 # ... and steady: its quarters within this, the last at most half of it over the first (a
+ # decaying tail, a swell's start is not a bed)
+BED_RISE_DB = 10.0 # ... and the first 10 ms from the onset this much louder (the sound stands out of it)
+BED_NFFT = 1024 # the bed's spectrum: STFT power per bin (Hann, hop 1/4), the median over the pre-roll (47 Hz
+ # bins: a music bed's partials stand apart)
+BED_OVER = 2.0 # strip_bed subtracts this x the bed's power from every cell ...
+BED_FLOOR_DB = -25.0 # ... keeping at least this much of it (no musical-noise holes)
+BED_LINE_DB = 10.0 # bed_noise: a bin this far over the median of its 15 neighbours is a steady partial (a sine)
+
+
+def _stft(m, nfft=BED_NFFT, pad=True):
+ from scipy import signal
+ return signal.stft(m, SR, window='hann', nperseg=nfft, noverlap=nfft - nfft // 4,
+ **({} if pad else {'boundary': None, 'padded': False}))[2]
+
+
+def bed_of(x):
+ """the bed under a recording, from its pre-roll (the films' clips carry 60 ms before the onset): {'t_on' s (the
+ onset: the first 1 ms frame within 30 dB of the loudest that stands BED_RISE_DB over the median of the frames
+ before it, refined to a modal.onsets strike within 3 ms), 'level_db' (the pre-roll re the loudest 50 ms), 'n'
+ (the recording's length, samples), 'nfft', 'psd_db' (the median STFT power per bin over the pre-roll,
+ scipy.signal.stft scaling)}, or None: no such onset between BED_MIN_S and BED_PRE_MAX_S, a pre-roll quieter than
+ BED_DB (the recording's own floor) or louder than BED_MAX_DB, or not steady or rising (BED_STEADY_DB)"""
+ from .modal import onsets
+ m = Q.mono(Q.load(x))
+ if len(m) < int(0.03 * SR) or not np.any(m):
+ return None
+ e = _env_frames(m, 0.001) # 1 ms frames: the onset is the first one BED_RISE_DB over the
+ top = e.max() # median of those before it (from 3 ms: the edit fade-in)
+ k0 = int(round((BED_MIN_S + 0.005) / 0.001))
+ k = next((k for k in range(k0, min(len(e), int(BED_PRE_MAX_S / 0.001))) if e[k] >= top - 30
+ and e[k] >= np.median(e[3:k - 2]) + BED_RISE_DB), None)
+ if k is None:
+ return None
+ near = [int(o) for o in onsets(m) if abs(o - k * 48) <= 144] # a strike onset within 3 ms refines it
+ on = min(near, key=lambda o: abs(o - k * 48)) if near else k * 48
+ a, b = int(0.003 * SR), on - int(0.002 * SR)
+ if b - a < int(BED_MIN_S * SR):
+ return None
+ pre = m[a:b]
+ db = lambda v: 10 * np.log10(np.mean(v ** 2) + 1e-20)
+ lv = db(pre) - Q.peak_rms_db(m)
+ q = [db(v) for v in np.array_split(pre, 4)]
+ if (not BED_DB <= lv <= BED_MAX_DB or max(q) - min(q) > BED_STEADY_DB
+ or q[-1] - q[0] > BED_STEADY_DB / 2): # rising: a swell's or a whoosh's own start, not a bed
+ return None
+ P = np.median(np.abs(_stft(pre, pad=False)) ** 2, axis=1) # whole frames only (no zero-padded edges)
+ return {'t_on': round(on / SR, 6), 'level_db': round(float(lv), 2), 'n': len(m), 'nfft': BED_NFFT,
+ 'psd_db': [round(float(v), 1) for v in 10 * np.log10(P + 1e-30)]}
+
+
+def strip_bed(x, bed):
+ """x without its bed (bed_of): every STFT cell keeps sqrt(max(1 - BED_OVER x bed power / its power, BED_FLOOR))
+ of itself (the gain from the mono sum, smoothed over 3 frames, the same on both channels), and the pre-roll is
+ silenced up to 0.5 ms before the onset (a 1.5 ms raised-cosine fade-in): the sound alone, from its onset"""
+ from scipy import signal
+ X = Q.load(x)
+ two = X.ndim == 2
+ X2 = X if two else X[:, None]
+ n, nfft = len(X2), int(bed['nfft'])
+ B = 10 ** (np.asarray(bed['psd_db'], float) / 10)[:, None]
+ Zm = _stft(X2.mean(1), nfft)
+ G = np.sqrt(np.maximum(1.0 - BED_OVER * B / (np.abs(Zm) ** 2 + 1e-30), 10 ** (BED_FLOOR_DB / 10)))
+ G = signal.lfilter(np.ones(3) / 3, [1.0], np.pad(G, ((0, 0), (1, 1)), mode='edge'), axis=1)[:, 2:]
+ out = np.zeros_like(X2)
+ for c in range(X2.shape[1]):
+ y = signal.istft(_stft(X2[:, c], nfft) * G, SR, window='hann', nperseg=nfft, noverlap=nfft - nfft // 4)[1]
+ out[:, c] = np.pad(y, (0, max(0, n - len(y))))[:n]
+ on = int(round(float(bed['t_on']) * SR))
+ e, f = max(0, on - int(0.0005 * SR)), int(0.0015 * SR)
+ ramp = np.zeros(n)
+ ramp[e:] = 1.0
+ s = max(0, e - f)
+ ramp[s:e] = 0.5 - 0.5 * np.cos(np.pi * np.arange(e - s) / max(e - s, 1))
+ out *= ramp[:, None]
+ return out if two else out[:, 0]
+
+
+def bed_noise(bed, n, seed=0):
+ """(n,) the bed layer of a fit of a stripped recording: a steady noise with the bed's spectrum, its lines
+ (BED_LINE_DB) as steady sines, over the recording's length (bed['n'] samples; a 2 ms fade-in, a 10 ms fade-out,
+ like the films' clips), silent after it"""
+ from scipy import signal
+ from scipy.ndimage import median_filter
+ nfft = int(bed['nfft'])
+ nb = min(int(n), int(bed.get('n', n)))
+ if nb <= 0:
+ return np.zeros(int(n))
+ w = np.hanning(nfft + 1)[:-1]
+ c = (w ** 2).sum() / w.sum() ** 2 * np.log(2) # white noise's median STFT power per bin (scipy scaling)
+ L = np.asarray(bed['psd_db'], float)
+ med = median_filter(L, 15, mode='nearest')
+ B = 10 ** (L / 10)
+ r = np.random.default_rng([int(seed) & 0xFFFFFFFF, 77])
+ t = np.arange(nb) / SR
+ lines = np.zeros(nb) # a music bed's steady partials (a drone, a held chord) as sines:
+ for j in range(1, len(L) - 1): # as noise they would flicker 5-10 dB where the bed is steady
+ if L[j] - med[j] >= BED_LINE_DB and L[j] >= L[j - 1] and L[j] >= L[j + 1]:
+ a, b_, c_ = L[j - 1], L[j], L[j + 1]
+ d = 0.5 * (a - c_) / (a - 2 * b_ + c_) if (a - 2 * b_ + c_) < 0 else 0.0
+ amp = np.sqrt(4.0 * B[j - 1:j + 2].sum() / 1.5) # Hann: the peak bin A^2/4, its neighbours A^2/16
+ lines += amp * np.cos(2 * np.pi * (j + d) * SR / nfft * t + r.uniform(0, 2 * np.pi))
+ B[j - 1:j + 2] = 10 ** (med[j - 1:j + 2] / 10)
+ Z = _stft(r.standard_normal(nb + nfft), nfft) * np.sqrt(B[:, None] / c)
+ y = signal.istft(Z, SR, window='hann', nperseg=nfft, noverlap=nfft - nfft // 4)[1]
+ y = np.pad(y, (0, max(0, nb - len(y))))[:nb] + lines
+ fi, fo = min(nb, int(0.002 * SR)), min(nb, int(0.01 * SR))
+ y[:fi] *= np.linspace(0.0, 1.0, fi)
+ y[nb - fo:] *= np.linspace(1.0, 0.0, fo)
+ return np.pad(y, (0, int(n) - nb))
# ================================================================================================ fitting
@@ -291,8 +473,12 @@ def fit_kind(x, kind, fast=False, name=None, **opts):
kw.update(opts)
m = F['spectral'].fit(X, name=name, **kw)
m['channels'] = channels_of(X)
+ elif kind == 'gesture': # opts: tones= a sines fit of the same x to take the tones from
+ kw = dict(fast=True) if fast else {}
+ kw.update(opts)
+ m = F['gesture'].fit(X, name=name, **kw)
else:
- raise ValueError(f'unknown fitter {kind!r} (modal, sines, spectral; hybrids via fit_hybrid)')
+ raise ValueError(f'unknown fitter {kind!r} (modal, sines, spectral, gesture; hybrids via fit_hybrid)')
if name:
m['source'] = name
return m
@@ -306,6 +492,7 @@ def deterministic(model):
L = float(m['length_s'])
m['noise'] = dict(m['noise'], env=[[[0.0, -240.0], [L, -240.0]] for _ in m['noise']['env']])
m['side'] = None
+ m.pop('side_t', None)
m['channels'] = 1
elif k == 'sines':
m['residual'] = {'bands': [1000.0], 'times': [0.0, float(m['duration'])], 'db': [[-150.0, -150.0]]}
@@ -332,6 +519,29 @@ def _fit_n(X, y):
return out
+HYBRID_SIDE_SEEDS = 8 # fit_hybrid: stereo renders of a tone layer with a side tried for the residual's side
+
+
+def _layer_take(resid, X, m):
+ """what a deterministic hybrid layer takes out of the residual: its render (seed 0) as (len(X), channels of X).
+ A sines layer with a fitted side (0.4) renders that side in the hybrid (_render_hybrid), so for a stereo recording
+ the residual leaves it out too, or the air layer models the tones' side again as noise and the side is counted
+ twice. The side's quadrature part has a random sign per seed (shaped_side): of HYBRID_SIDE_SEEDS stereo renders
+ (the mid the same in all) the one that leaves the least side in the residual is taken, and the mono render when
+ none leaves less than it (the tone layer's side does not match the recording's: nothing to take out)"""
+ y = _fit_n(X, _render(m, seed=0, final=False))
+ if X.shape[1] != 2 or kind_of(m) != 'sines' or not m.get('side'):
+ return y
+ left = lambda r: float(np.sum((resid[:, 0] - r[:, 0] - resid[:, 1] + r[:, 1]) ** 2))
+ best, e = y, left(y)
+ for s in range(HYBRID_SIDE_SEEDS):
+ r = _fit_n(X, _render(m, seed=s, final=False, stereo=True))
+ es = left(r)
+ if es < e:
+ best, e = r, es
+ return best
+
+
def fit_hybrid(x, layers=('modal', 'spectral'), fast=False, base=None, name=None):
"""layers fitted in turn on what the previous ones leave (module doc). base: {kind: model} single-kind fits of
the same x to reuse for the first layer"""
@@ -351,7 +561,7 @@ def fit_hybrid(x, layers=('modal', 'spectral'), fast=False, base=None, name=None
m = fit_kind(resid, k, fast=fast)
if not last:
m = deterministic(m)
- resid = resid - _fit_n(X, render(m, seed=0))
+ resid = resid - _layer_take(resid, X, m)
out.append({'role': ROLE[k], 'gain_db': 0.0, 'model': m})
return {'kind': 'hybrid', 'version': VERSION, 'sr': SR, 'length_s': round(len(X) / SR, 6),
'channels': channels_of(X), 'layers': out, 'source': name}
@@ -361,14 +571,42 @@ def _cand_fit(X, cand, fast, base, name):
if cand.startswith('hybrid'):
layers = HYBRIDS[cand.split(':', 1)[1]] if ':' in cand else ('modal', 'spectral')
return fit_hybrid(X, layers, fast=fast, base=base, name=name)
+ if cand == 'gesture': # the tones of the sines candidate, when there is one
+ return fit_kind(X, cand, fast=fast, name=name, tones=base.get('sines'))
m = fit_kind(X, cand, fast=fast, name=name)
base[cand] = m
return m
PRE_HOP = 0.00025 # s: the attack envelope grid of the post stage
+PRE_HP_HZ = 200.0 # a noise model's attack clamp (fit_post(noise=True), pre['hp_hz']): its correction is
+ # high-passed here (the clamp follows the noise's own 0.25 ms flicker: its +-9 dB steps
+ # spread energy into the empty low bands, a whoosh +8 dB at 31-99 Hz), and the stage is
+ # kept only when the clamp lowers the score (on a smooth whoosh it blunts the onset)
+NOISE_KINDS = ('spectral', 'gesture') # the kinds whose post stage is fitted with noise=True
ATTACK_S = 0.006 # s: the attack window after the onset whose contour the post stage follows, at least,
ATTACK_MAX_S = 0.025 # or up to 4 ms past the loudest frame within this much after the onset (a later main strike)
+GATE_HOP = 0.0005 # s: the gate's envelope grid (1 ms windows, half overlapping)
+GATE_TOL_DB = 4.0 # the gate cuts where the post-processed renders stand more than this over the original ...
+GATE_KEEP_DB = 1.0 # ... down to this much over it
+GATE_MIN_CUT_DB = 6.0 # a cut region is kept when it reaches at least this deep (shallower: the EQ's business)
+GATE_FALL_DB = 12.0 # ... and where the original lies this far under its loudest of the GATE_LOOK_S before
+GATE_LOOK_S = 0.02 # (a gap, a stop, a fast decay; not the random dips of a sustained noise)
+GATE_QUIET_DB = 40.0 # (or lies this far under its top: a long silence after a stop is still a gap)
+GATE_MIN_S = 0.003 # ... and that lasts this long (shorter notches are a noise's or a beat's flicker)
+GATE_DIP_DB = 6.0 # a run of pulses: between two of the original's pulses (peaks of its 5 ms envelope, 6 dB
+GATE_DIP_TOL_DB = 2.0 # prominence) the frames this far under the lower pulse are a dip, cut where the renders
+ # stand GATE_DIP_TOL_DB over the original (a smoother render merges the pulses)
+GATE_DEPTH_DB = -80.0 # the deepest cut (a hard stop into digital silence)
+GATE_EDIT_S = 0.012 # the clip's own edit fade-out (its last 10 ms): not a gate, the cut before it holds
+GATE_RDP_DB = 0.5 # breakpoint tolerance of the stored gain curve
+GATE_CLAP_TOL = 0.005 # a gate that lowers the score is still dropped when it costs more CLAP than this
+GATE_MAP_S = 0.002 # _gate_map: under a modal spacing the gate's last GATE_MAP_S before a strike (the cut's
+ # reopening, a frame before the original rises) moves with that strike, the cut before it
+ # with the strike before (held across a longer gap, cut off where a shorter one ends)
+GATE_SMOOTH_S = 0.001 # render: the gate's gain (linear) smoothed by a Hann window this long: its breakpoints sit
+ # 0.5 ms apart, and a 30-80 dB step between two of them put a kink into a loud tone or ring
+ # that the takes' audit heard as a click (a pop's gesture takes: 4 of 4 clicked at the cuts)
def _env_frames(m, hop):
@@ -377,7 +615,7 @@ def _env_frames(m, hop):
return 10 * np.log10((m[:nb * k].reshape(nb, k) ** 2).mean(1) + 1e-20) if nb else np.zeros(0)
-def fit_post(X, renders, iters=2, max_db=12.0, envelope=False):
+def fit_post(X, renders, iters=2, max_db=12.0, envelope=False, gate=True, judge=None, sharp=True, noise=False):
"""the post stage of a model, measured by analysis by synthesis (renders = a few seeds of the model):
eq_db a smooth third-octave EQ (minimum phase, forge.fx.eq_bands; 28 gains at tone.CEN) that moves the
renders' mean balance onto the original's
@@ -385,17 +623,34 @@ def fit_post(X, renders, iters=2, max_db=12.0, envelope=False):
pre the original's 0.25 ms envelope (dB) before its first onset (the first frame within 20 dB of the
loudest) and, for a sharp onset (>= 15 dB/ms, quality.attack), over the ATTACK_S after it (or to
4 ms past the loudest frame of the next ATTACK_MAX_S: a key's bottom-out after its first touch):
- {'t_on' s, 'db' [...], 'attack_db' [...]}. Band noise and window smear leak energy before a strike
- and soften it (a pre-echoed, blunt attack); _pre_clamp keeps the attack as clean as the recording's
+ {'t_on' s, 'db' [...], 'attack_db' [...], 'v': 2 (sharp=True)}. Band noise and window smear leak energy
+ before a strike and soften it (a pre-echoed, blunt attack); _pre_clamp keeps the attack as clean as
+ the recording's. noise=True (a NOISE_KINDS model; 0.4): 'hp_hz' PRE_HP_HZ (the clamp's correction
+ high-passed: gain steps that follow a noise's own flicker spread energy into empty low bands), and
+ 'pre' is dropped unless it lowers the score (as the gate is judged). On the films' dev clips: a
+ whoosh's gesture fit 10.46 -> 8.67 (no clamp), a swish's 12.68 -> 11.37 (high-passed), a glitch run
+ keeps the plain clamp's sharp first burst (9.29, 10.09 without it)
env (envelope=True only) after the attack: the renders' mean 5 ms envelope error as a gain curve ({'hop',
'db'}, -12..+6 dB, smoothed): a tail that rings too long, a fade-out the model missed. Off by default:
on the 10 short validation refs it lowered mel dB 3.69 -> 3.62 and raised envelope r 0.977 -> 0.986,
- but CLAP fell 0.925 -> 0.918 (a key press 0.955 -> 0.905)"""
+ but CLAP fell 0.925 -> 0.918 (a key press 0.955 -> 0.905)
+ gate (gate=True) a cut-only gain curve after the onset (_fit_gate): where the renders stand GATE_TOL_DB over
+ the original (the gaps of a run of pulses, a hard stop, a tail the model lets ring on) they are cut to
+ it, with edges as sharp as the original's ({'pts': [[t s, dB], ...]}). Modes and partials decay
+ exponentially and cannot stop; a gated pulse falls 30-60 dB in 5 ms. Kept only when it lowers the
+ renders' mean quality.score, or with judge(list of post-processed renders) -> (score, CLAP cos), when
+ it lowers that score and costs at most GATE_CLAP_TOL of CLAP (fit(clap=True) passes one: on the
+ films' dev clips a gate that helped every model-free measure still cost CLAP on 4 of 67)"""
from ..fx import eq_bands
from ..tone import profile
xm = Q.mono(X)
target = profile(xm)
ys = [Q.mono(y) for y in renders]
+
+ def sc(p): # (score, CLAP) of the renders through a post stage p
+ if judge is None:
+ return float(np.mean([Q.score(Q.quality(xm, apply_post(y, p))) for y in ys])), 0.0
+ return judge([apply_post(y, p) for y in renders])
tot = np.zeros(len(target))
for _ in range(iters):
cur = 10 * np.log10(np.mean([10 ** (profile(eq_bands(y, tot) if tot.any() else y) / 10) for y in ys], 0))
@@ -416,9 +671,17 @@ def fit_post(X, renders, iters=2, max_db=12.0, envelope=False):
if on >= 2 or (struck and len(e) > on + 4):
post['pre'] = {'t_on': round(on * PRE_HOP, 6), 'db': [round(float(v), 1) for v in e[:on]],
'attack_db': [round(float(v), 1) for v in e[on:on + n_att]] if struck else []}
+ if sharp:
+ post['pre']['v'] = 2
+ if noise:
+ post['pre']['hp_hz'] = PRE_HP_HZ
# the clamp pins the attack to the original's contour: re-level what it leaves
g2 = Q.peak_rms_db(xm) - float(np.mean([Q.peak_rms_db(apply_post(y, post)) for y in ys]))
post['gain_db'] = round(float(g + g2), 2)
+ if noise: # a noise's clamp only where it helps (judged like the gate)
+ bare = {'eq_db': post['eq_db'], 'gain_db': round(float(g), 2)}
+ if sc(bare)[0] < sc(post)[0]:
+ post = bare
if envelope:
post['env'] = _fit_env(xm, [apply_post(y, post) for y in ys], post)
# the gain curve moves the balance (tail and attack differ in colour): one more EQ + level pass on top
@@ -431,6 +694,15 @@ def fit_post(X, renders, iters=2, max_db=12.0, envelope=False):
post['eq_db'] = [round(float(v), 2) for v in np.clip(np.asarray(post['eq_db']) + d, -max_db, max_db)]
g3 = Q.peak_rms_db(xm) - float(np.mean([Q.peak_rms_db(apply_post(y, post)) for y in ys]))
post['gain_db'] = round(float(post['gain_db'] + g3), 2)
+ if gate:
+ gt = _fit_gate(xm, [apply_post(y, post) for y in ys], post)
+ if gt:
+ cand = dict(post, gate=gt)
+ g4 = Q.peak_rms_db(xm) - float(np.mean([Q.peak_rms_db(apply_post(y, cand)) for y in ys]))
+ cand['gain_db'] = round(float(post['gain_db'] + g4), 2)
+ (s1, c1), (s0, c0) = sc(cand), sc(post)
+ if s1 < s0 and c1 >= c0 - GATE_CLAP_TOL:
+ post = cand
return post
@@ -458,22 +730,141 @@ def _fit_env(xm, ys, post):
return {'hop': ENV_HOP, 'db': [round(float(v), 2) for v in d]}
-def _env_gain(n, env, scale=1.0):
+def _env_half(m, hop=GATE_HOP):
+ """power (dB) of 2-hop windows on a `hop` grid: frame k covers [k hop, (k + 2) hop), its centre is (k + 1) hop"""
+ k = max(1, int(round(hop * SR)))
+ nb = len(m) // k
+ if nb < 3:
+ return np.zeros(0)
+ p = (m[:nb * k].reshape(nb, k) ** 2).mean(1)
+ return 10 * np.log10(0.5 * (p[:-1] + p[1:]) + 1e-20)
+
+
+def _runs(mask):
+ """[(first, last)] index pairs of the runs of True in a boolean array"""
+ d = np.diff(np.r_[0, mask.astype(np.int8), 0])
+ return list(zip(np.nonzero(d == 1)[0], np.nonzero(d == -1)[0] - 1))
+
+
+def _fit_gate(xm, ys, post, pulses=True):
+ """the gate of a post stage (fit_post): a cut-only gain curve, {'pts': [[t s, dB], ...]} (linear in dB between
+ breakpoints, 0 dB before the first; the last value holds past the end, so a hard stop stays shut in a take that
+ rings out), or None when nothing needs cutting. On 1 ms windows (GATE_HOP grid, a 3-frame median: one frame's
+ flicker is not a gap) from the original's onset: where the original has fallen GATE_FALL_DB under its loudest of
+ the GATE_LOOK_S before (or lies GATE_QUIET_DB under its top) and the renders' mean (ys, through the rest of the
+ post stage) stands more than GATE_TOL_DB over it, or (pulses=True) in a dip between two of its pulses where they
+ stand GATE_DIP_TOL_DB over it, the renders are cut to GATE_KEEP_DB over the original (at most GATE_DEPTH_DB). A
+ run of cut frames survives when it reaches GATE_MIN_CUT_DB and lasts GATE_MIN_S; inside it the cut is smoothed (a
+ 5-frame median, then 1-2-1), its edges are not: the gain falls within a frame of the original's fall and is back
+ to 0 dB a frame before the original rises again (the next pulse's onset is never cut). The clip's own edit fade
+ (its last GATE_EDIT_S) is not a gate: the cut before it holds through it"""
+ from scipy.ndimage import maximum_filter1d, median_filter
+ from .modal import _rdp
+ eo = _env_half(xm)
+ if len(eo) < 8 or not ys:
+ return None
+ er = []
+ for y in ys:
+ e = _env_half(Q.mono(y))
+ er.append(10 ** (np.pad(e, (0, max(0, len(eo) - len(e))), constant_values=-200.0)[:len(eo)] / 10))
+ er = 10 * np.log10(np.mean(er, 0) + 1e-20)
+ eo, er = median_filter(eo, 3, mode='nearest'), median_filter(er, 3, mode='nearest') # a 1-frame flicker of a
+ n = len(eo) # noise is not a gap
+ look = int(round(GATE_LOOK_S / GATE_HOP))
+ fell = ((eo <= maximum_filter1d(eo, look + 1, origin=look // 2) - GATE_FALL_DB) # under its loudest just before
+ | (eo <= eo.max() - GATE_QUIET_DB)) # or quiet, however long
+ top = max(eo.max(), er.max())
+ pre = post.get('pre') or {}
+ t_on = pre.get('t_on')
+ if t_on is None:
+ t_on = float(np.argmax(eo > eo.max() - 20)) * GATE_HOP
+ i0 = int(np.ceil(t_on / GATE_HOP))
+ d = eo - er
+ dip = np.zeros(n, bool) # the dips between the original's pulses
+ if pulses:
+ from scipy.signal import find_peaks
+ e5 = np.convolve(eo, np.ones(10) / 10, mode='same')
+ pk = [p for p in find_peaks(e5, prominence=6.0, distance=16)[0] if e5[p] > e5.max() - 25]
+ for p, q in zip(pk[:-1], pk[1:]):
+ seg = np.arange(p, q)
+ dip[seg[e5[seg] < min(e5[p], e5[q]) - GATE_DIP_DB]] = True
+ hit = ((d < -GATE_TOL_DB) & fell) | ((d < -GATE_DIP_TOL_DB) & dip)
+ cut = np.where(hit & (er > top - 90.0), np.maximum(d + GATE_KEEP_DB, GATE_DEPTH_DB), 0.0)
+ cut = np.minimum(cut, 0.0)
+ cut[:i0] = 0.0
+ i_edit = max(i0, n - int(round(GATE_EDIT_S / GATE_HOP)))
+ cut[i_edit:] = cut[i_edit - 1] if i_edit > i0 else 0.0
+ out = np.zeros(n)
+ n_min = int(round(GATE_MIN_S / GATE_HOP))
+ for a, b in _runs(cut < 0):
+ seg = cut[a:b + 1]
+ if seg.min() > -GATE_MIN_CUT_DB or (len(seg) < n_min and b + 1 < n):
+ continue
+ if len(seg) >= 5:
+ seg = median_filter(seg, 5, mode='nearest')
+ seg = np.convolve(np.pad(seg, 1, mode='edge'), [0.25, 0.5, 0.25], 'valid')
+ seg = np.minimum(seg, 0.0)
+ if b + 1 < n and len(seg) > 1: # open a frame early: the original rises in frame b + 1
+ seg[-1] = 0.0
+ out[a:b + 1] = seg
+ if not (out < -0.5).any():
+ return None
+ t = (np.arange(n) + 1) * GATE_HOP # frame centres
+ keep = _rdp(t, out, GATE_RDP_DB)
+ return {'pts': [[round(float(t[i]), 5), round(float(out[i]), 1)] for i in keep]}
+
+
+def _gate_gain(n, gate, tmap=None):
+ """the gate's gain per sample (linear, GATE_SMOOTH_S smooth); tmap: the knobs' time map of the gate's breakpoints
+ (render)"""
+ pts = np.asarray(gate['pts'], float).reshape(-1, 2)
+ t, g = pts[:, 0], pts[:, 1]
+ if tmap is not None:
+ t, g = tmap(t, g)
+ a = 10 ** (np.interp(np.arange(n) / SR, t, g, left=0.0, right=float(g[-1]) if len(g) else 0.0) / 20)
+ k = int(round(GATE_SMOOTH_S * SR)) | 1
+ if k > 1 and n > k:
+ w = np.hanning(k + 2)[1:-1]
+ a = np.convolve(np.pad(a, k // 2, mode='edge'), w / w.sum(), mode='valid')
+ return a
+
+
+def _env_gain(n, env, scale=1.0, anchor=0.0):
d = np.asarray(env['db'], float)
hop = float(env['hop']) * scale
tc = (np.arange(len(d)) + 0.5) * hop * SR
+ if anchor and scale != 1.0: # time scaled about the anchor (a gesture's first strike)
+ tc = (anchor + ((np.arange(len(d)) + 0.5) * float(env['hop']) - anchor) * scale) * SR
return 10 ** (np.interp(np.arange(n), tc, d, left=0.0, right=float(d[-1]) if len(d) else 0.0) / 20)
-def _pre_clamp(y, pre, scale=1.0):
+def _smooth_cuts(gdb):
+ """widen each cut by a frame and smooth with 1-2-1 (smooth edges)"""
+ neg = np.minimum(gdb, 0.0)
+ neg = np.minimum(neg, np.minimum(np.r_[neg[1:], 0.0], np.r_[0.0, neg[:-1]]))
+ gdb = np.where(gdb < 0, neg, gdb)
+ return np.convolve(np.pad(gdb, 1, mode='edge'), [0.25, 0.5, 0.25], 'valid') if len(gdb) else gdb
+
+
+def _pre_clamp(y, pre, scale=1.0, anchor=0.0):
"""the attack through the original's envelope (0.25 ms frames): before the onset y may not stand above the
original (+1.5 dB; only cut), over the ATTACK_S after it y follows the original's contour (+-9 dB, the correction
fading out over the last 2 ms), so a smeared or pre-echoed attack comes back as sharp as the recording's.
- scale: time factor of the render (spectral length / speed)"""
+ scale: time factor of the render (spectral length / speed, gesture length) about `anchor` (s; a gesture with
+ tones stretches about their first strike, tones['onset'], so its onset stays where it was: the pre-onset window
+ ends at anchor + (t_on - anchor) x scale, and the original's pre-roll envelope is read through the same map;
+ 0: about the start). A stage fitted with sharp=True (0.4, pre['v'] = 2)
+ smooths the pre-onset cut and the attack contour apart: the edge smoothing carried the pre-onset cut into the
+ strike's first frame (half of a -20 dB cut, all but silencing it after a stripped bed's -200 dB pre-roll).
+ pre['hp_hz'] (a noise model's stage, 0.4): the correction (y x (gain - 1)) is high-passed there (4th-order
+ Butterworth, zero phase) before it is added: it cannot put energy into the bands under it"""
+ v2 = pre.get('v') == 2
k = int(round(PRE_HOP * SR))
ref_pre = np.asarray(pre.get('db') or [], float)
ref_att = np.asarray(pre.get('attack_db') or [], float)
- n_pre = int(round(float(pre['t_on']) * scale / PRE_HOP))
+ t_on = float(pre['t_on'])
+ about = bool(anchor) and scale != 1.0
+ n_pre = int(round((anchor + (t_on - anchor) * scale if about else t_on * scale) / PRE_HOP))
n_att = len(ref_att) if scale == 1.0 else int(round(len(ref_att) * scale))
nb = min(len(y) // k, n_pre + n_att)
if nb < 2:
@@ -482,31 +873,44 @@ def _pre_clamp(y, pre, scale=1.0):
e = _env_frames(m[:nb * k], PRE_HOP)
gdb = np.zeros(nb)
if n_pre > 0 and len(ref_pre):
- want = np.interp(np.arange(min(n_pre, nb)) / max(n_pre - 1, 1), np.linspace(0, 1, len(ref_pre)), ref_pre)
+ u = np.arange(min(n_pre, nb)) / max(n_pre - 1, 1)
+ if about: # the new pre-onset frames back to the original's time
+ t_new = u * (anchor + (t_on - anchor) * scale)
+ u = np.clip((anchor + (t_new - anchor) / scale) / max(t_on, 1e-9), 0.0, 1.0)
+ want = np.interp(u, np.linspace(0, 1, len(ref_pre)), ref_pre)
gdb[:len(want)] = np.minimum(0.0, want + 1.5 - e[:len(want)])
if n_att > 0 and len(ref_att) and nb > n_pre:
j = np.arange(n_pre, nb)
want = np.interp((j - n_pre) / max(n_att - 1, 1), np.linspace(0, 1, len(ref_att)), ref_att)
fade = np.clip((n_pre + n_att - 1 - j) / max(1, int(round(0.002 / PRE_HOP))), 0.0, 1.0)
gdb[n_pre:] = np.clip(want - e[n_pre:nb], -9.0, 9.0) * fade
- neg = np.minimum(gdb, 0.0) # widen each cut by a frame (smooth edges)
- neg = np.minimum(neg, np.minimum(np.r_[neg[1:], 0.0], np.r_[0.0, neg[:-1]]))
- gdb = np.where(gdb < 0, neg, gdb)
- gdb = np.convolve(np.pad(gdb, 1, mode='edge'), [0.25, 0.5, 0.25], 'valid')
+ if v2 and 0 < n_pre < nb:
+ gdb = np.r_[_smooth_cuts(gdb[:n_pre]), _smooth_cuts(gdb[n_pre:])]
+ else:
+ gdb = _smooth_cuts(gdb)
n_end = nb * k
tt = (np.arange(nb) + 0.5) * k
gs = 10 ** (np.interp(np.arange(n_end), np.r_[tt, n_end], np.r_[gdb, 0.0]) / 20)
y = np.array(y, copy=True)
- if y.ndim == 2:
- y[:n_end] *= gs[:, None]
- else:
- y[:n_end] *= gs
+ g2 = gs[:, None] if y.ndim == 2 else gs
+ hp = float(pre.get('hp_hz') or 0.0)
+ if hp > 0: # the correction high-passed (zero phase): no energy below hp
+ from scipy import signal
+ n_pad = min(len(y), n_end + int(0.02 * SR))
+ d = np.zeros_like(y[:n_pad])
+ d[:n_end] = y[:n_end] * (g2 - 1.0)
+ sos = signal.butter(4, hp, 'highpass', fs=SR, output='sos')
+ y[:n_pad] += signal.sosfiltfilt(sos, d, axis=0)
+ return y
+ y[:n_end] *= g2
return y
-def apply_post(y, post, scale=1.0, tail=1.0):
- """a render through a model's post stage (EQ, gain, attack clamp, envelope calibration). scale: the render's
- time factor (spectral length / speed); tail: the decay knobs' factor on the envelope after the attack"""
+def apply_post(y, post, scale=1.0, tail=1.0, tmap=None, anchor=0.0):
+ """a render through a model's post stage (EQ, gain, attack clamp, envelope calibration, gate). scale: the
+ render's time factor (spectral length / speed, gesture length) about `anchor` (s: a gesture with tones stretches
+ about their first strike; 0 = the start); tail: the decay knobs' factor on the envelope after the attack;
+ tmap: (t, dB) -> (t, dB), where the knobs moved the gate's breakpoints (default: scaled by `scale`)"""
if not post:
return y
from ..fx import eq_bands
@@ -514,40 +918,76 @@ def apply_post(y, post, scale=1.0, tail=1.0):
y = eq_bands(y, post['eq_db'])
y = y * 10 ** (float(post.get('gain_db', 0.0)) / 20)
if post.get('pre'):
- y = _pre_clamp(y, post['pre'], scale)
+ y = _pre_clamp(y, post['pre'], scale, anchor)
if post.get('env') and post['env'].get('db'):
- g = _env_gain(len(y), post['env'], scale * tail)
+ g = _env_gain(len(y), post['env'], scale * tail, anchor)
+ y = y * (g[:, None] if y.ndim == 2 else g)
+ if post.get('gate') and post['gate'].get('pts'):
+ if tmap is None and scale != 1.0:
+ tmap = lambda t, g: (t * scale, g)
+ g = _gate_gain(len(y), post['gate'], tmap)
y = y * (g[:, None] if y.ndim == 2 else g)
return y
def fit(x, kind='auto', layers=None, candidates=None, clap=False, fast=False, seeds=(0,), name=None, verbose=False,
- collect=None, post=True):
+ collect=None, post=True, bed=True, space=True, confirm=None):
"""a model of x (path or 48 kHz array). kind: 'auto' | 'modal' | 'sines' | 'spectral' | 'hybrid' (layers=
('modal', 'spectral') by default). auto: candidates (default plan(character(x))) are all fitted, rendered with
`seeds` and scored (quality.score; clap=True adds CLAP; the mean over seeds), the best is returned with a
- 'fit_engine' entry. collect: a dict that receives every candidate's model ({candidate: model})"""
+ 'fit_engine' entry. collect: a dict that receives every candidate's model ({candidate: model}).
+ bed=True: a recording with a bed under it (bed_of: a film's music) is stripped first (strip_bed) and the sound
+ alone is fitted, from its onset, and the post stage is measured on it; the bed is kept as post['bed']: render
+ puts a steady noise with its spectrum back under the sound (render(bed=False): the sound alone), and the
+ candidates are scored that way against the original (energy the bed covers is not excess; energy the fit adds
+ over it is). space=True: a stereo recording whose tail is decorrelated gets the diffuse side layer of fit.space
+ (post['space']), fitted on the chosen model only, on its dry render (seed 0, through its post stage): it costs
+ 0.6-1.6 s, and fitted per candidate it changed no choice on the films' dev clips (the stereo score terms only
+ separate candidates that already sound alike). confirm (default CONFIRM; 0 = off; with one seed only): the best
+ this many candidates are scored on CONFIRM_SEEDS too and the choice is made on the mean (each row's 'score2'),
+ then the CLAP margin rule (CLAP_MARGIN, module doc) may pick a candidate CLAP clearly prefers. With a bed, the
+ winner's kind is also fitted on the recording as it is (BED_CHECK; a 'plain:<kind>' row) and the bed is dropped
+ when that scores lower (fit_engine['bed']: 'score_stripped', 'score_plain', 'kept')"""
import os
if name is None and isinstance(x, (str, os.PathLike)):
name = os.path.basename(str(x))
- X = Q.load(x)
+ X0 = X = Q.load(x)
t0 = time.time()
- if kind != 'auto':
+ bd = bed_of(X) if bed else None
+ if bd:
+ X = strip_bed(X, bd) # the fitters and the post stage see the sound alone; the
+ if kind != 'auto': # candidates are scored with the bed back on the original
if kind == 'hybrid':
m = fit_hybrid(X, layers or ('modal', 'spectral'), fast=fast, name=name)
else:
m = fit_kind(X, kind, fast=fast, name=name)
if post:
- m['post'] = fit_post(X, [render(m, seed=s) for s in (0, 1)])
+ m['post'] = fit_post(X, [_render(m, seed=s, final=False) for s in (0, 1)],
+ noise=kind_of(m) in NOISE_KINDS)
+ _add_space(m, X, space)
+ _set_take(m, bd)
m['fit_engine'] = {'kind': kind, 'seconds': round(time.time() - t0, 2)}
return m
c = character(X)
cands = list(candidates or plan(c))
base, rows, best = {}, [], None
+ kept = {}
clap_e = None
seeds = tuple(int(s) for s in seeds) or (0,)
if clap:
- clap_e = Q.clap_embed([X])[0]
+ clap_e = Q.clap_embed([X0])[0]
+ judge = _judge(X0, clap_e, bd) # the post stage's gate is kept by the same score, with CLAP
+
+ def scored(m, ss): # the scores of a model's renders on seeds ss against the
+ ys = [render(m, seed=s) for s in ss] # original, the first one's quality, the CLAP cosines
+ E = Q.clap_embed(ys) if clap_e is not None else None
+ qs = []
+ for i, y in enumerate(ys):
+ q = Q.quality(X0, y)
+ if E is not None:
+ q['clap_cos'] = round(float(clap_e @ E[i]), 4)
+ qs.append(q)
+ return [Q.score(q) for q in qs], qs[0], [q.get('clap_cos') for q in qs]
for cand in cands:
t1 = time.time()
try:
@@ -560,32 +1000,185 @@ def fit(x, kind='auto', layers=None, candidates=None, clap=False, fast=False, se
tf = time.time() - t1
if collect is not None:
collect[cand] = m
- qs = []
if post:
- r0 = [render(m, seed=s) for s in (0, 1)]
- m['post'] = fit_post(X, r0)
- ys = [render(m, seed=s) for s in seeds]
- E = Q.clap_embed(ys) if clap_e is not None else None
- for i, y in enumerate(ys):
- q = Q.quality(X, y)
- if E is not None:
- q['clap_cos'] = round(float(clap_e @ E[i]), 4)
- qs.append(q)
- sc = float(np.mean([Q.score(q) for q in qs]))
- row = {'candidate': cand, 'score': round(sc, 3), 'fit_s': round(tf, 2), 'quality': qs[0]}
+ r0 = [_render(m, seed=s, final=False) for s in (0, 1)]
+ m['post'] = fit_post(X, r0, judge=judge, noise=kind_of(m) in NOISE_KINDS)
+ _set_take(m, bd, vary=False) # the bed under the renders scored against the original
+ scs, q0, _ = scored(m, seeds)
+ sc = float(np.mean(scs))
+ row = {'candidate': cand, 'score': round(sc, 3), 'fit_s': round(tf, 2), 'quality': q0}
rows.append(row)
if verbose:
- print(f' {cand}: score {sc:.2f} ({Q.text(qs[0])}) in {tf:.1f} s')
+ print(f' {cand}: score {sc:.2f} ({Q.text(q0)}) in {tf:.1f} s')
+ kept[cand] = (m, row)
if best is None or sc < best[0]:
best = (sc, cand, m)
if best is None:
raise RuntimeError('every fitter failed: ' + '; '.join(r.get('error', '') for r in rows))
+ confirm = CONFIRM if confirm is None else int(confirm)
+ more = [s for s in CONFIRM_SEEDS if s not in seeds]
+ confirmed = confirm > 1 and len(seeds) == 1 and len(kept) > 1 and bool(more)
+ if confirmed:
+ top = sorted(kept, key=lambda k: kept[k][1]['score'])[:confirm] # the winner's curse: a candidate with a
+ for k in top: # lot of noise in it wins on one lucky
+ s2, _, c2 = scored(kept[k][0], more) # draw (module doc)
+ kept[k][1]['score2'] = round(float(np.mean([kept[k][1]['score']] + s2)), 3)
+ if clap_e is not None:
+ kept[k][1]['clap2'] = round(float(np.mean([kept[k][1]['quality']['clap_cos']] + c2)), 4)
+ k = min(top, key=lambda k: kept[k][1]['score2'])
+ best = (kept[k][1]['score2'], k, kept[k][0])
+ if verbose:
+ print(f' confirmed on seeds {more}: ' + ', '.join(f"{k} {kept[k][1]['score2']:.2f}" for k in top))
+ if CLAP_MARGIN and confirmed and clap_e is not None: # a candidate CLAP clearly prefers, close on the score
+ rows_ = {k: v[1] for k, v in kept.items()}
+ alts = _clap_alts(rows_, best[1])
+ for k in alts: # on the confirm seeds too (a lucky seed-0 draw is not enough)
+ r_ = kept[k][1]
+ if r_.get('clap2') is None:
+ s2, _, c2 = scored(kept[k][0], more)
+ r_['score2'] = round(float(np.mean([r_['score']] + s2)), 3)
+ r_['clap2'] = round(float(np.mean([r_['quality']['clap_cos']] + c2)), 4)
+ k = _clap_pick(rows_, best[1], alts)
+ if k:
+ kept[k][1]['chosen_by'] = 'clap'
+ if verbose:
+ print(f" {k} chosen by CLAP ({kept[k][1]['clap2']:.3f} against {kept[best[1]][1]['clap2']:.3f})")
+ best = (kept[k][1]['score2'], k, kept[k][0])
+ bed_info = None
+ if bd:
+ bed_info = {'level_db': bd['level_db'], 't_on': bd['t_on']}
+ if BED_CHECK: # analysis by synthesis decides whether the bed is one
+ t1 = time.time()
+ try:
+ b0 = {}
+ if best[1] == 'gesture': # its tones come from a sines fit of the same recording
+ _cand_fit(X0, 'sines', fast, b0, name)
+ mp = _cand_fit(X0, best[1], fast, b0, name)
+ if post:
+ mp['post'] = fit_post(X0, [_render(mp, seed=s, final=False) for s in (0, 1)],
+ judge=_judge(X0, clap_e, None), noise=kind_of(mp) in NOISE_KINDS)
+ scs, q0, _ = scored(mp, list(seeds) + (more if confirmed else []))
+ sp_ = round(float(np.mean(scs)), 3)
+ rows.append({'candidate': f'plain:{best[1]}', 'score': sp_, 'fit_s': round(time.time() - t1, 2),
+ 'quality': q0})
+ bed_info.update(score_stripped=round(float(best[0]), 3), score_plain=sp_, kept=True)
+ if verbose:
+ print(f' {best[1]} on the recording as it is (no bed stripped): score {sp_:.2f} against '
+ f'{best[0]:.2f}')
+ if sp_ < best[0]: # the "bed" was the sound's own start (a whoosh's approach)
+ best, bd, X = (sp_, best[1], mp), None, X0
+ bed_info['kept'] = False
+ except Exception as e: # the stripped fit stands
+ rows.append({'candidate': f'plain:{best[1]}', 'error': f'{type(e).__name__}: {e}'})
m = best[2]
+ if post:
+ _add_space(m, X, space)
+ _set_take(m, bd)
m['fit_engine'] = {'kind': 'auto', 'chosen': best[1], 'character': c, 'candidates': rows,
'seeds': list(seeds), 'seconds': round(time.time() - t0, 2)}
+ sp = (m.get('post') or {}).get('space')
+ if sp:
+ m['fit_engine']['space'] = dict(sp.get('fit') or {}, tail_corr=sp.get('tail_corr'))
+ if bed_info:
+ m['fit_engine']['bed'] = bed_info
return m
+CLAP_MARGIN = 0.02 # fit(): a candidate whose CLAP beats the choice's by this much, on seed 0 and on the mean over
+CLAP_TOL = 1.5 # the confirm seeds, with a score at most this much higher (seed 0 and the mean) and an
+CLAP_ENV_TOL = 0.05 # envelope r at most this much lower, is chosen instead (the highest such mean CLAP). The
+ # model-free terms (mel, onset, envelope, excess) decided near-ties against a clearly closer
+ # sound: on the held-out check 5 of 6 regressions were a gesture fit chosen with 0.015-0.06
+ # less CLAP than another candidate. On the dev clips (0.015-0.03 margins, score tolerances 1-4
+ # tried): 5 choices change, all to a higher CLAP, grades 30/26/20/2 -> 33/24/20/1, takes 46
+ # -> 46; a tolerance of 4 also took a candidate with an envelope r of 0.58 (the guard)
+BED_CHECK = True # fit(): a recording with a bed (bed_of) is also fitted as it is with the winning candidate's
+ # kind, and the bed is dropped when that scores lower on the same seeds (analysis by
+ # synthesis): a whoosh's own steady approach 13.6 dB under its peak read as a music bed, and
+ # every candidate fitted on the stripped rest fell (CLAP 0.64 against 0.88)
+
+
+def _clap_alts(rows, best):
+ """the candidates the CLAP margin rule weighs against the choice `best` ({candidate: row}, rows with 'score' and
+ 'quality' as fit() keeps them): CLAP_MARGIN more CLAP on seed 0, a seed-0 score at most CLAP_TOL higher"""
+ rb = rows[best]
+ c0 = rb['quality'].get('clap_cos')
+ if c0 is None:
+ return []
+ return [k for k, r in rows.items() if k != best and r['quality'].get('clap_cos') is not None
+ and r['quality']['clap_cos'] >= c0 + CLAP_MARGIN and r['score'] <= rb['score'] + CLAP_TOL]
+
+
+def _clap_pick(rows, best, alts):
+ """the alternative (of _clap_alts) the CLAP margin rule chooses, or None: its mean CLAP over the confirm seeds
+ ('clap2') CLAP_MARGIN over the choice's, its mean score ('score2') at most CLAP_TOL over it and its envelope r at
+ most CLAP_ENV_TOL under it; the highest such mean CLAP"""
+ rb = rows[best]
+ ok = [k for k in alts if rows[k].get('clap2') is not None and rb.get('clap2') is not None
+ and rows[k]['clap2'] >= rb['clap2'] + CLAP_MARGIN and rows[k]['score2'] <= rb['score2'] + CLAP_TOL
+ and rows[k]['quality']['env_corr'] >= rb['quality']['env_corr'] - CLAP_ENV_TOL]
+ return max(ok, key=lambda k: rows[k]['clap2']) if ok else None
+
+
+def _judge(X0, clap_e, bd):
+ """the post stage's judge (fit_post(judge=)): (mean score, mean CLAP) of renders against the original X0 with the
+ bed bd under them (None: none), or None without CLAP"""
+ if clap_e is None:
+ return None
+
+ def judge(ys):
+ ys = [_with_bed(y, bd, s) for s, y in enumerate(ys)]
+ c = [float(clap_e @ e) for e in Q.clap_embed(ys)]
+ return (float(np.mean([Q.score(dict(Q.quality(X0, y), clap_cos=v)) for y, v in zip(ys, c)])),
+ float(np.mean(c)))
+ return judge
+
+
+CONFIRM = 3 # fit(): the best this many candidates (seed-0 score) are scored on CONFIRM_SEEDS too, chosen
+CONFIRM_SEEDS = (2, 3) # by the mean (each row's 'score2'): on short noise-led sounds (keys, ticks) one seed's CLAP
+ # spread 0.84-0.92 and its score 7-13, and the seed-0 choice took the lucky draws (a key's
+ # gesture fit scored 7.11 on seed 0 and 8.0-12.9 on seeds 1-5; an air-only gesture fit at CLAP
+ # 0.906 on seed 0 averaged 0.885 over seeds 1-4, and its takes failed the take check). Not
+ # seeds 0 and 1: the post stage is fitted on those. Costs 2-30 s a fit (6 renders + CLAP)
+TAKE_VARY = {'modal': 0.5, 'hybrid': 0.5} # fit(): a model's 'vary_scale' (render multiplies the vary knob by it).
+ # Modal / hybrid takes drifted from their fit at the bank's vary 0.3 (on
+ # the films' clips CLAP to the target -0.047 / -0.075, 10 / 0 of 38 / 9
+ # passing the take check; sines +0.003, 38 of 65): 0.3 now jitters them
+ # as 0.15 did
+
+
+def _set_take(m, bd, vary=True):
+ """the fit's take settings: the vary scale of its kind (TAKE_VARY) and its stripped bed (post['bed'])"""
+ s = TAKE_VARY.get(kind_of(m))
+ if vary and s is not None:
+ m['vary_scale'] = s
+ if bd:
+ m['post'] = dict(m.get('post') or {}, bed=bd)
+
+
+def _with_bed(y, bd, seed=0):
+ """a dry render with the bed layer under it (render's default for a fit of a stripped recording)"""
+ if not bd:
+ return y
+ b = bed_noise(bd, len(y), seed)
+ return y + (b[:, None] if np.ndim(y) == 2 else b)
+
+
+def _add_space(m, X, on=True):
+ """the diffuse side layer (fit.space) of a stereo recording with a decorrelated tail (X: the sound alone, a
+ stripped bed's too), fitted on the model's dry render (seed 0, through its post stage, without a bed, before the
+ final fade and limit): post['space'], or nothing"""
+ m.get('post', {}).pop('space', None)
+ if not on or channels_of(X) != 2 or 'post' not in m:
+ return
+ from . import space as SP
+ if not SP.needs_space(X):
+ return
+ sp = SP.fit_space(X, SR, dry=_render(m, seed=0, bed=False, final=False, space=False))
+ if sp:
+ m['post']['space'] = sp
+
+
# ================================================================================================ rendering
def _layer_render(m, seed, two, **kn):
F = _fitters()
@@ -616,8 +1209,9 @@ def _render_hybrid(model, seed=0, pitch=0.0, decay=1.0, bright=0.0, vary=0.0, wi
m = L['model']
k = kind_of(m)
s = (int(seed) * 7 + 101 * (i + 1)) & 0x7FFFFFFF
- if k == 'sines':
- kn = {'transpose': pitch, 'decay': decay, 'brightness': bright, 'vary': vary, 'ring': ring}
+ if k == 'sines': # the tonal layer keeps its fitted side (0.4: sines 'side')
+ kn = {'transpose': pitch, 'decay': decay, 'brightness': bright, 'vary': vary, 'ring': ring,
+ 'stereo': two and bool(m.get('side')), 'width': width}
elif k == 'modal':
size = 2.0 ** (-pitch / 12.0)
kn = {'size': size, 'damping': size / max(decay, 1e-3), 'brightness': bright, 'vary': vary,
@@ -658,6 +1252,10 @@ def _render_raw(model, seed=0, **knobs):
knobs['gain_db'] = knobs.pop('gain')
knobs.setdefault('stereo', two)
y = F['sines'].render(model, seed, knobs=knobs)
+ elif k == 'gesture':
+ if 'gain' in knobs:
+ knobs['gain_db'] = knobs.pop('gain')
+ y = F['gesture'].render(model, seed, **knobs)
else:
if 'gain' in knobs:
knobs['gain_db'] = knobs.pop('gain')
@@ -668,7 +1266,7 @@ def _render_raw(model, seed=0, **knobs):
return np.asarray(y, np.float64)
-def render(model, seed=0, take=None, **knobs):
+def render(model, seed=0, take=None, bed=None, **knobs):
"""the sound of a model (float64, 48 kHz): (n, 2) when the source was stereo, else (n,), through the model's post
stage (fitted EQ + gain) when it has one. knobs: knobs_of(model) names (identity values re-render the fit);
unknown names raise.
@@ -677,21 +1275,127 @@ def render(model, seed=0, take=None, **knobs):
the trimmed recording stopped (modal / sines ring=True), and the result is high-passed at TAKE_DC_HZ (no DC
offset, no infrasound: a noise band modulated by its envelope spreads energy down to DC, and a new seed of a
fit could carry an offset of 1-2 % of its peak). take=False keeps the recording's length and every bit of its
- low end: what the fit's quality is measured on (the identity render, seed 0, is that)."""
+ low end: what the fit's quality is measured on (the identity render, seed 0, is that).
+ A model's 'vary_scale' (fit: TAKE_VARY) multiplies the vary knob. post['space'] (fit.space) adds the diffuse side
+ after the post stage (the width knob scales it; a take lets it ring out). bed (not a knob; default None = when the
+ model has one): a fit of a stripped recording (post['bed']) puts a steady noise with its bed's spectrum under the
+ sound over the recording's length (bed_noise: the original's context, so the fit and its takes compare with the
+ original as before); bed=False renders the sound alone (a kit's or a bank's take).
+ Every render of a model fitted by 0.4 ends with sines.finish: a fade-in (FADE_IN_S, shorter when the sound's onset
+ comes earlier: 0.25 ms at least) and a gain down to PEAK_DB dBFS when it peaks over that (no clipping; the
+ reference clips peak at about -3). A model fitted before 0.4 (finished(): a modal, sines or hybrid model without
+ the 'coherent' flag, the 0.3 bank's fit entries, or a spectral model without 'finish') renders as it did, sample
+ for sample: the presets' takes stay the same"""
+ return _render(model, seed, take, bed, **knobs)
+
+
+FADE_IN_S = 0.002 # render(): the fade-in at the start of every render (it starts at zero, not mid-bed) ...
+FADE_MIN_S = 0.00025 # ... shortened to 0.5 ms before the onset (the first 1 ms frame 40 dB under the loudest),
+ # to this at least, so a clip cut into its own attack keeps the attack
+PEAK_DB = -1.0 # ... and the peak limit (a gain only): 18 of the films' 464 takes had peaked over 0 dBFS
+
+
+def _render(model, seed=0, take=None, bed=None, final=True, space=True, **knobs):
+ """render() with the internal switches: final=False leaves out the fade-in and the peak limit (renders the fit
+ engine measures a model's level and post stage from), space=False the space layer (its dry)"""
knobs = dict(knobs)
take = float(knobs.get('vary') or 0.0) > 0 if take is None else bool(take)
if kind_of(model) == 'morph':
w = knobs.pop('morph', model.get('amount', 0.5))
- return render(morph(model['a'], model['b'], w), seed, take=take, **knobs)
+ return _render(morph(model['a'], model['b'], w), seed, take, bed, final, space, **knobs)
+ if knobs.get('vary') and model.get('vary_scale') is not None:
+ knobs['vary'] = float(knobs['vary']) * float(model['vary_scale'])
knobs.setdefault('ring', take)
y = _render_raw(model, seed, **knobs)
k = kind_of(model)
- scale = float(knobs.get('length', 1.0)) / float(knobs.get('speed', 1.0)) if k == 'spectral' else 1.0
+ scale = {'spectral': lambda: float(knobs.get('length', 1.0)) / float(knobs.get('speed', 1.0)),
+ 'gesture': lambda: float(knobs.get('length', 1.0))}.get(k, lambda: 1.0)()
tail = {'modal': lambda: float(knobs.get('size', 1.0)) / float(knobs.get('damping', 1.0)),
'sines': lambda: float(knobs.get('stretch', 1.0)) * float(knobs.get('decay', 1.0)),
'hybrid': lambda: float(knobs.get('decay', 1.0))}.get(k, lambda: 1.0)()
- y = apply_post(y, model.get('post'), scale, tail)
- return _dc_block(y) if take else y
+ post = model.get('post') or {}
+ # gesture.render stretches a gesture with tones about their first strike (its onset stays): the post stage too
+ anchor = float((model.get('tones') or {}).get('onset', 0.0)) if k == 'gesture' else 0.0
+ y = apply_post(y, post, scale, tail, _gate_map(model, knobs) if post.get('gate') else None, anchor)
+ if space and post.get('space'): # fit.space: the diffuse side, after the post stage
+ from . import space as SP
+ y = SP.render_space(post['space'], y, seed=seed, width=float(knobs.get('width', 1.0)), ring=take,
+ limit=False)
+ if post.get('bed') and bed is not False:
+ b = bed_noise(post['bed'], len(y), seed)
+ y = y + (b[:, None] if y.ndim == 2 else b)
+ y = _dc_block(y) if take else y
+ if not final or not finished(model):
+ return y
+ from .sines import finish
+ fade = float(np.clip(Q.onset_index(Q.mono(y), -40.0) / SR - 0.0005, FADE_MIN_S, FADE_IN_S))
+ return finish(y, SR, fade_s=fade, peak_db=PEAK_DB)
+
+
+def finished(model):
+ """does a render of this model end with the 0.4 finish (render())? Every fit made by 0.4 does (its modal and sines
+ models carry 'coherent', its spectral models 'finish', a morph of them the flag of either side; gesture models
+ are new in 0.4); a modal, sines, hybrid or spectral model fitted before 0.4 does not"""
+ k = kind_of(model)
+ if k in ('modal', 'sines'):
+ return bool(model.get('coherent'))
+ if k == 'hybrid':
+ return any((L.get('model') or {}).get('coherent') for L in model['layers'])
+ if k == 'spectral':
+ return bool(model.get('finish'))
+ return True
+
+
+def _gate_map(model, knobs):
+ """the gate's time map under a model's knobs ((t, dB) -> (t, dB)), None when the knobs leave time alone: a sines
+ stretch scales time about the first onset, a spectral length / speed scales it; a modal spacing moves every
+ strike, and the cuts after a strike move with it (decays do not move a gate: a gate is the designer's cut, a
+ longer ring is cut at the same place)"""
+ k = kind_of(model)
+ if k == 'sines' and float(knobs.get('stretch', 1.0)) != 1.0:
+ on, st = float(model.get('onset', 0.0)), float(knobs['stretch'])
+ return lambda t, g: (on + (t - on) * st, g)
+ if k == 'spectral':
+ sc = float(knobs.get('length', 1.0)) / float(knobs.get('speed', 1.0))
+ return None if sc == 1.0 else (lambda t, g: (t * sc, g))
+ if k == 'gesture' and float(knobs.get('length', 1.0)) != 1.0: # the whole gesture about the tones' first strike
+ on, st = float((model.get('tones') or {}).get('onset', 0.0)), float(knobs['length'])
+ return lambda t, g: (on + (t - on) * st, g)
+ if k == 'modal' and float(knobs.get('spacing', 1.0)) != 1.0 and len(model['strikes']) > 1:
+ t0 = np.array([s['t'] for s in model['strikes']], float)
+ tn = [t0[0]]
+ for i in range(1, len(t0)): # modal._plan's strike times
+ tn.append(tn[-1] + max(0.003, (t0[i] - t0[i - 1]) * float(knobs['spacing'])))
+ tn = np.asarray(tn)
+
+ def f(t, g):
+ t, g = np.asarray(t, float), np.asarray(g, float)
+ if not len(t):
+ return t, g
+ G = lambda u: float(np.interp(u, t, g, left=0.0, right=float(g[-1])))
+ j = np.clip(np.searchsorted(t0, t, side='right') - 1, 0, len(t0) - 1)
+ ot, og = [], []
+ for i in range(len(t0)):
+ d, sel = tn[i] - t0[i], j == i
+ if i == len(t0) - 1:
+ ot.append(t[sel] + d)
+ og.append(g[sel])
+ break
+ s, ns = max(t0[i + 1] - GATE_MAP_S, t0[i]), max(tn[i + 1] - GATE_MAP_S, tn[i])
+ a = sel & (t < s) & (t + d < ns) # the cut after strike i moves with it, up to the new split
+ ot.append(t[a] + d)
+ og.append(g[a])
+ e = min(s + d, ns) # held at the split's value across a longer gap; a shorter gap
+ ot.append([e - 1e-6, ns]) # ends the cut at its value there
+ og.append([G(e - d), G(s)])
+ b = sel & (t >= s) # the reopening before strike i + 1 moves with that strike
+ ot.append(t[b] + (tn[i + 1] - t0[i + 1]))
+ og.append(g[b])
+ tt, gg = np.concatenate(ot), np.concatenate(og)
+ ok = tt > np.r_[-np.inf, np.maximum.accumulate(tt)[:-1]] # strictly increasing for np.interp
+ return tt[ok], gg[ok]
+ return f
+ return None
def _dc_block(y, hz=TAKE_DC_HZ):
@@ -777,12 +1481,54 @@ def _mix_curves(ta, va, tb, vb, w):
return g, (1 - w) * np.interp(g, ta, va) + w * np.interp(g, tb, vb)
+def _anchor_times(A, B):
+ """the paired anchor times a morph of two models of one kind warps time through: the strikes (modal), the onsets
+ (sines), the loudness peak (spectral), between the start and the end of each"""
+ k = kind_of(A)
+ if k == 'modal':
+ SA, SB = A['strikes'], B['strikes']
+ n = min(len(SA), len(SB))
+ return ([0.0] + [s['t'] for s in SA[:n]] + [A['length_s']], [0.0] + [s['t'] for s in SB[:n]] + [B['length_s']])
+ if k == 'sines':
+ oa = list(A.get('onsets') or [A.get('onset', 0.0)])
+ ob = list(B.get('onsets') or [B.get('onset', 0.0)])
+ n = min(len(oa), len(ob))
+ return [0.0] + oa[:n] + [A['duration']], [0.0] + ob[:n] + [B['duration']]
+ if k == 'spectral':
+ def peak(M):
+ E = 10 ** (np.asarray(M['env']['db'], float) / 10)
+ return float(M['env']['t'][int(np.argmax(E.sum(1)))])
+ return [0.0, peak(A), A['dur']], [0.0, peak(B), B['dur']]
+ raise ValueError(f'a {k} model has no anchors of its own')
+
+
+def _morph_maps(a, b, w):
+ """(a -> morph, b -> morph): the time maps a morph of a and b puts its strikes, onsets or loudness peak on
+ (_anchors through _anchor_times), where the post stage's gate follows them; a gesture: its tones' when both have
+ tones of one kind, else its air's; a hybrid: its first deterministic layer both have, else its air's. None when
+ there is none (the gates then mix on their own times)"""
+ k = kind_of(a)
+ if k in ('modal', 'sines', 'spectral'):
+ return _anchors(*_anchor_times(a, b), w)[:2]
+ if k == 'gesture':
+ ta, tb = a.get('tones'), b.get('tones')
+ if ta and tb and ta.get('kind') == tb.get('kind'):
+ return _morph_maps(ta, tb, w)
+ return _morph_maps(a['air'], b['air'], w)
+ if k == 'hybrid':
+ la = {L['role']: L['model'] for L in a['layers']}
+ lb = {L['role']: L['model'] for L in b['layers']}
+ for role in ('tone', 'body', 'air'):
+ if role in la and role in lb and kind_of(la[role]) == kind_of(lb[role]):
+ return _morph_maps(la[role], lb[role], w)
+ return None
+
+
def _morph_modal(A, B, w):
from .modal import _rdp
SA, SB = A['strikes'], B['strikes']
k = min(len(SA), len(SB))
- wa, wb, tm = _anchors([0.0] + [s['t'] for s in SA[:k]] + [A['length_s']],
- [0.0] + [s['t'] for s in SB[:k]] + [B['length_s']], w)
+ wa, wb, tm = _anchors(*_anchor_times(A, B), w)
strikes = []
for i in range(k):
strikes.append({'t': round(float(tm[i + 1]), 6),
@@ -820,11 +1566,15 @@ def _morph_modal(A, B, w):
sa = np.asarray(A.get('side') or np.zeros(len(B['side'])), float)
sb = np.asarray(B.get('side') or np.zeros(len(A['side'])), float)
side = [round(float(v), 4) for v in (1 - w) * sa + w * sb]
+ from .sines import morph_side
+ side_t = morph_side(A.get('side_t'), B.get('side_t'), w, wa, wb)
return {'kind': 'modal', 'version': max(A['version'], B['version']), 'sr': SR,
'length_s': round(float(tm[-1]), 6), 'channels': max(A.get('channels', 1), B.get('channels', 1)),
'fade_out_s': (1 - w) * float(A.get('fade_out_s', 0.0)) + w * float(B.get('fade_out_s', 0.0)),
'strikes': strikes, 'sub': sub, 'noise': {'edges': list(na['edges']), 'h_erb': na['h_erb'], 'env': env},
- 'side': side, 'fit': {'modes': int(sum(len(s['modes']) for s in strikes))}}
+ 'side': side, **({'side_t': side_t} if side_t else {}),
+ **({'coherent': True} if A.get('coherent') or B.get('coherent') else {}),
+ 'fit': {'modes': int(sum(len(s['modes']) for s in strikes))}}
def _morph_partial(p, q, w, wa, wb):
@@ -856,11 +1606,11 @@ def _shift_partial(p, warp, gain_db):
def _morph_sines(A, B, w):
- from .sines import partial_freq, summary
+ from .sines import morph_side, partial_freq, summary
oa = list(A.get('onsets') or [A.get('onset', 0.0)])
ob = list(B.get('onsets') or [B.get('onset', 0.0)])
k = min(len(oa), len(ob))
- wa, wb, tm = _anchors([0.0] + oa[:k] + [A['duration']], [0.0] + ob[:k] + [B['duration']], w)
+ wa, wb, tm = _anchors(*_anchor_times(A, B), w)
PA, PB = A['partials'], B['partials']
en = lambda p: 10 ** (max(v for _, v in p['amp']) / 10) * (p['t1'] - p['t0'] + 0.01)
pairs, ua, ub = _assign([partial_freq(p) for p in PA], [en(p) for p in PA],
@@ -894,6 +1644,8 @@ def _morph_sines(A, B, w):
'residual': {'bands': list(ra['bands']), 'times': [float(v) for v in g],
'db': [[round(float(v), 2) for v in row] for row in lev]},
'width': (1 - w) * float(A.get('width', -120.0)) + w * float(B.get('width', -120.0)),
+ 'side': morph_side(A.get('side'), B.get('side'), w, wa, wb),
+ **({'coherent': True} if A.get('coherent') or B.get('coherent') else {}),
'channels': max(int(A.get('channels', 1)), int(B.get('channels', 1))), 'analysis': {}}
m['summary'] = summary(m)
return m
@@ -902,11 +1654,7 @@ def _morph_sines(A, B, w):
def _morph_spectral(A, B, w):
if A['bands'] != B['bands']:
raise ValueError('spectral models with different bands cannot be morphed')
-
- def peak(M):
- E = 10 ** (np.asarray(M['env']['db'], float) / 10)
- return float(M['env']['t'][int(np.argmax(E.sum(1)))])
- wa, wb, tm = _anchors([0.0, peak(A), A['dur']], [0.0, peak(B), B['dur']], w)
+ wa, wb, tm = _anchors(*_anchor_times(A, B), w)
ta, tb = wa(np.asarray(A['env']['t'], float)), wb(np.asarray(B['env']['t'], float))
g = np.unique(np.round(np.concatenate([ta, tb]), 5))
Ea, Eb = np.asarray(A['env']['db'], float), np.asarray(B['env']['db'], float)
@@ -946,7 +1694,33 @@ def _morph_spectral(A, B, w):
'bands': A['bands'], 'fit': {}, 'env': {'t': [float(v) for v in g],
'db': [[round(float(v), 2) for v in r] for r in E]},
'stereo': stereo, 'fine': fine, 'texture': tex, 'ridges': ridges,
- 'channels': max(int(A.get('channels', 1)), int(B.get('channels', 1)))}
+ 'channels': max(int(A.get('channels', 1)), int(B.get('channels', 1))),
+ **({'finish': True} if A.get('finish') or B.get('finish') else {})}
+
+
+def _morph_gesture(A, B, w):
+ """tones of one kind morph as sines or modal models do (tones only one side has, or tones of two kinds, which
+ cannot be paired: the nearer side's, faded with its weight by gesture.gain_tones), the air as spectral models
+ morph; the trajectory is measured again on the morphed air"""
+ from .gesture import gain_tones
+ from .spectral import trajectory
+ ta, tb = A.get('tones'), B.get('tones')
+ tones = None
+ if ta and tb and ta.get('kind') == tb.get('kind'):
+ tones = dict(_morph(ta, tb, w), layer='deterministic')
+ if tones['kind'] == 'modal':
+ tones['onset'] = round(float(tones['strikes'][0]['t']), 6) if tones.get('strikes') else 0.0
+ else:
+ if ta and tb: # sines tones against modal tones: the nearer side's
+ ta, tb = (ta, None) if w < 0.5 else (None, tb)
+ if (ta and w < 1) or (tb and w > 0):
+ tones = gain_tones(ta, 20 * np.log10(1 - w)) if ta else gain_tones(tb, 20 * np.log10(w))
+ air = _morph_spectral(A['air'], B['air'], w)
+ two = max(int(A.get('channels', 1)), int(B.get('channels', 1)))
+ if tones:
+ tones['channels'] = two
+ return {'kind': 'gesture', 'version': max(A['version'], B['version']), 'sr': SR, 'dur': air['dur'],
+ 'channels': two, 'source': None, 'tones': tones, 'air': air, 'traj': trajectory(air), 'fit': {}}
def _mix_texture(ta, tb, w):
@@ -971,7 +1745,10 @@ def _mix_texture(ta, tb, w):
return dom
-def _mix_post(pa, pb, w):
+def _mix_post(pa, pb, w, maps=None):
+ """the post stages of a morph's two sources mixed with weight w; maps: (a -> morph, b -> morph) time maps
+ (_morph_maps), through which each gate's breakpoints move onto the morph's timeline before they mix (a cut sits
+ where the morph's strikes and gaps are, not where its source's were)"""
if not pa and not pb:
return None
za = {'eq_db': [0.0] * 28, 'gain_db': 0.0}
@@ -985,6 +1762,10 @@ def _mix_post(pa, pb, w):
u = np.linspace(0, 1, nb)
mix = (1 - w) * np.interp(u, np.linspace(0, 1, len(ra)), ra) + w * np.interp(u, np.linspace(0, 1, len(rb)), rb)
out['pre'] = {'t_on': round(t_on, 6), 'db': [round(float(v), 1) for v in mix], 'attack_db': []}
+ if pa['pre'].get('v') == 2 and pb['pre'].get('v') == 2:
+ out['pre']['v'] = 2
+ if pa['pre'].get('hp_hz') and pb['pre'].get('hp_hz'):
+ out['pre']['hp_hz'] = max(float(pa['pre']['hp_hz']), float(pb['pre']['hp_hz']))
aa = np.asarray(pa['pre'].get('attack_db') or [], float)
ab = np.asarray(pb['pre'].get('attack_db') or [], float)
if len(aa) and len(ab):
@@ -993,6 +1774,19 @@ def _mix_post(pa, pb, w):
mix = ((1 - w) * np.interp(u, np.linspace(0, 1, len(aa)), aa)
+ w * np.interp(u, np.linspace(0, 1, len(ab)), ab))
out['pre']['attack_db'] = [round(float(v), 1) for v in mix]
+ ga, gb = (pa.get('gate') or {}).get('pts'), (pb.get('gate') or {}).get('pts')
+ if ga or gb: # the gates: cuts mix in dB (one side's alone fades with its weight)
+ A = np.asarray(ga or [[0.0, 0.0]], float).reshape(-1, 2)
+ B = np.asarray(gb or [[0.0, 0.0]], float).reshape(-1, 2)
+ if maps is not None: # each source's cuts onto the morph's timeline
+ A[:, 0], B[:, 0] = maps[0](A[:, 0]), maps[1](B[:, 0])
+ t = np.unique(np.r_[A[:, 0], B[:, 0]])
+ g = ((1 - w) * np.interp(t, A[:, 0], A[:, 1], left=0.0) + w * np.interp(t, B[:, 0], B[:, 1], left=0.0))
+ out['gate'] = {'pts': [[round(float(a), 5), round(float(b), 1)] for a, b in zip(t, g)]}
+ for key in ('space', 'bed'): # the space layer and the bed: the nearer source's
+ src = (pa if w < 0.5 else pb).get(key)
+ if src:
+ out[key] = copy.deepcopy(src)
return out
@@ -1002,12 +1796,18 @@ def morph(a, b, amount=0.5):
if w in (0.0, 1.0) and kind_of(a) == kind_of(b) != 'morph': # the endpoints are the sources, exactly
return copy.deepcopy({k: v for k, v in (a if w == 0.0 else b).items() if k != 'fit_engine'})
m = _morph(a, b, amount)
- p = _mix_post(a.get('post'), b.get('post'), w)
+ p = _mix_post(a.get('post'), b.get('post'), w, _morph_maps(a, b, w) if _gated(a, b) else None)
if p:
m['post'] = p
+ if a.get('vary_scale') is not None or b.get('vary_scale') is not None:
+ m['vary_scale'] = round((1 - w) * float(a.get('vary_scale', 1.0)) + w * float(b.get('vary_scale', 1.0)), 4)
return m
+def _gated(a, b):
+ return any(((m.get('post') or {}).get('gate') or {}).get('pts') for m in (a, b))
+
+
def _morph(a, b, amount=0.5):
"""a model between two fitted models of the same kind (0 = a, 1 = b): paired modes / partials interpolate
(frequency, decay, level geometric), unpaired ones fade with their side's weight, envelopes interpolate in dB on
@@ -1025,6 +1825,8 @@ def _morph(a, b, amount=0.5):
return _morph_sines(a, b, w)
if ka == 'spectral':
return _morph_spectral(a, b, w)
+ if ka == 'gesture':
+ return _morph_gesture(a, b, w)
la = {L['role']: L for L in a['layers']}
lb = {L['role']: L for L in b['layers']}
layers = []
@@ -1106,6 +1908,8 @@ def summary(model):
return head + F['sines'].describe(model)
if k == 'spectral':
return head + F['spectral'].summary(model)
+ if k == 'gesture':
+ return head + F['gesture'].summary(model)
if k == 'morph':
ind = lambda t: t.replace('\n', '\n ')
return (f"morph {model['amount']:.2f} between two {kind_of(model['a'])} models:\n a: "
diff --git a/earshot/forge/fit/gesture.py b/earshot/forge/fit/gesture.py
new file mode 100644
index 0000000..0c93f1d
--- /dev/null
+++ b/earshot/forge/fit/gesture.py
@@ -0,0 +1,411 @@
+"""Gesture fitter: a long designed gesture (whoosh, swish, riser, reverse swell, camera move, glitch run) -> a tones +
+air model -> fresh takes of it.
+
+ from earshot.forge.fit import gesture
+ m = gesture.fit('whoosh.wav') # or a 48 kHz array; tones=<a sines fit of the same x> reuses it
+ m = gesture.fit(x, tones=modal_fit) # the tones from a modal fit instead (modes, contact, sub)
+ y = gesture.render(m, seed=3) # float64 at 48 kHz, (n, 2) for a stereo source: new air, same tones
+ y = gesture.render(m, seed=4, length=1.2, sweep=1.3, doppler=2.0, motion=0.6, proximity=0.5, vary=0.2)
+ print(gesture.summary(m))
+ a = gesture.air(x, y_det) # the forced air layer of any deterministic render y_det: a detail
+ # noise model of what y_det leaves in x (magnitude domain)
+
+Why a model of its own: the films' whooshes, swells, risers and camera moves are not noise. They are designed: steady
+and gliding tonal lines (a reverse cymbal's partials, a riser's whistle, the bed under a camera move), short tonal
+grains, clicks, stutter gates and a noise body, panned and moving. Fitted alone, the sines fitter turns all of it into
+60-180 partials (the noise as partials: takes ring metallic) and the noise model into a smooth wash (CLAP 0.44-0.73
+on them). The hybrid of the two subtracts the partials' render from the waveform, which leaves every tone it does not
+cancel in phase in the residual, where the noise layer smears it over its band. The gesture model splits the sound in
+the magnitude domain instead:
+
+ tones the deterministic part of a sines fit (partials with their glides, strike atoms and, for a stereo source,
+ the fitted side: panned and decorrelated per band), or of a modal fit (modes, contact modes, sub wave, the
+ side over time); dropped when it carries under TONE_MIN of the energy. tones['onset'] is the first strike
+ air a detail noise model (spectral.fit over spectral.DETAIL and AIR_FIT: a 5 ms gesture kept to 0.5 dB with
+ keyframes that follow cells down to 70 dB under the loudest, so gaps and stutters stay dark; no denoise, up
+ to 100 tonal ridges, the texture statistics) of the RESIDUAL: the recording through a soft STFT
+ mask that takes the tones' magnitude out, |R|^2 = max(|X|^2 - ALPHA |T|^2, 0) per bin with the recording's
+ phase. ALPHA 3 over-subtracts: a partial's skirt and its estimation error stay out of the air, where the
+ ridges would re-make them as jittered whistles. The band envelopes carry the sweep, the per-group L/R level
+ difference and correlation per keyframe the pan and width over time, the texture (flutter, crackle) the
+ grain statistics: a take is fresh air around the same tones, not a re-jittered bank of noise partials, so
+ takes do not ring
+ traj the air's trajectory (spectral.trajectory: level, centroid, brightness, pan and width per keyframe, the peak
+ and its 20 dB rise / fall), what the motion knobs read
+ fit {'alpha', 'tones_kind', 'tone_share', 'partials' (the tones' partials or modes), 'air_keys', 'ridges',
+ 'seconds'}
+
+Measured on the 7 dev gesture clips of the replication benchmark (camera_air, glitch, reverse_swell x2, riser, swish,
+whoosh_fast; eval/forge/replicate.py's dev split, scored the way auto.fit(clap=True) scores a candidate: bed stripped,
+the post stage with the CLAP judge, seed 0 with the bed back on) against the 0.4 fit engine's choice without this
+kind: the gesture scores lower on 4 of the 7, with a higher CLAP on each (camera move 0.855 -> 0.880, reverse swells
+0.858 -> 0.935 and 0.808 -> 0.842, riser 0.871 -> 0.921), and its takes pass the take check. It loses on the short or
+bursty ones: a 0.24 s swish (CLAP 0.845 against modal's 0.884), a grainy whoosh (0.795 against the sines fit's
+0.796) and a glitch run of hard bursts (0.905 against 0.885, but a blunter first onset on the 0.125 ms sharpness
+measure costs it the score). ALPHA 3, the 5 ms gesture and the 70 dB keyframe weights against the first version's 1,
+10 ms and 50 dB: mean score 10.21 -> 9.47, CLAP 0.857 -> 0.875. Modal tones scored worse on 6 of the 7: an option,
+not the default. In the fit engine's candidate plans (auto.plan: every sound under 1 s, and second in the long
+gesture / mixed / glide / ringing plans) the gesture model won 38 of the 78 dev clips of the replication benchmark,
+short ones too (keys, pops, ticks: tones plus a little air beat the sines fit's residual noise).
+
+The fit engine's use of this module (fit/auto.py), kept stable:
+ KIND, VERSION, KNOBS (auto.KNOBS' format: name -> (search range spec, identity value, what it does); without tones
+ tone_db and vary do nothing and auto.knobs_of leaves them out)
+ fit(x, name=None, tones=None, fast=False, **opts) -> model. tones: the sines (or modal) candidate of the same x
+ (None: fit a sines model here, tones_kind='modal' a modal one; False: air only); fast=True for tests and
+ previews. model['channels'], ['dur'], ['tones'] (a deterministic sines or modal model, or None), ['air'] (a
+ spectral_noise model), ['traj']
+ render(model, seed=0, **knobs) -> float64 (n, 2) or (n,); knobs: KNOBS plus gain_db, stereo, ring (a take). The
+ length knob stretches about tones['onset'] (auto._gate_map moves a post-stage gate with it)
+ air(x, y_det=None, alpha=ALPHA, fast=False, **spectral_fit_opts) -> a spectral_noise model (the forced air layer)
+ gain_tones(det, gain_db) (a morph fading the tones only one side has), summary, save, load, check
+auto.py: KINDS, kind_of, knobs_of, channels_of, fit_kind('gesture', tones=...), _cand_fit('gesture': the tones of the
+sines candidate, so 'gesture' comes after 'sines' in ALL and in every plan), _render_raw, the post stage (a NOISE_KINDS
+member: the attack clamp high-passed and kept only where it helps; the length knob's time scale), _gate_map,
+_morph_gesture (tones of two kinds, sines and modal, cannot be interpolated: the nearer side's, faded with gain_tones),
+summary; knobs.FIT_LENGTH maps a bank entry's dur knob to `length`.
+
+render(model, seed, **knobs) knobs (identity values re-render the fit; KNOBS has the search ranges):
+ length (time-stretch: the tones about their first strike, the air with them), sweep (x the air's centroid
+ excursion), doppler (semitones of a pass-by pitch bend of the whole sound, a time warp turning at the air's peak),
+ proximity (-1..1: the air's quieter keyframes duller and quieter still), pan (x the air's L/R level differences),
+ motion (-1..1: a pass-by pan of the whole sound), width (x the tones' side and the air's width), roughness (the
+ air's texture depth), tone_db / air_db (layer levels), vary (seeded jitter of the tones; the air is new noise on
+ every seed anyway), gain_db, stereo (None = the source's channels), ring (a take: partials ring out with their
+ release, the air decays 60 dB over at least RING_S past the clip's end, and the end gets a 10 ms fade instead of
+ stopping on the recording's last sample).
+Pure numpy / scipy. Deterministic: render(model, seed, knobs) is a pure function.
+"""
+import copy
+import json
+import os
+
+import numpy as np
+
+KIND = 'gesture'
+VERSION = 1
+SR = 48000
+ALPHA = 3.0 # mask over-subtraction: |R|^2 = max(|X|^2 - ALPHA |T|^2, 0)
+AIR_FIT = {'gesture_ms': 5.0, 'quiet_db': 70.0, # the air's spectral.fit options over spectral.DETAIL: a 5 ms
+ 'grain_norm_ms': 5.0, 'grain_halves': True} # gesture, keyframes that follow cells down to 70 dB under the
+ # loudest (the gaps stay dark); a grain texture levelled over 5 ms (>= 20 clicks) and only
+ # when stationary (0.4 round C2: short noise-led fits, keys and ticks fitted as air, had takes
+ # whose CLAP spread 0.84-0.92 from seed to seed because one draw put its loudest grains where
+ # the sound is quiet; on 20 dev clips the takes' median gap under seed 0 0.046 -> 0.031, 'CLAP
+ # drops > 0.06' 7 -> 5, grades 9/6/5/0 -> 13/3/4/0)
+MASK_FFT = 1024 # the mask's STFT (hop a quarter)
+TONE_MIN = 0.05 # tones under this share of the energy are dropped (a pure noise gesture)
+MIN_PARTIAL_MS = 0.0 # partials shorter than this (birth to release end) are left to the air
+FADE_S = 0.010 # a take's end fade (ring=True)
+RING_S = 0.06 # a take's air decays 60 dB over at least this past the clip's end (ring=True)
+
+# knob -> (search range spec for a forge SPACE, identity value, what it does): the format of auto.KNOBS
+KNOBS = {
+ 'length': ((0.8, 1.25, 'log'), 1.0, 'time-stretch of the gesture (tones about their first strike)'),
+ 'sweep': ((0.7, 1.43, 'log'), 1.0, "x the air's spectral centroid excursion"),
+ 'doppler': ((-2.0, 2.0), 0.0, 'semitones of a pass-by pitch bend turning at the peak (+ = approaching higher)'),
+ 'proximity': ((-0.5, 1.0), 0.0, 'quieter keyframes duller and quieter still (closer feel above 0)'),
+ 'pan': ((0.5, 1.5), 1.0, "x the air's L/R level differences"),
+ 'motion': ((-0.6, 0.6), 0.0, 'a pass-by pan of the whole sound (-1 left .. +1 right)'),
+ 'width': ((0.5, 1.5), 1.0, "x the tones' side and the air's width"),
+ 'roughness': ((0.5, 1.5), 1.0, "the air's texture depth (flutter, crackle)"),
+ 'tone_db': ((-6.0, 6.0), 0.0, 'level of the tones'),
+ 'air_db': ((-6.0, 6.0), 0.0, 'level of the air'),
+ 'vary': ((0.0, 0.4), 0.0, 'seeded jitter of the tones: new takes of the same gesture'),
+}
+
+
+def _load(x):
+ from . import quality as Q
+ X = Q.load(x)
+ if X.ndim == 1:
+ X = np.stack([X, X], 1)
+ elif X.shape[1] == 1:
+ X = np.repeat(X, 2, 1)
+ return np.asarray(X[:, :2], np.float64)
+
+
+def _two(y, n):
+ """y (mono or stereo, any length) as (n, 2)"""
+ y = np.asarray(y, np.float64)
+ if y.ndim == 1:
+ y = np.stack([y, y], 1)
+ out = np.zeros((n, 2))
+ k = min(n, len(y))
+ out[:k] = y[:k, :2]
+ return out
+
+
+def check(model):
+ if not isinstance(model, dict) or model.get('kind') != KIND:
+ raise ValueError(f"not a {KIND} model (kind={model.get('kind') if isinstance(model, dict) else None!r})")
+ if int(model.get('version', 0)) > VERSION:
+ raise ValueError(f"model version {model.get('version')} is newer than this renderer ({VERSION})")
+
+
+# ---------------------------------------------------------------------------------------------------- split
+
+
+def residual(X, Y, alpha=ALPHA, n_fft=MASK_FFT):
+ """X minus Y in the STFT magnitude domain, per channel: X through the soft mask sqrt(max(1 - alpha |Y|^2 / |X|^2,
+ 0)) with X's phase (Hann, hop n_fft / 4). Where Y explains a bin the bin goes; where Y is quiet X stays as it is;
+ nothing Y has and X has not is ever added (a waveform subtraction adds every tone it does not cancel in phase)"""
+ from scipy import signal
+ X = np.asarray(X, np.float64)
+ Y = np.asarray(Y, np.float64)
+ one = X.ndim == 1
+ X2 = X[:, None] if one else X
+ Y2 = (Y[:, None] if Y.ndim == 1 else Y)
+ n = len(X2)
+ kw = dict(fs=SR, window='hann', nperseg=n_fft, noverlap=n_fft - n_fft // 4)
+ out = np.zeros_like(X2)
+ for c in range(X2.shape[1]):
+ _, _, ZX = signal.stft(X2[:, c], **kw)
+ yc = Y2[:n, min(c, Y2.shape[1] - 1)]
+ _, _, ZY = signal.stft(np.pad(yc, (0, n - len(yc))), **kw)
+ g = np.sqrt(np.clip(1.0 - alpha * np.abs(ZY) ** 2 / (np.abs(ZX) ** 2 + 1e-30), 0.0, 1.0))
+ _, r = signal.istft(ZX * g, **kw)
+ out[:min(n, len(r)), c] = r[:n]
+ return out[:, 0] if one else out
+
+
+def _deterministic(ms, two, min_ms=0.0):
+ """the tones of a sines or modal model, without their noise: partials + strike atoms (sines; partials that last
+ under min_ms, birth to release end, are left to the air) or modes + contact + sub (modal); the fitted side kept for
+ a stereo source"""
+ m = copy.deepcopy({k: v for k, v in ms.items() if k not in ('post', 'fit_engine')})
+ m['layer'] = 'deterministic'
+ m['channels'] = 2 if two else 1
+ if m.get('kind') == 'modal':
+ L = float(m['length_s'])
+ m['noise'] = dict(m['noise'], env=[[[0.0, -240.0], [L, -240.0]] for _ in m['noise']['env']])
+ m['onset'] = round(float(m['strikes'][0]['t']), 6) if m.get('strikes') else 0.0
+ if not two:
+ m['side'] = None
+ m.pop('side_t', None)
+ return m
+ if min_ms > 0:
+ m['partials'] = [p for p in m['partials'] if (p['t1'] + p['rel'] - p['t0']) * 1000.0 >= min_ms]
+ m['residual'] = {'bands': [1000.0], 'times': [0.0, float(m['duration'])], 'db': [[-150.0, -150.0]]}
+ if not two:
+ m['side'] = None
+ return m
+
+
+def _tones(det, seed, two, n=None, stretch=1.0, vary=0.0, width=1.0, gain_db=0.0, ring=False):
+ """a render of the tones: (n, 2) when n is given, else as the fitter renders it. stretch: time factor about the
+ first strike (a modal model: strike spacing x stretch, decays x stretch)"""
+ if det.get('kind') == 'modal':
+ from . import modal
+ s = max(float(stretch), 1e-3)
+ y = modal.render(det, seed, spacing=s, damping=1.0 / s, vary=float(vary), width=float(width),
+ gain=float(gain_db), ring=bool(ring), stereo=bool(two))
+ else:
+ from . import sines
+ y = sines.render(det, seed, knobs=dict(stretch=float(stretch), vary=float(vary), width=float(width),
+ gain_db=float(gain_db), ring=bool(ring), stereo=bool(two)))
+ y = np.asarray(y, np.float64)
+ return y if n is None else _two(y, n)
+
+
+def gain_tones(det, gain_db):
+ """a copy of a gesture's tones (a deterministic sines or modal model) gain_db louder: sines partial levels and
+ strike atoms; modal modes, contact modes and the sub wave (a morph fades the tones only one side has with it)"""
+ t = copy.deepcopy(det)
+ g = 10 ** (float(gain_db) / 20)
+ if t.get('kind') == 'modal':
+ for st in t.get('strikes') or ():
+ for key in ('modes', 'attack'):
+ rows = np.asarray(st.get(key) or [], float)
+ if rows.size:
+ nested = rows.ndim == 2
+ rows = rows.reshape(-1, 4)
+ rows[:, 2] *= g
+ st[key] = [[float(v) for v in r] for r in rows] if nested else [float(v) for v in rows.ravel()]
+ if t.get('sub'):
+ t['sub'] = dict(t['sub'], x=[float(v) * g for v in t['sub']['x']])
+ return t
+ for p in t['partials']:
+ p['amp'] = [[a, v + float(gain_db)] for a, v in p['amp']]
+ for tr in t.get('transients', []):
+ tr['atoms'] = [[f, tau, a * g, ph] for f, tau, a, ph in tr['atoms']]
+ return t
+
+
+def _n_tones(det):
+ """how many tones a deterministic model holds: partials (sines) or modes over every strike (modal)"""
+ if not det:
+ return 0
+ if det.get('kind') == 'modal':
+ return int(sum(np.asarray(s['modes'], float).reshape(-1, 4).shape[0] for s in det.get('strikes') or ()))
+ return len(det['partials'])
+
+
+DETAIL_FAST = dict(iters=1, texture=False, seeds=(1,))
+
+
+def _air_tail(am, tail_s):
+ """a copy of an air model that goes on for tail_s past its end, falling 60 dB (linear in dB) from its last
+ keyframe: a take's air decays under the ringing tones"""
+ am = dict(am)
+ t_end = float(am['dur']) + float(tail_s)
+ env = am['env']
+ am['env'] = {'t': list(env['t']) + [round(t_end, 4)], 'db': list(env['db']) + [[v - 60.0 for v in env['db'][-1]]]}
+ if am.get('stereo'):
+ st = am['stereo']
+ am['stereo'] = dict(st, ild=list(st['ild']) + [st['ild'][-1]], icc=list(st['icc']) + [st['icc'][-1]])
+ am['dur'] = round(t_end, 4)
+ return am
+
+
+def air(x, y_det=None, alpha=ALPHA, fast=False, name=None, mask_fft=MASK_FFT, **kw):
+ """the air layer of a gesture or a hybrid: a detail noise model (spectral.DETAIL; kw override it) of x, or of what
+ the deterministic render y_det leaves in x (residual(): the magnitude domain, never a waveform subtraction). The
+ forced air layer of a hybrid: fitted even when the deterministic layers explain almost all of x (the air under a
+ tone is what CLAP and ears miss first)"""
+ from . import spectral
+ X = _load(x)
+ R = X if y_det is None else residual(X, _two(y_det, len(X)), alpha, mask_fft)
+ opts = dict(spectral.DETAIL, **AIR_FIT)
+ if fast:
+ opts.update(DETAIL_FAST)
+ opts.update(kw)
+ m = spectral.fit(R, name=name, **opts)
+ m['channels'] = 1 if np.allclose(X[:, 0], X[:, 1]) else 2
+ return m
+
+
+# ------------------------------------------------------------------------------------------------------ fit
+
+
+def fit(x, name=None, tones=None, alpha=ALPHA, tone_min=TONE_MIN, min_partial_ms=MIN_PARTIAL_MS, fast=False,
+ tones_kind='sines', verbose=False, **air_kw):
+ """a gesture model of x (path or 48 kHz array, mono or (n, 2)). tones: a sines or modal model of the same x to take
+ the tones from (the fit engine passes its candidate of that kind), None = fit one here (tones_kind 'sines' or
+ 'modal'), False = air only. Partials shorter than min_partial_ms go to the air. air_kw: options for the air's
+ spectral.fit (over AIR_FIT, over spectral.DETAIL)"""
+ import time
+ from . import spectral
+ if name is None and isinstance(x, (str, os.PathLike)):
+ name = os.path.basename(str(x))
+ t0 = time.time()
+ X = _load(x)
+ n = len(X)
+ two = not np.allclose(X[:, 0], X[:, 1])
+ e_x = float((X ** 2).sum()) + 1e-30
+ det, share, yt = None, 0.0, None
+ if tones is not False:
+ ms = tones if isinstance(tones, dict) else _fit_tones(X if two else X[:, 0], tones_kind, fast)
+ det = _deterministic(ms, two, min_partial_ms)
+ yt = _tones(det, 0, two, n)
+ share = float((yt ** 2).sum()) / e_x
+ if share < tone_min:
+ det, yt = None, None
+ t1 = time.time()
+ am = air(X, yt, alpha, fast=fast, name=name, **air_kw)
+ m = {'kind': KIND, 'version': VERSION, 'sr': SR, 'dur': round(n / SR, 6), 'channels': 2 if two else 1,
+ 'source': name, 'tones': det, 'air': am, 'traj': spectral.trajectory(am),
+ 'fit': {'alpha': alpha, 'tones_kind': det.get('kind') if det else None, 'tone_share': round(share, 4),
+ 'partials': _n_tones(det), 'air_keys': len(am['env']['t']), 'ridges': len(am.get('ridges') or ()),
+ 'seconds': {'tones': round(t1 - t0, 2), 'air': round(time.time() - t1, 2)}}}
+ if verbose:
+ print(' ' + summary(m))
+ return m
+
+
+def _fit_tones(x, kind, fast):
+ if kind == 'modal':
+ from . import modal
+ return modal.fit(x, **({'passes': 1} if fast else {}))
+ if kind != 'sines':
+ raise ValueError(f"tones_kind {kind!r}: 'sines' or 'modal'")
+ from . import sines
+ return sines.fit(x, **({'windows': (1024,)} if fast else {}))
+
+
+# --------------------------------------------------------------------------------------------------- render
+
+
+def render(model, seed=0, length=1.0, sweep=1.0, doppler=0.0, proximity=0.0, pan=1.0, motion=0.0, width=1.0,
+ roughness=1.0, tone_db=0.0, air_db=0.0, vary=0.0, gain_db=0.0, stereo=None, ring=False):
+ """the sound of a gesture model (float64, 48 kHz): (n, 2) for a stereo source (or stereo=True), else (n,). See the
+ module doc for the knobs; identity values re-render the fit (seed 0 = the fit as measured)"""
+ from . import spectral
+ check(model)
+ two = (model.get('channels', 1) == 2) if stereo is None else bool(stereo)
+ s = max(float(length), 1e-3)
+ n = max(64, int(round(model['dur'] * s * SR)))
+ det = model.get('tones')
+ t = None
+ if det:
+ t = _tones(det, seed, two, None, stretch=s, vary=float(vary), width=float(width), gain_db=float(tone_db),
+ ring=bool(ring))
+ am = model['air']
+ if ring: # a take: the air decays under the ringing tones instead of stopping on the clip's end
+ am = _air_tail(am, max(RING_S, (len(t) / SR / s - model['dur']) if t is not None else 0.0))
+ a = np.asarray(spectral.render(am, (int(seed) * 7 + 101) & 0x7FFFFFFF, length=s, sweep=sweep,
+ roughness=roughness, width=width, pan=pan, proximity=proximity, gain_db=air_db,
+ limit=False), np.float64)
+ if t is not None:
+ # the tones stretch about their first strike, the air from 0: move the air onto the tones
+ sh = int(round(float(det.get('onset', 0.0)) * (1.0 - s) * SR))
+ a = np.concatenate([np.zeros((sh, 2)), a]) if sh > 0 else a[-sh:]
+ m = max(n, len(t), len(a)) if ring else n
+ y = _two(a, m) + _two(t, m)
+ else:
+ y = _two(a, max(n, len(a)) if ring else n)
+ if doppler or motion:
+ tr = model['traj']
+ t_pk, tau = tr['peak_s'] * s, spectral._motion_tau(tr, s)
+ y = spectral.pass_pan(spectral.bend(y, t_pk, tau, doppler), t_pk, tau, motion)
+ if gain_db:
+ y = y * 10 ** (float(gain_db) / 20)
+ if ring:
+ k = min(len(y) // 3, int(FADE_S * SR))
+ if k > 1:
+ y[-k:] *= (0.5 + 0.5 * np.cos(np.linspace(0, np.pi, k)))[:, None]
+ return y if two else y.mean(1)
+
+
+# ---------------------------------------------------------------------------------------------- words, io
+
+
+def summary(model):
+ """one paragraph: length, tones, air, trajectory, size"""
+ from . import spectral
+ check(model)
+ det = model.get('tones')
+ f = model.get('fit') or {}
+ tr = model['traj']
+ lv = np.asarray(tr['level_db'])
+ pk = int(np.argmax(lv))
+ w = [f"{model.get('source') or 'gesture'}: {model['dur']:.2f} s gesture"]
+ if det and det.get('kind') == 'modal':
+ fr = [float(v) for st in det['strikes'] for v in np.asarray(st['modes'], float).reshape(-1, 4)[:, 0]] or [0.0]
+ w.append(f"tones: modal, {len(det['strikes'])} strike(s), {len(fr)} modes {min(fr):.0f}-{max(fr):.0f} Hz "
+ f"({100 * f.get('tone_share', 0.0):.0f} % of the energy)")
+ elif det:
+ fr = [p['freq'][0][1] for p in det['partials']] or [0.0]
+ w.append(f"tones: {len(det['partials'])} partials {min(fr):.0f}-{max(fr):.0f} Hz "
+ f"({100 * f.get('tone_share', 0.0):.0f} % of the energy)")
+ else:
+ w.append('no tones (air only)')
+ w.append('air: ' + spectral.summary(model['air']).split(': ', 1)[-1])
+ w.append(f"peak at {tr['peak_s']:.2f} s (rise {1000 * tr['rise_s']:.0f} ms, fall {1000 * tr['fall_s']:.0f} ms), "
+ f"centroid {tr['centroid_hz'][0]:.0f} -> {tr['centroid_hz'][pk]:.0f} -> {tr['centroid_hz'][-1]:.0f} Hz, "
+ f"pan {min(tr['pan_db']):+.1f}..{max(tr['pan_db']):+.1f} dB, width {np.median(tr['width']):.2f}")
+ w.append(f"{len(json.dumps(model)) / 1024:.0f} kB JSON")
+ return '; '.join(w)
+
+
+def save(model, path):
+ with open(path, 'w', encoding='utf-8') as fh:
+ json.dump(model, fh, separators=(',', ':'))
+ return path
+
+
+def load(path):
+ with open(path, encoding='utf-8') as fh:
+ m = json.load(fh)
+ check(m)
+ return m
diff --git a/earshot/forge/fit/modal.py b/earshot/forge/fit/modal.py
index 56baa0d..d912ef3 100644
--- a/earshot/forge/fit/modal.py
+++ b/earshot/forge/fit/modal.py
@@ -19,7 +19,13 @@ What the model holds (a plain JSON-serialisable dict, kind 'modal', version 1):
crossovers on the ERB-rate axis, `edges` Hz) x breakpoints [t s, dB] (power, linear in dB between breakpoints).
Each band's power is the original's minus the deterministic part's, so modes + noise keep the band energy.
- fade_out_s: the recording's linear fade-out (estimated; the modes are fitted un-faded and faded at render).
-- side: per noise band, the side / mid amplitude ratio of a stereo original (None for mono).
+- side: per noise band, the side / mid amplitude ratio of a stereo original (None for mono): one ratio for the whole
+ sound (models from before 0.4 render with it).
+- side_t (0.4, stereo only): the side over time, sines.side_model of the original: per octave band at keyframes
+ after the onset, 'pan' (the side's part coherent with the mid, signed) and 'diffuse' (the rest, re the mid). A
+ static ratio set by a wide attack left the tails too wide (a pop's tail L/R correlation 0.30 against 0.72).
+- coherent (0.4: True on every fit): vary and the knobs keep the structure the fit gave the modes (render below).
+ Models without it, the 0.3 bank's fit entries among them, render exactly as before.
- fit: diagnostics (mode count, the modes' energy share, the residual's level).
How it fits (`fit`):
@@ -38,15 +44,25 @@ How it fits (`fit`):
4. per strike the attack ramp that leaves the least residual in the first ms; the sub wave; the contact modes
(full-rate ESPRIT on the first 2.5 ms of the residual);
5. band envelopes (analytic-signal power; 0.5 ms smoothing from 8 ms before to 15 ms after onsets, 6 ms after,
- 30 ms in long tails), simplified to breakpoints (Ramer-Douglas-Peucker, 1 dB near strikes, 2 dB in tails).
+ 30 ms in long tails; over the PRE_CAP_S before an onset no band rises above its level just before it),
+ simplified to breakpoints (Ramer-Douglas-Peucker, 1 dB near strikes, 2 dB in tails).
render(model, seed, knobs): size (x: mode frequencies / size, decays * size, the noise spectrum shifted with them),
damping (decays / damping, noise tails shortened), brightness (dB / octave around 1 kHz on modes and noise),
hardness (x: a shorter contact and +4 dB / octave per doubling on the modes and the attack noise), noise (dB on
the noise bands), width (x side), room (0..1 wet of a small stochastic room), vary (0..1: seeded jitter of mode
-frequencies / decays / amplitudes, strike timing and noise bands: new takes of the same object), spacing (x the
-time between strikes), gain (dB). seed changes only the noise (and, with vary > 0, the jitter). The side of a
-stereo render is the mid's Hilbert transform shaped per band by `side`.
+frequencies / decays / amplitudes and noise bands: new takes of the same object), spacing (x the time between
+strikes), gain (dB). seed changes only the noise (and, with vary > 0, the jitter). A `coherent` model (0.4) keeps the
+structure the least squares fit gave its modes (close modes, and a later strike's modes, shape and cancel the earlier
+ones' ring): its jitter is a smooth random function of log frequency (6 cosines of 0.25-1.25 cycles per octave), the
+same for every strike, its strikes keep their times, and when a knob moves a strike or a mode's frequency (vary, size,
+spacing) every mode keeps its phase on the first strike's timeline. Without it (models fitted before 0.4) every mode
+is detuned on its own (0.6 % x vary) and the strike times jitter (2 ms x vary): even 0.1 % undid the cancellations, so
+a key's takes rang on 2.5x the recording's tail (on the films' modal and hybrid dev fits the takes' CLAP to the
+recording was 0.75-0.87 against 0.89-0.94 for vary 0; coherent, it is as for vary 0).
+The side of a stereo render is made from the mid: with `side_t` per octave band the band's mid times its pan gain plus
+the band's Hilbert transform times its diffuse gain, both linear between keyframes (which move with the strikes and
+stretch with the decays, the bands with the modes); else the mid's Hilbert transform shaped per band by `side`.
ring: a take that rings out (default None: on when vary > 0; off, the render keeps the recording's length and
fade-out, what the fit is measured against). A trimmed recording stops while its modes still ring (a key's release
70 ms before the cut), and vary lengthens some decays further; with ring the render goes on until every mode is
@@ -54,7 +70,9 @@ RING_DB (60 dB) under the loudest strike's mode sum, at least RING_MIN_S (RING_L
above RING_LOUD_DB at the cut) and at most RING_EXT x the sound's span longer, and the added stretch fades 60 dB
(linear in dB) instead of the recording's fade-out (the noise keeps falling at its last slope). A fitted model often
holds a few faint poles with decays of seconds (noise the fitter took for modes); the cap keeps a take short, the
-fade keeps it from stopping on them. The fit engine's takes (auto.render(take=True), the bank) ring out.
+fade keeps it from stopping on them. The fit engine's takes (auto.render(take=True), the bank) ring out. A coherent
+model's ring-out continues its noise un-faded too, falling at its slope from before the recording's fade-out (from the
+fade, the modes rang on alone in the added stretch: a comb in the tail).
What it is not for: glides / chirps (a rising pop, a zip) split into many false strikes; designed impacts with long
reverb tails fit, but the tail is mostly noise bands.
Pure numpy / scipy.
@@ -82,6 +100,16 @@ RING_EXT = 0.5 # mode sum, at most this x the sound's span (first st
RING_MIN_S = 0.06 # least this much longer; the added stretch fades RING_DB (linear in dB)
RING_LOUD_DB = -50.0 # modes still above this at the recording's end (re the loudest strike's mode sum) ...
RING_LOUD_S = 0.2 # ... get at least this long a fade (a fast fade from a loud ring reads as a cut)
+STRIKE_RAMP_S = 0.00015 # a later strike's modes and contact start over this raised-cosine ramp (7 samples): from
+ # full amplitude they put a step into the ring of the strikes before it, which the audit
+ # hears as a click (an impact's 4 takes and its fit clicked at two later strikes). The first
+ # strike (the onset) keeps its step; a model without 'coherent' (fitted before 0.4: the
+ # bank's fit entries) renders as before
+RAMPS_MS = (0.0, 0.05, 0.1, 0.2, 0.35, 0.5, 0.8, 1.2) # the attack ramps a strike's modes may rise over (ms)
+PRE_CAP_S = 0.002 # the noise bands may not rise over this long before an onset (0.4; 0 = as before): the
+ # analysis smeared a strike's noise up to 1.5 ms ahead of it, 10-20 dB over the recording,
+ # the main cause of blunt modal onsets (on the films' 27 modal dev clips the onset error
+ # fell 0.39 -> 0.25 octave, a glockenspiel note's 3.6 -> 0.1; CLAP and mel unchanged)
# ------------------------------------------------------------------------------------------------ helpers
@@ -534,11 +562,12 @@ def _rdp(t, v, tol):
return np.nonzero(keep)[0]
-def _fit_noise(m, det, ons, tol_db=1.0, edges=NOISE_EDGES, h=NOISE_H):
+def _fit_noise(m, det, ons, tol_db=1.0, edges=NOISE_EDGES, h=NOISE_H, pre_cap=0.0):
"""band envelopes of the stochastic part: per band, the original's power minus the deterministic part's power
(analytic-signal power, 0.5 ms smoothing from 8 ms before to 15 ms after onsets, 6 ms elsewhere), so the
rendered modes + noise keep the original's band energy even where the modes misfit; a list per band of [t, dB]
- breakpoints"""
+ breakpoints. pre_cap (s): over this long before each onset a band stays at its level just before (no rise ahead
+ of the strike)"""
n = len(m)
n_fft = sfft.next_fast_len(n + 16384)
W = _masks(n_fft, edges, h)
@@ -572,6 +601,15 @@ def _fit_noise(m, det, ons, tol_db=1.0, edges=NOISE_EDGES, h=NOISE_H):
po, pd = bandpow(R, b), bandpow(Rd, b)
P_all.append(np.maximum(po - pd, 1e-6 * po))
P_all = np.array(P_all)
+ if pre_cap > 0: # the analysis smears a strike's energy ahead of it (the 0.5 ms
+ step = int(GRID * SR) # smoothing, the band filters' pre-ring): over the pre_cap s
+ k_cap = int(round(pre_cap / GRID)) # before an onset no band may rise above its level just before
+ for o in ons:
+ io = int(o // step)
+ a = io - k_cap
+ if a >= 1:
+ ref = P_all[:, max(0, a - 2):a + 1].min(axis=1)
+ P_all[:, a:io + 1] = np.minimum(P_all[:, a:io + 1], ref[:, None])
fl = 10 * np.log10(P_all.max() + 1e-30) - 100.0
env = []
for b in range(len(W)):
@@ -585,7 +623,9 @@ def _fit_noise(m, det, ons, tol_db=1.0, edges=NOISE_EDGES, h=NOISE_H):
def fit(x, *args, **kw):
"""fit a modal model to a struck / ringing recording (path or 48 kHz array); returns the model dict. Options:
max_strikes, kmax (ESPRIT order cap per band), nmax (window, decimated samples), passes, skip_ms (the LS starts
- this long after each onset), prune_db, max_modes (per strike), onset_rel, noise_tol_db, analysis_s, verbose"""
+ this long after each onset), prune_db, max_modes (per strike), onset_rel, noise_tol_db, analysis_s, verbose,
+ ramps (the attack ramps tried, ms), pre_cap_s (no noise rise this long before an onset), side_t (a stereo
+ source: the side over time, model['side_t'])"""
try: # small matrices: one BLAS thread is ~5x faster than a contended pool
from threadpoolctl import threadpool_limits
except ImportError:
@@ -595,7 +635,8 @@ def fit(x, *args, **kw):
def _fit(x, max_strikes=10, kmax=16, nmax=384, passes=2, skip_ms=0.5, prune_db=-50.0, max_modes=80, onset_rel=0.2,
- noise_tol_db=1.0, analysis_s=3.0, attack_ms=2.5, verbose=False):
+ noise_tol_db=1.0, analysis_s=3.0, attack_ms=2.5, verbose=False, ramps=RAMPS_MS, pre_cap_s=PRE_CAP_S,
+ side_t=True):
X = _load(x)
m_orig = X.mean(axis=1)
n = len(m_orig)
@@ -704,7 +745,7 @@ def _fit(x, max_strikes=10, kmax=16, nmax=384, passes=2, skip_ms=0.5, prune_db=-
base = m - mode_sig
w0, w1 = o, min(n, o + skip + int(0.0015 * SR))
best, rb = np.inf, 0.0
- for rms in (0.0, 0.05, 0.1, 0.2, 0.35, 0.5, 0.8, 1.2):
+ for rms in ramps:
y = _synth_strike(w1 + int(0.002 * SR), o, fq, tq, aq, pq, rms / 1000) if rows else None
err = np.sum((base[w0:w1] - (y[w0:w1] if y is not None else 0)) ** 2)
if err < best * (1 - 1e-9):
@@ -734,8 +775,8 @@ def _fit(x, max_strikes=10, kmax=16, nmax=384, passes=2, skip_ms=0.5, prune_db=-
model = {'kind': KIND, 'version': VERSION, 'sr': SR, 'length_s': round(n / SR, 6), 'channels': int(X.shape[1]),
'fade_out_s': fo, 'strikes': strikes, 'sub': sub,
'noise': {'edges': list(NOISE_EDGES), 'h_erb': NOISE_H,
- 'env': _fit_noise(m_orig, det, ons, noise_tol_db)},
- 'side': None}
+ 'env': _fit_noise(m_orig, det, ons, noise_tol_db, pre_cap=pre_cap_s)},
+ 'side': None, 'coherent': True}
if X.shape[1] == 2:
side = 0.5 * (X[:, 0] - X[:, 1])
n_fft = sfft.next_fast_len(n)
@@ -743,6 +784,9 @@ def _fit(x, max_strikes=10, kmax=16, nmax=384, passes=2, skip_ms=0.5, prune_db=-
Ms, Ss = np.abs(np.fft.rfft(m_orig, n_fft)) ** 2, np.abs(np.fft.rfft(side, n_fft)) ** 2
rho = np.sqrt(((W ** 2) @ Ss + 1e-30) / ((W ** 2) @ Ms + 1e-30))
model['side'] = [round(float(min(v, 1.5)), 4) for v in rho]
+ if side_t:
+ from .sines import side_model
+ model['side_t'] = side_model(X)
pm = float(np.sum(m_orig ** 2)) + 1e-30
model['fit'] = {'skip_ms': skip_ms, 'modes': int(sum(len(s['modes']) for s in strikes)),
'mode_share_db': round(float(10 * np.log10(np.sum(mode_sig ** 2) / pm + 1e-30)), 2),
@@ -758,6 +802,7 @@ def _plan(model, seed, size, damping, brightness, hardness, noise, vary, spacing
envelope (bands x grid, dB) on a GRID-second grid, the band centres and the output length (samples). ring: when
a mode outlives the recording's length, render on until every mode is RING_DB down (see render)"""
rv = np.random.default_rng([int(seed) & 0xFFFFFFFF, 1])
+ coherent = bool(model.get('coherent')) # a 0.4 fit (module doc); older models keep the per-mode jitter
st = model['strikes']
t0 = [s['t'] for s in st]
tn = []
@@ -765,17 +810,32 @@ def _plan(model, seed, size, damping, brightness, hardness, noise, vary, spacing
if i == 0:
tn.append(t)
else:
- gap = (t - t0[i - 1]) * spacing + vary * 0.002 * rv.standard_normal()
+ gap = (t - t0[i - 1]) * spacing + (0.0 if coherent else vary * 0.002 * rv.standard_normal())
tn.append(tn[-1] + max(0.003, gap))
+
+ def field(): # a smooth random function of log frequency (unit variance)
+ c, p = rv.uniform(0.25, 1.25, 6), rv.uniform(0.0, 2 * np.pi, 6)
+ return lambda fr: np.sqrt(1 / 3) * np.cos(2 * np.pi * np.log2(np.maximum(fr, 20.0))[:, None] * c + p).sum(1)
+ if coherent:
+ jf, jt, ja = field(), field(), field()
+ shifted = coherent and (vary > 0 or size != 1.0 or spacing != 1.0)
stretch = max(size / damping, 1e-3)
tilt = brightness + 4.0 * np.log2(max(hardness, 1e-3))
strikes = []
for i, s in enumerate(st):
md = np.array(s['modes'], float).reshape(-1, 4)
+ f0 = md[:, 0]
k = len(md)
- f = md[:, 0] / size * (1 + 0.006 * vary * rv.standard_normal(k))
- tau = md[:, 1] * stretch * np.exp(0.12 * vary * rv.standard_normal(k))
- a = md[:, 2] * 10 ** ((tilt * np.log2(np.maximum(f, 20.0) / 1000.0) + 1.5 * vary * rv.standard_normal(k)) / 20)
+ if coherent:
+ zf, zt, za = jf(f0), jt(f0), ja(f0)
+ else:
+ zf, zt, za = rv.standard_normal(k), rv.standard_normal(k), rv.standard_normal(k)
+ f = f0 / size * (1 + 0.006 * vary * zf)
+ tau = md[:, 1] * stretch * np.exp(0.12 * vary * zt)
+ a = md[:, 2] * 10 ** ((tilt * np.log2(np.maximum(f, 20.0) / 1000.0) + 1.5 * vary * za) / 20)
+ ph = md[:, 3]
+ if shifted and i > 0: # every mode keeps its phase on the first strike's timeline
+ ph = ph + 2 * np.pi * (f * (tn[i] - tn[0]) - f0 * (t0[i] - t0[0]))
keep = f < 0.47 * SR
at = np.array(s.get('attack', []), float).reshape(-1, 4)
fa = at[:, 0] / size
@@ -783,7 +843,7 @@ def _plan(model, seed, size, damping, brightness, hardness, noise, vary, spacing
aa = at[:, 2] * 10 ** ((tilt * np.log2(np.maximum(fa, 20.0) / 1000.0)) / 20)
ka = fa < 0.47 * SR
strikes.append((tn[i], s['ramp_ms'] / 1000.0 / max(hardness, 1e-3), f[keep], tau[keep], a[keep],
- md[keep, 3], fa[ka], ta[ka], aa[ka], at[ka, 3]))
+ ph[keep], fa[ka], ta[ka], aa[ka], at[ka, 3]))
L0 = model['length_s']
t_last0, t_last = t0[-1], tn[-1]
length = t_last + (L0 - t_last0) * max(1.0, stretch)
@@ -811,7 +871,8 @@ def _plan(model, seed, size, damping, brightness, hardness, noise, vary, spacing
rel = g_new - tn_a[seg]
rel_old = np.where(rel < hold, rel, hold + (rel - hold) / stretch)
told = t0_a[seg] + rel_old
- end = np.append(t0_a[1:], L0)[seg]
+ # a coherent model's ring-out goes on un-faded like its modes (module doc)
+ end = np.append(t0_a[1:], L0 - (float(model.get('fade_out_s', 0.0)) if ring and coherent else 0.0))[seg]
new_len = np.append(tn_a[1:] - tn_a[:-1], 1e9)[seg]
old_len = np.append(t0_a[1:] - t0_a[:-1], 1e9)[seg]
moved = (stretch != 1.0) | (np.abs(new_len - old_len) > 1e-9)
@@ -841,7 +902,8 @@ def _plan(model, seed, size, damping, brightness, hardness, noise, vary, spacing
sub = model.get('sub')
sub = (int(round((tn[0] - t0[0]) * SR)), sub['x']) if sub else None
return {'strikes': strikes, 'E': E, 'cen': cen, 'edges': edges, 'h': nz['h_erb'], 'n': n_out, 'sub': sub,
- 'fade': 0.0 if ring else float(model.get('fade_out_s', 0.0)), 'ring': ring or None}
+ 'fade': 0.0 if ring else float(model.get('fade_out_s', 0.0)), 'ring': ring or None, 'told': told,
+ 'coherent': coherent}
def _ring_end(strikes, t_end):
@@ -901,7 +963,7 @@ def _ring(L, f, tau, a, ph, block=4096):
def _modes(plan, n):
"""the strikes' modes (with their contact ramps and contact modes) and the sub wave, summed, faded"""
y = np.zeros(n)
- for t, ramp, f, tau, a, ph, fa, ta, aa, pa in plan['strikes']:
+ for i, (t, ramp, f, tau, a, ph, fa, ta, aa, pa) in enumerate(plan['strikes']):
o = int(round(t * SR))
if o >= n:
continue
@@ -911,6 +973,10 @@ def _modes(plan, n):
k = min(R, len(ys))
ys[:k] *= 0.5 - 0.5 * np.cos(np.pi * np.arange(k) / R)
ys += _ring(n - o, fa, ta, aa, pa) # the contact: fast modes from the onset, no ramp
+ R = int(round(STRIKE_RAMP_S * SR)) if i and plan.get('coherent') else 0 # ... a later strike's step is a
+ if R > 1: # click (STRIKE_RAMP_S; a 0.4 fit's, older ones as
+ k = min(R, len(ys)) # they were)
+ ys[:k] *= 0.5 - 0.5 * np.cos(np.pi * np.arange(k) / R)
y[o:] += ys
if plan.get('sub'): # the slow pressure pulse (decimated waveform)
shift, xs = plan['sub']
@@ -952,6 +1018,32 @@ def _room(y, rng, wet, rt60=0.35):
return out
+def _side_t(mid, sd, told, size=1.0, sign=1.0):
+ """the side over time (model['side_t']) made from the mid: per octave band (log-frequency triangles that sum to
+ one, the centres / size like the modes) the band's mid times its pan gain plus the band's Hilbert transform times
+ its diffuse gain, the gains read at the original's time of each GRID frame (told: the plan's time map, so the
+ keyframes move with the strikes and stretch with the decays) and linear between frames"""
+ n = len(mid)
+ n_fft = sfft.next_fast_len(n + 8192)
+ F = np.fft.rfft(mid, n_fft)
+ F[0] = 0.0
+ lf = np.log2(np.maximum(np.fft.rfftfreq(n_fft, 1.0 / SR), 1.0))
+ lc = np.log2(np.asarray(sd['bands'], float) / size)
+ eye = np.eye(len(lc))
+ tk = np.asarray(sd['times'], float)
+ grid = np.arange(len(told)) * GRID
+ ts = np.arange(n) / SR
+ side = np.zeros(n)
+ for b in range(len(lc)):
+ W = np.interp(lf, lc, eye[b])
+ pb, db = np.asarray(sd['pan'][b], float), np.asarray(sd['diffuse'][b], float)
+ if np.abs(pb).max() > 1e-4:
+ side += np.interp(ts, grid, np.interp(told, tk, pb)) * np.fft.irfft(F * W, n_fft)[:n]
+ if db.max() > 1e-4:
+ side += np.interp(ts, grid, np.interp(told, tk, db)) * np.fft.irfft(F * W * (-1j * sign), n_fft)[:n]
+ return side
+
+
def render_model(model, seed=0, size=1.0, damping=1.0, brightness=0.0, hardness=1.0, noise=0.0, width=1.0,
room=0.0, vary=0.0, spacing=1.0, gain=0.0, stereo=None, ring=None):
ring = vary > 0 if ring is None else bool(ring)
@@ -966,11 +1058,13 @@ def render_model(model, seed=0, size=1.0, damping=1.0, brightness=0.0, hardness=
if two:
side = np.zeros(n)
rho = model.get('side')
- if rho and width > 0: # the side: the mid's Hilbert transform (uncorrelated with it at zero lag, same
+ sign = -1.0 if np.random.default_rng([s, 3]).uniform() < 0.5 else 1.0
+ if model.get('side_t') and width > 0: # the side over time (0.4), made from the mid
+ side = _side_t(mid, model['side_t'], plan['told'], size, sign) * width
+ elif rho and width > 0: # the side: the mid's Hilbert transform (uncorrelated with it at zero lag, same
n_fft = sfft.next_fast_len(n + 16384) # envelope), shaped per band by the original's
W = _masks(n_fft, plan['edges'], plan['h']) # side / mid ratio
gain_f = np.sqrt(((np.asarray(rho) ** 2)[:, None] * W ** 2).sum(0)) * width
- sign = -1.0 if np.random.default_rng([s, 3]).uniform() < 0.5 else 1.0
H = np.fft.rfft(mid, n_fft) * gain_f * (-1j * sign)
H[0] = 0.0
side = np.fft.irfft(H, n_fft)[:n]
@@ -1027,7 +1121,7 @@ def summary(model):
return '\n'.join(lines)
-_CODE_FUNCS = (_plan, _ring_end, _masks, _ring, _modes, _noise, _room, render_model)
+_CODE_FUNCS = (_plan, _ring_end, _masks, _ring, _modes, _noise, _room, _side_t, render_model)
def _literal(model):
@@ -1050,6 +1144,10 @@ def _literal(model):
out.append(f' {json.dumps(env, separators=(",", ":"))},')
out.append(' ]},')
out.append(f" 'side': {model['side']!r},")
+ if model.get('coherent'):
+ out.append(" 'coherent': True, # vary and the knobs keep the modes' structure (0.4)")
+ if model.get('side_t'):
+ out.append(f" 'side_t': {json.dumps(model['side_t'], separators=(',', ':'))}, # the side over time")
sub = model.get('sub')
out.append(f" 'sub': {json.dumps(sub, separators=(',', ':')) if sub else None}, # waveform under 20 Hz")
out.append('}')
@@ -1065,7 +1163,8 @@ def to_code(model, name='sound'):
f'noise=, width=, room=, vary=, spacing=, gain=) -> float array at 48 kHz.\n"""\n')
body = [head, 'import sys', '', 'import numpy as np', 'from scipy import fft as sfft', 'from scipy import signal', '',
f'SR = {SR}', f'GRID = {GRID!r}', f'SUB_DEC = {SUB_DEC}', f'RING_DB = {RING_DB!r}', f'RING_EXT = {RING_EXT!r}',
- f'RING_MIN_S = {RING_MIN_S!r}', f'RING_LOUD_DB = {RING_LOUD_DB!r}', f'RING_LOUD_S = {RING_LOUD_S!r}', '',
+ f'RING_MIN_S = {RING_MIN_S!r}', f'RING_LOUD_DB = {RING_LOUD_DB!r}', f'RING_LOUD_S = {RING_LOUD_S!r}',
+ f'STRIKE_RAMP_S = {STRIKE_RAMP_S!r}', '',
_literal(model), '', '']
for fn in _CODE_FUNCS:
body.append(inspect.getsource(fn))
diff --git a/earshot/forge/fit/quality.py b/earshot/forge/fit/quality.py
index 2645ebe..5e57105 100644
--- a/earshot/forge/fit/quality.py
+++ b/earshot/forge/fit/quality.py
@@ -7,8 +7,9 @@
fl = floor(lambda s: render(model, seed=s)) # quality(render seed 11, render seed 12): the noise floor
pair_png('pair.png', original, resynth) # side-by-side spectrograms + envelopes + 1/3-octave balance
-Metrics (both sounds mono-summed; the ORIGINAL's onset trims both by the same amount, so a timing error still counts;
-each is levelled to its own loudest 50 ms RMS, so a level offset is reported apart as `level_db`):
+Metrics (both sounds mono-summed, except the stereo ones; the ORIGINAL's onset trims both by the same amount, so a
+timing error still counts; each is levelled to its own loudest 50 ms RMS, so a level offset is reported apart as
+`level_db`):
mel_db multi-resolution log-mel distance: 64 mel bands 30 Hz - 20 kHz, FFT frames 256 / 1024 / 4096 (hop 1/4),
mean |dB| over the cells within 60 dB of the louder sound's top (floored 80 dB down), averaged over
the three resolutions (mel_256 / mel_1024 / mel_4096 are reported too). 0 = identical spectrograms.
@@ -24,11 +25,21 @@ each is levelled to its own loudest 50 ms RMS, so a level offset is reported apa
missing_db the same the other way (the original has it, the resynthesis does not)
t40_ms (original, resynth): peak until the 5 ms envelope first falls 40 dB
level_db resynth minus original, loudest 50 ms RMS (dB)
+ side_db (original, resynth): side / mid energy (dB, floored at -40) over the original's active span (its onset
+ to the end of its tail, fit.space.active_span); M = (L + R) / 2, S = (L - R) / 2, a mono sound has none
+ width_db per-band width error: mean |difference| of the two sounds' side / mid (dB, each floored at -40) in the
+ octave bands of fit.space.side_powers over that span, weighted by the original's band level (bands
+ within 40 dB of its loudest, linear in dB); 0 = the same width and spread of width over frequency
+ tail_corr (original, resynth): L/R correlation over the original's tail (fit.space.tail_span: from its loudest
+ 10 ms + 30 ms to the end of its tail; None when shorter than 20 ms); a reverberant space decorrelates
+ tail_corr_err |difference| of the two (0 when either is None)
clap_cos (clap=True) cosine of the two CLAP embeddings (1 = the same to CLAP; same-category real refs sit at
0.4-0.95 to each other)
describe (describe=True) the top tag of `earshot describe` of each sound
`score(q)` folds them into one number for choosing between fits (lower is better). `floor(render_fn)` measures two
-renders of one model with different seeds: a stochastic model can never beat that floor on mel_db.
+renders of one model with different seeds: a stochastic model can never beat that floor on mel_db. Before 0.4 the
+score was mono only: a wide recording re-made as a near-mono one (sines renders of the films' wide dings: side / mid
+-16 dB against -5, tail correlation 0.92 against 0.39) scored as well as a wide one.
Pure numpy (matplotlib only for pair_png; the CLAP / describe models only when asked).
"""
import os
@@ -222,6 +233,46 @@ def excess(a, b, tol=6.0, depth=100.0):
return (float(np.maximum(0.0, EB - EA - tol)[act].mean()), float(np.maximum(0.0, EA - EB - tol)[act].mean()))
+def _align2(a, b):
+ """a, b as (n, 2) trimmed at a's onset (5 ms lead) and zero-padded to one length (not levelled: the stereo
+ measures are ratios)"""
+ from .space import _stereo
+ A, B = _stereo(a), _stereo(b)
+ on = max(0, onset_index(A.mean(1)) - int(0.005 * SR))
+ A, B = A[on:], B[on:]
+ n = max(len(A), len(B))
+ return np.pad(A, ((0, n - len(A)), (0, 0))), np.pad(B, ((0, n - len(B)), (0, 0)))
+
+
+def stereo(a, b, floor_db=-40.0, depth_db=40.0):
+ """the stereo measures of b against its original a (module doc): {'side_db': (a, b), 'width_db', 'tail_corr':
+ (a, b), 'tail_corr_err'}"""
+ from .space import active_span, corr, side_powers, tail_span
+ A, B = _align2(a, b)
+ if not (np.ndim(a) == 2 or np.ndim(b) == 2) or len(A) < 64:
+ return {'side_db': (floor_db, floor_db), 'width_db': 0.0, 'tail_corr': (None, None), 'tail_corr_err': 0.0}
+ _, _, end = active_span(A)
+ end = max(end, min(len(A), int(0.02 * SR)))
+ side, ratio, level = [], [], []
+ for X in (A, B):
+ P = side_powers(X[:end])
+ pm, ps = P['PM'].sum(1) + 1e-30, P['PS'].sum(1) + 1e-30
+ side.append(round(float(max(floor_db, _db(ps.sum() / pm.sum()))), 2))
+ ratio.append(np.clip(_db(ps / pm), floor_db, 10.0))
+ level.append(_db(pm))
+ w = np.clip((level[0] - level[0].max() + depth_db) / depth_db, 0.0, 1.0)
+ out = {'side_db': tuple(side),
+ 'width_db': round(float((w * np.minimum(np.abs(ratio[0] - ratio[1]), 30.0)).sum() / max(w.sum(), 1e-9)), 3)}
+ span = tail_span(A)
+ ca = cb = None
+ if span is not None:
+ i, j = span
+ ca, cb = corr(A[i:j, 0], A[i:j, 1]), corr(B[i:j, 0], B[i:j, 1])
+ out['tail_corr'] = (None if ca is None else round(ca, 4), None if cb is None else round(cb, 4))
+ out['tail_corr_err'] = round(abs(ca - cb), 4) if ca is not None and cb is not None else 0.0
+ return out
+
+
def quality(original, resynth, clap=False, describe=False):
"""the fit-quality metrics dict of a resynthesis against its original (module doc)"""
xa, xb = load(original), load(resynth)
@@ -240,6 +291,7 @@ def quality(original, resynth, clap=False, describe=False):
q['t40_ms'] = (round(t40(a), 1), round(t40(b), 1))
q['level_db'] = round(peak_rms_db(xb) - peak_rms_db(xa), 2)
q['len_s'] = (round(len(a) / SR, 4), round(len(b) / SR, 4))
+ q.update(stereo(xa, xb))
if clap:
q['clap_cos'] = round(clap_cos(xa, xb), 4)
if describe:
@@ -247,13 +299,23 @@ def quality(original, resynth, clap=False, describe=False):
return q
+WIDTH_W = 0.04 # score per dB of per-band width error (capped at WIDTH_CAP_DB)
+WIDTH_CAP_DB = 20.0
+TAIL_CORR_W = 1.0 # score per unit of tail L/R correlation error
+
+
def score(q):
"""one number to choose between fits of one recording (lower is better): mel dB + 0.3 x tone dB
+ 8 x (1 - envelope r) + 1 x onset error (octaves, capped at 2) + 1.5 x excess dB + 1 x missing dB
- [+ 20 x (1 - CLAP cos) when measured]. On the 12 validation refs (in-sample) the model-free part alone chose fits
- with a mean CLAP cos of 0.927 (the best candidates: 0.944); without excess / missing it chose 0.856"""
+ + 0.04 x width dB (capped at 20) + 1 x tail correlation error [+ 20 x (1 - CLAP cos) when measured].
+ On the 12 validation refs (in-sample) the model-free part alone chose fits with a mean CLAP cos of 0.927 (the best
+ candidates: 0.944); without excess / missing it chose 0.856. The stereo terms (0.4) cost a mono re-make of a wide
+ recording (width 15 dB, tail correlation off by 0.6) about 1.2, what 0.06 of CLAP costs: they decide between
+ candidates that sound alike, not over a clearly closer one (on 29 wide-category dev clips of the replication
+ benchmark they changed no winner). Mono recordings: both are 0"""
s = (q['mel_db'] + 0.3 * q['tone_db'] + 8.0 * (1.0 - q['env_corr']) + 1.0 * min(q['onset_err'], 2.0)
- + 1.5 * q.get('excess_db', 0.0) + 1.0 * q.get('missing_db', 0.0))
+ + 1.5 * q.get('excess_db', 0.0) + 1.0 * q.get('missing_db', 0.0)
+ + WIDTH_W * min(q.get('width_db', 0.0), WIDTH_CAP_DB) + TAIL_CORR_W * q.get('tail_corr_err', 0.0))
if q.get('clap_cos') is not None:
s += 20.0 * (1.0 - q['clap_cos'])
return float(s)
@@ -303,6 +365,12 @@ def text(q, fl=None):
f"(sharpness {q['sharp_db_ms'][0]:.1f} vs {q['sharp_db_ms'][1]:.1f} dB/ms), "
f"excess {q.get('excess_db', 0):.2f} / missing {q.get('missing_db', 0):.2f} dB, "
f"t40 {q['t40_ms'][0]:.0f} vs {q['t40_ms'][1]:.0f} ms, level {q['level_db']:+.2f} dB")
+ sd = q.get('side_db')
+ if sd and max(sd) > -40:
+ s += f", side {sd[0]:.0f} vs {sd[1]:.0f} dB (width err {q.get('width_db', 0):.1f} dB)"
+ tc = q.get('tail_corr') or (None, None)
+ if tc[0] is not None and tc[1] is not None:
+ s += f", tail corr {tc[0]:.2f} vs {tc[1]:.2f}"
if q.get('clap_cos') is not None:
s += f", CLAP cos {q['clap_cos']:.3f}"
if q.get('describe'):
diff --git a/earshot/forge/fit/sines.py b/earshot/forge/fit/sines.py
index ea2985e..91f101a 100644
--- a/earshot/forge/fit/sines.py
+++ b/earshot/forge/fit/sines.py
@@ -38,13 +38,30 @@ Model (plain JSON-serialisable dict):
'transients': [{'t': start s, 'atoms': [[f Hz, tau s, amplitude, phase], ...]}, ...] (one per strike)
'residual': {'bands': [Hz centres, 1/3 octave], 'times': [s, ...], 'db': [[dBFS per time] per band],
'hf': [[Hz, dB], ...]},
- 'width': side/mid energy of the source in dB (stereo renders), 'analysis': {...}, 'summary': {...}}
+ 'width': side/mid energy of the source in dB, 'side': {'bands': [Hz, octaves], 'times': [s, ...],
+ 'pan': [[gain per time] per band], 'diffuse': [[gain per time] per band]} (a stereo source; else None),
+ 'coherent': True (0.4; see vary below), 'analysis': {...}, 'summary': {...}}
The residual bands are triangles on log frequency that sum to one; a band's dB is the RMS of the band-passed residual.
The top band (16 kHz) is flat to Nyquist and the lowest (50 Hz) flat to DC. Above 16 kHz the noise follows the
recording's own roll-off `hf` (its residual's spectrum re 16 kHz, made monotone); models fitted before 0.3 have no
`hf` and keep the flat top band, which leaves noise above 20 kHz that the recording did not have (hf_rolloff(x)
measures it for them). The lowest band, modulated by its envelope, spreads energy down to DC: a new seed can carry an
offset of 1-2 % of its peak; the fit engine's takes (auto.render(take=True)) are DC-blocked for that.
+The side (0.4; a stereo source): per octave band and keyframe (dense after the first strike, sparse in the tail) the
+source's side split into its part coherent with the mid ('pan', a signed gain: L = M (1 + g), R = M (1 - g)) and the
+rest ('diffuse', a gain re the mid), measured by space.side_powers. A stereo render makes the side FROM the render's
+mid: each band's mid times its pan gain plus the band's quadrature (Hilbert transform, uncorrelated with it) times its
+diffuse gain (space.shaped_side). So the side follows the mid (a ringing partial's tail stays as wide as the source's)
+and partials are panned and decorrelated like the source's; the mono sum is the mono render. On the 42 wide or
+decorrelated stereo dev targets of the replication benchmark (sines fits alone): side / mid error 13.4 dB median ->
+0.15 dB (33 -> 0 off by more than 3 dB), tail L/R correlation error 0.39 -> 0.04. Models fitted before 0.4
+have only 'width' and keep their old side (a fresh draw of the residual noise times 10^(width / 20), which sits far
+under the mid when the sound is tonal).
+Renders start at zero (a FADE_IN_S half-cosine fade-in, not mid-way into a bed; shorter, down to 0.25 ms, when the
+first strike comes earlier: a clip cut into its own attack keeps it) and are peak-limited to PEAK_DB dBFS by a gain
+(finish(); a hybrid's deterministic layer is not limited: the hybrid's sum is what counts). On the 78 dev targets of
+the replication benchmark: CLAP -0.0002 median, 7 renders over -1 dBFS (6 over 0 dBFS) -> 0, 44 starting off zero -> 0.
+A model fitted before 0.4 (no 'coherent' flag: the 0.3 bank's fit entries) renders as it did, with neither.
render(model, seed, knobs) knobs (all default to the identity, which re-renders the fit):
transpose (semitones; every partial and the residual's spectrum move together, relative partials kept),
@@ -52,10 +69,14 @@ render(model, seed, knobs) knobs (all default to the identity, which re-renders
peak, frequency trajectories keep their timing), brightness (dB per octave about the partials' centroid),
vibrato (cents) + vibrato_rate (Hz), drift (cents of slow random wander), residual (gain on the noise),
transient (gain on the noise in the first 15 ms after each strike), vary (0-1: seeded per-partial detune,
- level and decay jitter, band jitter: new takes of the same object), partials (keep the N strongest),
+ level and decay jitter, band jitter: new takes of the same object; a model with 'coherent' (0.4: every fit) draws
+ the partials' jitter from smooth random functions of log frequency, vary_field(), so a harmonic series' upper
+ partials move together and keep their ratios: independently detuned, they went off the harmonic grid and the
+ takes rang or read as aliasing (on the films' dev fits 4 of 21 failing takes passed at vary 0)), partials (keep
+ the N strongest),
inharm (stretch of partial ratios about the lowest strong partial: f/f_low -> (f/f_low)^(1+inharm)),
- phases ('fit' or 'random'), gain_db, stereo (False: mono; True: (n, 2) with the residual decorrelated by width),
- ring (None: True when vary > 0; True renders until the last partial's release ends, so a decay that vary
+ phases ('fit' or 'random'), gain_db, stereo (False: mono; True: (n, 2) with the fitted side), width (factor on
+ the side), ring (None: True when vary > 0; True renders until the last partial's release ends, so a decay that vary
lengthens is not cut at the recording's length, and fades the residual noise out over RING_FADE_S past the
recording's length instead of stopping on it).
@@ -76,24 +97,32 @@ WINDOWS = (512, 1024, 2048, 4096)
DEFAULT_KNOBS = {'transpose': 0.0, 'stretch': 1.0, 'decay': 1.0, 'brightness': 0.0, 'vibrato': 0.0,
'vibrato_rate': 5.5, 'drift': 0.0, 'residual': 1.0, 'transient': 1.0, 'vary': 0.0, 'partials': None,
- 'inharm': 0.0, 'phases': 'fit', 'gain_db': 0.0, 'stereo': False, 'ring': None}
+ 'inharm': 0.0, 'phases': 'fit', 'gain_db': 0.0, 'stereo': False, 'width': 1.0, 'ring': None}
HF_POINTS = (16000.0, 17000.0, 18000.0, 19000.0, 20000.0, 21000.0, 22000.0, 23000.0, 24000.0) # the roll-off grid
RING_FADE_S = 0.06 # ring: the residual noise fades out over this past the recording's length
+SIDE_BANDS = (125.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0) # the side's octave bands
+SIDE_STEPS = ((0.1, 0.01), (0.3, 0.02), (0.7, 0.04), (1e9, 0.08)) # side keyframes after the onset: (until s, step s)
+SIDE_TOL = 0.04 # keyframes are dropped where the gains stay this close to the line through their neighbours
+SIDE_QUIET_DB = 50.0 # a keyframe this far under its band's loudest takes the band's overall side
+FADE_IN_S = 0.002 # every render fades in over this (it starts at zero, not mid-way into a bed)
+PEAK_DB = -1.0 # and is peak-limited to this (dBFS) by a gain
# ================================================================================================ small helpers
def _load(x, sr=SR):
- """(mono float64 signal, side/mid dB) from a path or an array (mono or (n, 2))"""
+ """(mono float64 signal, side/mid dB, the (n, 2) source or None) from a path or an array (mono or (n, 2))"""
if isinstance(x, str):
from ... import audio
x = audio.load(x, sr=SR)
sr = SR
x = np.asarray(x, np.float64)
width = -120.0
+ st = None
if x.ndim == 2:
if x.shape[1] == 2:
mid, side = x.mean(1), (x[:, 0] - x[:, 1]) / 2
width = float(10 * np.log10((side ** 2).sum() / ((mid ** 2).sum() + 1e-20) + 1e-12))
+ st = x
x = mid
else:
x = x.mean(1)
@@ -101,7 +130,8 @@ def _load(x, sr=SR):
from math import gcd
g = gcd(int(sr), SR)
x = signal.resample_poly(x, SR // g, int(sr) // g)
- return x, max(width, -120.0)
+ st = None if st is None else signal.resample_poly(st, SR // g, int(sr) // g, axis=0)
+ return x, max(width, -120.0), st
def _db(v, floor=-200.0):
@@ -172,7 +202,7 @@ def _band_centers(lo=50.0, hi=16000.0, per_oct=3):
# ================================================================================================ synthesis kernels
-# These three functions are copied verbatim into to_code() output: keep them self-contained (numpy only, SR global).
+# These functions are copied verbatim into to_code() output: keep them self-contained (numpy only, SR global).
def band_masks(centers, nfft, sr=48000):
"""triangles on log frequency, one per band centre, that sum to one at every frequency (flat beyond the ends)"""
f = np.fft.rfftfreq(nfft, 1.0 / sr)
@@ -252,6 +282,20 @@ def hf_rolloff(x, sr=48000, points=(16000.0, 17000.0, 18000.0, 19000.0, 20000.0,
return out
+def finish(y, sr=48000, fade_s=0.002, peak_db=-1.0, limit=True):
+ """a render's edges: a half-cosine fade-in over fade_s (it starts at zero) and, when limit and its peak is over
+ peak_db dBFS, a gain down to it (no clipping, no limiter colour)"""
+ y = np.array(y, dtype=float)
+ k = min(len(y), int(round(fade_s * sr)))
+ if k > 1:
+ ramp = np.sin(0.5 * np.pi * np.arange(k) / k) ** 2
+ y[:k] *= ramp[:, None] if y.ndim == 2 else ramp
+ pk = float(np.abs(y).max()) if y.size else 0.0
+ if limit and pk > 10 ** (peak_db / 20):
+ y *= 10 ** (peak_db / 20) / pk
+ return y
+
+
def synth_atoms(transients, n, sr=48000):
"""strike transients: each a start time and damped sinusoids [f Hz, tau s, amp, phase] that start at full
amplitude (a click is a burst of very short modes)"""
@@ -883,6 +927,88 @@ def _analyze(x, sr, W, onsets, width, opts):
return {'model': model, 'det': det, 'res': res, 'atoms': res - res2, 'tracks': tracks}
+def _side_times(onset, duration, steps=SIDE_STEPS):
+ """keyframe times: 0, 10 ms before the onset, then SIDE_STEPS after it, and the end"""
+ tk = [0.0, onset - 0.01, onset]
+ t = onset
+ for until, step in steps:
+ while t + step <= onset + until and t + step < duration:
+ t += step
+ tk.append(t)
+ tk.append(duration)
+ return np.unique(np.round(np.clip(tk, 0.0, duration), 5))
+
+
+def side_model(st, duration=None, bands=SIDE_BANDS, tol=SIDE_TOL):
+ """the side of a stereo source ((n, 2) at 48 kHz) per octave band at keyframes (_side_times after the source's
+ onset, space.active_span, then dropped where a line through the neighbours stays within `tol`): 'pan' =
+ Re(S M*) / |M|^2 (the side's part coherent with the mid, signed) and 'diffuse' = sqrt((|S|^2 - Re(S M*)^2 / |M|^2)
+ / |M|^2) (the rest re the mid), from the STFT frames under a tent from the previous keyframe to the next (what
+ linear interpolation between keyframes spreads it over); a keyframe SIDE_QUIET_DB under its band's loudest (a
+ pre-roll, a faded tail) takes the band's whole-sound values, a band with hardly any energy the whole sound's"""
+ from .space import active_span, side_powers
+ st = np.asarray(st, np.float64)
+ duration = len(st) / SR if duration is None else float(duration)
+ P = side_powers(st, bands)
+ t, PM, PS, C = P['t'], P['PM'], P['PS'], P['C']
+ tk = _side_times(active_span(st)[0] / SR, duration)
+ J = len(tk)
+ edges = np.concatenate([[tk[0] - 0.01], tk, [tk[-1] + 0.01]])
+ sPM, sPS, sC = np.zeros((len(bands), J)), np.zeros((len(bands), J)), np.zeros((len(bands), J))
+ for j in range(J):
+ a, c_, b = edges[j], edges[j + 1], edges[j + 2]
+ w = np.clip(np.minimum((t - a) / max(c_ - a, 1e-6), (b - t) / max(b - c_, 1e-6)), 0.0, 1.0)
+ if not w.any(): # no frame under its tent: the nearest
+ w[int(np.argmin(np.abs(t - c_)))] = 1.0
+ sPM[:, j], sPS[:, j], sC[:, j] = PM @ w, PS @ w, C @ w
+
+ def gains(pm, ps, c):
+ pm = np.maximum(pm, 1e-30)
+ return np.clip(c / pm, -1.5, 1.5), np.clip(np.sqrt(np.maximum(ps - c ** 2 / pm, 0.0) / pm), 0.0, 2.0)
+ pan, dif = gains(sPM, sPS, sC)
+ tot = PM.sum(1)
+ gp, gd = gains(PM.sum(), PS.sum(), C.sum()) # the whole sound, all bands
+ for b in range(len(bands)):
+ if tot[b] < tot.max() * 1e-6:
+ bp, bd = gp, gd
+ else:
+ bp, bd = gains(tot[b], PS[b].sum(), C[b].sum())
+ quiet = sPM[b] < sPM[b].max() * 10 ** (-SIDE_QUIET_DB / 10) if tot[b] >= tot.max() * 1e-6 else np.ones(J, bool)
+ pan[b, quiet], dif[b, quiet] = bp, bd
+ keep = _simplify_multi(tk, np.vstack([pan, dif]), tol)
+ return {'bands': [float(b) for b in bands], 'times': [round(float(v), 5) for v in tk[keep]],
+ 'pan': [[round(float(v), 3) for v in row[keep]] for row in pan],
+ 'diffuse': [[round(float(v), 3) for v in row[keep]] for row in dif]}
+
+
+def morph_side(sa, sb, w, map_a=None, map_b=None):
+ """the side of a morph between two sines models' sides (fit.auto morph): each side's keyframe times through its
+ time map (map_a / map_b: arrays -> arrays, the morph's anchors; None = unchanged), the gains interpolated on the
+ union of the two grids; a model without a side counts as a centred one (all gains 0). None when neither has one"""
+ if not sa and not sb:
+ return None
+ bands = (sa or sb)['bands']
+ if sa and sb and list(sa['bands']) != list(sb['bands']):
+ raise ValueError('sines sides with different bands cannot be morphed')
+ grids = []
+ for s, f in ((sa, map_a), (sb, map_b)):
+ if s:
+ t = np.asarray(s['times'], float)
+ grids.append((np.asarray(f(t) if f else t, float), np.asarray(s['pan'], float),
+ np.asarray(s['diffuse'], float)))
+ else:
+ grids.append(None)
+ tt = np.unique(np.round(np.concatenate([g[0] for g in grids if g is not None]), 5))
+
+ def at(g, i):
+ return np.zeros((len(bands), len(tt))) if g is None else np.array([np.interp(tt, g[0], r) for r in g[i]])
+ pan = (1 - w) * at(grids[0], 1) + w * at(grids[1], 1)
+ dif = (1 - w) * at(grids[0], 2) + w * at(grids[1], 2)
+ return {'bands': [float(b) for b in bands], 'times': [float(v) for v in tt],
+ 'pan': [[round(float(v), 3) for v in row] for row in pan],
+ 'diffuse': [[round(float(v), 3) for v in row] for row in dif]}
+
+
def fit(x, sr=SR, windows=WINDOWS, max_partials=160, dyn_db=80.0, keep_db=60.0, prom_db=4.0, max_peaks=100, min_ms=15.0,
amp_tol_db=1.0, freq_tol_cents=1.0, res_range_db=140.0, transients=True, passes=2, snr_db=10.0, source=None):
"""a sines model of `x` (a path or an array at `sr`); every size in `windows` is tried and the model whose
@@ -890,7 +1016,7 @@ def fit(x, sr=SR, windows=WINDOWS, max_partials=160, dyn_db=80.0, keep_db=60.0,
if isinstance(x, str) and source is None:
import os
source = os.path.basename(x)
- x, width = _load(x, sr)
+ x, width, st = _load(x, sr)
opts = {'dyn_db': dyn_db, 'keep_db': keep_db, 'prom_db': prom_db, 'max_peaks': max_peaks, 'min_ms': min_ms,
'max_partials': max_partials, 'amp_tol_db': amp_tol_db, 'freq_tol_cents': freq_tol_cents,
'res_range_db': res_range_db, 'transients': transients, 'passes': passes, 'snr_db': snr_db}
@@ -905,6 +1031,9 @@ def fit(x, sr=SR, windows=WINDOWS, max_partials=160, dyn_db=80.0, keep_db=60.0,
if best is None or d < best[0]:
best = (d, m)
model = best[1]
+ if st is not None and width > -60.0:
+ model['side'] = side_model(st, model['duration'])
+ model['coherent'] = True
model['analysis']['lsd_db'] = round(best[0], 2)
model['analysis']['tried'] = tried
model['analysis']['onset_rise_ms'] = [round(o['rise'] * 1000, 2) for o in onsets]
@@ -1025,6 +1154,9 @@ def render(model, seed=0, knobs=None, **kw):
parts = parts[:int(k['partials'])]
identity = (s_ == 1.0 and d_ == 1.0 and ratio == 1.0 and vary == 0.0 and k['brightness'] == 0.0
and k['inharm'] == 0.0 and k['phases'] == 'fit')
+ coherent = bool(model.get('coherent')) and vary > 0
+ if coherent: # the jitter as smooth functions of log frequency (module doc)
+ jf, jl, jd = vary_field(r), vary_field(r), vary_field(r)
new = []
for p in parts:
if identity:
@@ -1034,9 +1166,13 @@ def render(model, seed=0, knobs=None, **kw):
am = np.asarray(p['amp'], float).copy()
fmed = partial_freq(p)
fac = ratio * (fmed / f_low) ** float(k['inharm']) if fmed > 0 else ratio
- detune = r.normal(0, 8.0 * vary)
- jit_db = r.normal(0, 2.0 * vary)
- jit_dec = float(np.exp(r.normal(0, 0.15 * vary)))
+ if coherent:
+ detune, jit_db = 8.0 * vary * jf(fmed), 2.0 * vary * jl(fmed)
+ jit_dec = float(np.exp(0.15 * vary * jd(fmed)))
+ else:
+ detune = r.normal(0, 8.0 * vary)
+ jit_db = r.normal(0, 2.0 * vary)
+ jit_dec = float(np.exp(r.normal(0, 0.15 * vary)))
ph = float(p['ph']) if k['phases'] == 'fit' else float(r.uniform(-np.pi, np.pi))
fq[:, 1] = fq[:, 1] * fac * 2 ** (detune / 1200)
am[:, 1] = am[:, 1] + jit_db + float(k['brightness']) * np.log2(max(fmed * ratio, 1.0) / f_ref)
@@ -1087,11 +1223,19 @@ def render(model, seed=0, knobs=None, **kw):
trans = model.get('transients', [])
if trans and (not identity or k['transient'] != 1.0):
tg = float(k['transient'])
- trans = [{'t': float(T(tr['t'])),
- 'atoms': [[f * ratio * 2 ** (r.normal(0, 30.0 * vary) / 1200), tau / ratio,
- a * tg * 10 ** ((r.normal(0, 2.0 * vary) + float(k['brightness'])
- * np.log2(max(f * ratio, 20.0) / f_ref)) / 20), ph]
- for f, tau, a, ph in tr['atoms']]} for tr in trans]
+ out = []
+ for tr in trans: # a coherent model jitters a strike's atoms together: they were fitted jointly
+ if coherent: # and cancel each other's step at the onset (6 atoms of a sub drop's strike: amp
+ d1, g1 = r.normal(0, 30.0 * vary), r.normal(0, 2.0 * vary) # up to 1.5, their sum there 0.015;
+ jit = [(d1, g1)] * len(tr['atoms']) # jittered apart, the takes clicked)
+ else:
+ jit = [(r.normal(0, 30.0 * vary), r.normal(0, 2.0 * vary)) for _ in tr['atoms']]
+ out.append({'t': float(T(tr['t'])),
+ 'atoms': [[f * ratio * 2 ** (dc / 1200), tau / ratio,
+ a * tg * 10 ** ((dg + float(k['brightness'])
+ * np.log2(max(f * ratio, 20.0) / f_ref)) / 20), ph]
+ for (f, tau, a, ph), (dc, dg) in zip(tr['atoms'], jit)]})
+ trans = out
y = y + synth_atoms(trans, n, sr)
hf = res.get('hf')
noise = synth_noise(bands, times, lev, n, seed, sr, hf=hf) if len(bands) else np.zeros(n)
@@ -1105,11 +1249,46 @@ def render(model, seed=0, knobs=None, **kw):
y = y + noise
if k['gain_db']:
y = y * 10 ** (float(k['gain_db']) / 20)
+ limit = _limited(model)
+ fade = fade_in_s(model)
if k['stereo']:
- side = synth_noise(bands, times, lev, n, seed + 1000003, sr, hf=hf) * 10 ** (model.get('width', -120.0) / 20)
- side *= 10 ** (float(k['gain_db']) / 20)
- return np.column_stack([y + side, y - side])
- return y
+ sd = model.get('side')
+ if sd: # the side made from the mid, per band (module doc); keyframes move like the noise
+ from .space import shaped_side
+ st = np.asarray(sd['times'], float)
+ piv = on + 0.010
+ st = T(np.where(st > piv, piv + (st - piv) * d_, st))
+ sign = -1.0 if np.random.default_rng((int(seed) * 31 + 7) & 0xFFFFFFFF).uniform() < 0.5 else 1.0
+ side = shaped_side(y, np.asarray(sd['bands'], float) * ratio, st, sd['pan'], sd['diffuse'], sign, sr)
+ else: # a model fitted before 0.4: its old side (module doc)
+ w_old = 10 ** (model.get('width', -120.0) / 20)
+ side = synth_noise(bands, times, lev, n, seed + 1000003, sr, hf=hf) * w_old
+ side *= 10 ** (float(k['gain_db']) / 20) # 0.3's two steps exactly (one differs in the last bits)
+ side = side * float(k['width'])
+ return finish(np.column_stack([y + side, y - side]), sr, fade, limit=limit)
+ return finish(y, sr, fade, limit=limit)
+
+
+def vary_field(r, parts=6):
+ """a smooth random function of frequency (Hz -> unit-variance value): `parts` cosines over log2 frequency of
+ 0.25-1.25 cycles per octave with random phases. A coherent model's vary draws its jitter from it: neighbouring
+ partials (a harmonic series' upper partials) move together and keep their ratios"""
+ c, p = r.uniform(0.25, 1.25, parts), r.uniform(0.0, 2 * np.pi, parts)
+ return lambda f: float(np.sqrt(2.0 / parts) * np.cos(2 * np.pi * np.log2(max(float(f), 20.0)) * c + p).sum())
+
+
+def fade_in_s(model):
+ """the render's fade-in: FADE_IN_S, shortened (to 0.25 ms at least) when the first strike comes earlier, so a clip
+ cut into its own attack keeps the attack; 0 for a model fitted before 0.4 (no 'coherent'): it renders as it did"""
+ if not model.get('coherent'):
+ return 0.0
+ return float(np.clip(float(model.get('onset', 1.0)) - 0.0005, 0.00025, FADE_IN_S))
+
+
+def _limited(model):
+ """is the render peak-limited to PEAK_DB? Not a hybrid's deterministic layer (the sum is what counts), not a model
+ fitted before 0.4 (it renders as it did)"""
+ return model.get('layer') != 'deterministic' and bool(model.get('coherent'))
# ================================================================================================ model -> code
@@ -1153,8 +1332,10 @@ def to_code(model, name='sound'):
for row in res['db']:
lines.append(f' {_lit(row)},')
lines.append(']')
+ lines.append(f"LIMIT = {_limited(model)} # peak-limit the render to -1 dBFS")
+ lines.append(f"FADE_S = {fade_in_s(model)!r} # the fade-in (s): the render starts at zero")
lines.append('')
- for fn in (band_masks, synth_partials, synth_atoms, synth_noise):
+ for fn in (band_masks, synth_partials, synth_atoms, synth_noise, finish):
lines.append('')
lines.append(inspect.getsource(fn).rstrip())
lines.append('')
@@ -1164,8 +1345,8 @@ def to_code(model, name='sound'):
" parts = [{'t0': p[0], 'rise': p[1], 't1': p[2], 'rel': p[3], 'ph': p[4], 'freq': p[5], 'amp': p[6]}",
' for p in PARTIALS]',
" trans = [{'t': t, 'atoms': a} for t, a in TRANSIENTS]",
- ' return (synth_partials(parts, n, SR) + synth_atoms(trans, n, SR)',
- ' + synth_noise(BANDS, TIMES, LEVELS, n, seed, SR, hf=HF))',
+ ' return finish(synth_partials(parts, n, SR) + synth_atoms(trans, n, SR)',
+ ' + synth_noise(BANDS, TIMES, LEVELS, n, seed, SR, hf=HF), SR, FADE_S, limit=LIMIT)',
'',
'',
"if __name__ == '__main__':",
diff --git a/earshot/forge/fit/space.py b/earshot/forge/fit/space.py
new file mode 100644
index 0000000..9e42bea
--- /dev/null
+++ b/earshot/forge/fit/space.py
@@ -0,0 +1,484 @@
+"""Space: the stereo picture of a sound, and a fitted diffuse tail layer (a small room) for recordings whose tail is
+decorrelated.
+
+ from earshot.forge.fit import space
+ sp = space.fit_space(x, 48000) # None when the tail is not decorrelated (L/R correlation > 0.7)
+ sp = space.fit_space(x, 48000, dry=y0) # only the diffuse side the dry render y0 lacks
+ y = space.render_space(sp, y0, seed=0) # y0 + the wet layer: (n, 2)
+ space.tail_corr(x) # the tail's L/R correlation (from the peak + 30 ms)
+ P = space.side_powers(x) # per band and frame: mid, side and cross power
+
+Why: a reverb tail is two different signals in L and R (a diffuse field: tail L/R correlation 0.1-0.5 in the films'
+dings, glass, bells, swells), while a model of a sound's mid rings the same signal in both channels (a render's tail
+correlation 0.9 and more). The wet layer is the dry render convolved with seeded noise impulse responses: a SIDE part
+(L + w, R - w), which leaves the mono sum alone (so every mono measure and CLAP), and a MID part (both channels), fitted
+only when the dry's own mono tail falls short of the recording's. A fit of the whole recording already rings its tail
+in mono, so the side part is usually all there is: then the width and the tail's L/R correlation change and nothing
+else. Convolving the dry render (not adding band noise) keeps a ringing tail tonal in the side too: the reverb of a bell
+is the bell's partials smeared, not hiss.
+
+Stereo measures (shared with fit/sines.py and fit/quality.py), on mid M = (L + R) / 2 and side S = (L - R) / 2:
+ side_powers per octave band (triangles on log frequency that sum to one, sines.band_masks) and STFT frame: PM, PS
+ and C = Re(S M*). C / PM is the side's part coherent with the mid (a pan: L = M (1 + g), R = M (1 - g));
+ PS - C^2 / PM what is left, a side uncorrelated with the mid (diffuse; rendered as the mid's quadrature
+ or as this module's wet layer)
+ active_span / tail_span onset to the last 1 ms frame above the clip's floor (replicate.props' definition: 3 dB
+ over the lower of the pre-roll level and the level before the clip's end, kept 12-45 dB under the
+ peak); the tail starts TAIL_START_S after the loudest 10 ms
+ tail_corr L/R correlation (zero lag) over the tail span
+
+Model (plain JSON): {'kind': 'space', 'version': 1, 'predelay': s, 'build': s (the impulse responses' rise),
+ 'bands': [Hz octave centres], 'rt60': [s per band], 'side_db': [dB per band re a unit-energy band IR; -150 = off],
+ 'mid_db': [the same for the mid part] (all -150: side only), 'tail_corr': the recording's tail correlation,
+ 'fit': {'need_db', 'mid_need_db', 'iters', 'dry', 'side_offset_db', 'tail_corr_dry', 'tail_corr_wet'}}
+
+fit_space (analysis by synthesis), over the tail span, per band: the side part must supply the recording's diffuse side
+(its side uncorrelated with its mid) minus what the dry render already has (its own diffuse side: a sines model's side,
+a modal side); the mid part the recording's mid energy minus the dry's, where the dry lacks at least MID_NEED_DB of it.
+No layer when the dry's tail correlation is within 0.1 of the recording's already, or when the side need is NEED_DB
+under the recording's own diffuse tail. Starting rt60s come from the decay of the recording's diffuse side per band;
+then unit-gain wet renders of the dry (the recording's own mid when no dry is given) are measured with the same
+analysis and each band's levels and rt60 corrected toward the needs (`iters` passes: the dry's own decay lengthens the
+convolved tail; rt60 at most RT_SPAN x the tail span). The mid part is dropped if it makes the mono mel distance
+(quality.mel_distance) worse. Last, the side part's overall level moves (+-OFFSET_DB, closed form over the tail, the
+products pooled over FIT_SEEDS renders) to where the tail correlation is closest to the recording's; the layer is kept
+only if that lands closer than the dry's. The wet side is made orthogonal to the dry's side and mid per band
+(_orthogonal), so a random wet never cancels the dry's own width.
+On the 29 wide-category dev clips of the replication benchmark it was fitted on 3 winners (a modal, two hybrids: tail
+correlation off by 0.21 median -> 0.02) and on none of the sines winners (their side already matches). Seeds spread
+around the fitted correlation by +-0.02 on a 230 ms bell tail but +-0.25 on 40-70 ms tails (their correlation is
+itself that uncertain; the mean over seeds still lands within 0.15 of the recording's, the dry's was 0.23-0.29 off).
+render_space(model, dry, seed, width, ring): the dry (mono or (n, 2)) plus the wet; width scales the side part;
+ring=True lets the wet ring out past the dry's end (a take). The result is peak-limited to -1 dBFS (sines.finish).
+Deterministic: render_space(model, dry, seed) is a pure function of its arguments. Pure numpy / scipy.
+"""
+import numpy as np
+from scipy import fft as sfft
+from scipy import signal
+
+from .sines import band_masks, finish
+
+SR = 48000
+KIND = 'space'
+VERSION = 1
+BANDS = (125.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0) # octave centres
+NFFT = 1024 # stereo analysis frames (Hann), every HOP samples
+HOP = 240
+TAIL_START_S = 0.03 # the tail starts this long after the loudest 10 ms (replicate.props)
+TAIL_CORR_MAX = 0.7 # fit_space models a tail only when the recording's tail L/R correlation is at most this
+MAX_RT = 3.0 # s: the longest rt60 a band may get (and the impulse response's length cap)
+RT_SPAN = 4.0 # and at most this many times the recording's tail span
+OFFSET_DB = 12.0 # the side's overall level may move this far (dB) onto the recording's tail correlation
+FLAT_ITERS = 4 # passes that flatten each band of the impulse responses (impulse())
+FIT_SEEDS = (0, 1, 2, 3) # renders whose tail products are pooled when the side level is set
+NEED_DB = -15.0 # no layer when what is missing is this far under the recording's own tail side energy
+MID_NEED_DB = -10.0 # a mid part only where the dry lacks at least this share of the tail's mid energy (dB)
+
+
+# ================================================================================================ helpers
+def _stereo(x):
+ """float64 (n, 2): a mono signal (n,) or (n, 1) is duplicated"""
+ x = np.asarray(x, np.float64)
+ if x.ndim == 1:
+ return np.stack([x, x], 1)
+ if x.shape[1] == 1:
+ return np.repeat(x, 2, 1)
+ return x[:, :2]
+
+
+def _resample(x, sr):
+ x = np.asarray(x, np.float64)
+ if int(sr) == SR:
+ return x
+ from math import gcd
+ g = gcd(int(sr), SR)
+ return signal.resample_poly(x, SR // g, int(sr) // g, axis=0)
+
+
+def _db(p, floor=1e-30):
+ return 10 * np.log10(np.maximum(p, floor))
+
+
+def is_stereo(x, tol_db=-60.0):
+ """True when x has two channels that differ (side energy above `tol_db` re the mid)"""
+ x = np.asarray(x, np.float64)
+ if x.ndim != 2 or x.shape[1] < 2:
+ return False
+ m, s = 0.5 * (x[:, 0] + x[:, 1]), 0.5 * (x[:, 0] - x[:, 1])
+ return bool(_db((s ** 2).sum()) - _db((m ** 2).sum()) > tol_db)
+
+
+def _frames_fft(m, nfft=NFFT, hop=HOP):
+ """Hann frames of m centred at k * hop (k = 0 .. len(m) // hop), rfft: (K, nfft // 2 + 1)"""
+ K = len(m) // hop + 1
+ xp = np.zeros((K - 1) * hop + nfft)
+ xp[nfft // 2:nfft // 2 + len(m)] = m[:len(xp) - nfft // 2]
+ fr = np.lib.stride_tricks.sliding_window_view(xp, nfft)[::hop][:K]
+ return np.fft.rfft(fr * np.hanning(nfft + 2)[1:-1], axis=1)
+
+
+def band_powers(m, centers=BANDS, nfft=NFFT, hop=HOP):
+ """per band (log-frequency triangles that sum to one) and frame: the power of the mono signal m, (B, K)"""
+ W = band_masks(centers, nfft, SR)
+ return (np.abs(_frames_fft(np.asarray(m, np.float64), nfft, hop)) ** 2 @ W.T).T
+
+
+def side_powers(x, centers=BANDS, nfft=NFFT, hop=HOP):
+ """{'t': frame centres (s), 'PM', 'PS', 'C': (B, K)}: per band and frame the mid power, the side power and the
+ real cross power Re(S M*) of mid M = (L + R) / 2 and side S = (L - R) / 2 (module doc)"""
+ X = _stereo(x)
+ M, S = 0.5 * (X[:, 0] + X[:, 1]), 0.5 * (X[:, 0] - X[:, 1])
+ W = band_masks(centers, nfft, SR)
+ FM, FS = _frames_fft(M, nfft, hop), _frames_fft(S, nfft, hop)
+ return {'t': np.arange(len(FM)) * hop / SR, 'PM': (np.abs(FM) ** 2 @ W.T).T, 'PS': (np.abs(FS) ** 2 @ W.T).T,
+ 'C': (np.real(FS * np.conj(FM)) @ W.T).T}
+
+
+def diffuse_power(P):
+ """the side power uncorrelated with the mid, PS - C^2 / PM (>= 0), from side_powers()"""
+ return np.maximum(P['PS'] - P['C'] ** 2 / np.maximum(P['PM'], 1e-30), 0.0)
+
+
+def _env1(m):
+ """1 ms hops, 5 ms window power envelope (replicate.env1)"""
+ n = len(m)
+ c = np.concatenate([[0.0], np.cumsum(m * m)])
+ w, hop = int(0.005 * SR), int(0.001 * SR)
+ cen = np.arange(0, n, hop)
+ a = np.clip(cen - w // 2, 0, n)
+ b = np.clip(cen + w // 2, 0, n)
+ return (c[b] - c[a]) / np.maximum(b - a, 1)
+
+
+def active_span(x):
+ """(onset, peak, end) sample indices of a sound (module doc): the onset is the first 1 ms frame within 30 dB of the
+ loudest 10 ms, the end the last frame above the floor threshold"""
+ m = _stereo(x).mean(1)
+ E1 = _env1(m)
+ if len(E1) < 3:
+ return 0, 0, len(m)
+ sm = np.convolve(E1, np.ones(10) / 10, mode='same')
+ ip = int(np.argmax(sm))
+ pk = float(sm[ip]) + 1e-30
+ if len(E1) >= 80:
+ fl = min(_db(float(np.median(E1[5:50]))), _db(float(np.median(E1[-25:-12]))))
+ thr = float(np.clip(fl + 3.0 - _db(pk), -45.0, -12.0))
+ else:
+ thr = -40.0
+ on = int(np.argmax(sm >= pk * 1e-3))
+ live = np.nonzero(sm >= pk * 10 ** (thr / 10))[0]
+ last = int(live[-1]) if len(live) else ip
+ return on * 48, ip * 48, min(len(m), last * 48 + 48)
+
+
+def tail_span(x, start_s=TAIL_START_S, min_s=0.02):
+ """(first, last) sample of a sound's tail (peak + start_s to the end of active_span); None if shorter than min_s"""
+ _, ip, end = active_span(x)
+ a = ip + int(start_s * SR)
+ return (a, end) if end - a >= int(min_s * SR) else None
+
+
+def corr(L, R):
+ """zero-lag correlation of two channels (None when either is silent)"""
+ eL, eR = float((L * L).sum()), float((R * R).sum())
+ if eL < 1e-20 or eR < 1e-20:
+ return None
+ return float((L * R).sum() / np.sqrt(eL * eR))
+
+
+def tail_corr(x, span=None):
+ """L/R correlation over the tail (tail_span unless `span` = (first, last) samples is given); None for no tail"""
+ X = _stereo(x)
+ span = tail_span(X) if span is None else span
+ if span is None:
+ return None
+ a, b = span
+ return corr(X[a:b, 0], X[a:b, 1])
+
+
+def shaped_side(mid, bands, times, pan, diffuse, sign=1.0, sr=SR):
+ """a side signal made from the mid: per band (band_masks of `bands`) the band's mid times its pan gain plus the
+ band's quadrature (Hilbert transform: uncorrelated with the mid at zero lag, the same envelope) times its diffuse
+ gain, both gains linear between keyframes at `times` (pan[b], diffuse[b]: one value per time)"""
+ mid = np.asarray(mid, np.float64)
+ n = len(mid)
+ if n == 0:
+ return np.zeros(0)
+ nfft = sfft.next_fast_len(n + 8192)
+ F = np.fft.rfft(mid, nfft)
+ F[0] = 0.0
+ W = band_masks(bands, nfft, sr)
+ t = np.arange(n) / sr
+ tk = np.asarray(times, np.float64)
+ side = np.zeros(n)
+ for b in range(len(bands)):
+ pb = np.asarray(pan[b], np.float64)
+ db = np.asarray(diffuse[b], np.float64)
+ if np.abs(pb).max() > 1e-4:
+ side += np.interp(t, tk, pb) * np.fft.irfft(F * W[b], nfft)[:n]
+ if db.max() > 1e-4:
+ side += np.interp(t, tk, db) * np.fft.irfft(F * W[b] * (-1j * sign), nfft)[:n]
+ return side
+
+
+# ================================================================================================ the wet layer
+PARTS = {'side': ('side_db', 4099), 'mid': ('mid_db', 8191)} # gains key, seed salt (independent noises)
+
+
+def _gains(model, part):
+ g = model.get(PARTS[part][0])
+ return np.full(len(model['bands']), -150.0) if g is None else np.asarray(g, np.float64)
+
+
+def impulse(model, part='side', seed=0, flat_iters=FLAT_ITERS):
+ """one part's impulse response: per band seeded noise (band_masks), a rise over `build` after `predelay`, an
+ exponential decay to -60 dB at the band's rt60, unit energy per band times 10^(gain / 20); zeros(1) when off.
+ Each band is flattened (`flat_iters` passes: its spectrum's magnitude set to the band's shape, its phase kept, the
+ envelope put back): plain decayed noise has a random power at every frequency (exponentially distributed), so a
+ tonal dry's few partials got a wet 2-3 dB (std) louder or softer from seed to seed; flattened, 0.3-1 dB"""
+ bands = np.asarray(model['bands'], np.float64)
+ rt = np.clip(np.asarray(model['rt60'], np.float64), 0.02, MAX_RT)
+ g = _gains(model, part)
+ on = g > -140
+ if not on.any():
+ return np.zeros(1)
+ pd, build = float(model.get('predelay', 0.0)), max(float(model.get('build', 0.005)), 1e-4)
+ L = int((pd + build + min(MAX_RT, float(rt[on].max()))) * SR) + 1
+ r = np.random.default_rng((int(seed) * 7919 + PARTS[part][1]) & 0xFFFFFFFF)
+ nfft = sfft.next_fast_len(2 * L)
+ F = np.fft.rfft(r.standard_normal(nfft))
+ W = band_masks(bands, nfft, SR)
+ t = np.arange(L) / SR - pd
+ rise = np.where(t >= 0, 1.0 - np.exp(-np.maximum(t, 0.0) / build), 0.0)
+ ir = np.zeros(L)
+ for b in np.nonzero(on)[0]:
+ env = rise * np.exp(-6.907755 * np.maximum(t, 0.0) / rt[b])
+ u = np.fft.irfft(F * W[b], nfft)[:L] * env
+ for _ in range(int(flat_iters)):
+ U = np.fft.rfft(u, nfft)
+ u = np.fft.irfft(W[b] * U / (np.abs(U) + 1e-20), nfft)[:L] * env
+ ir += u * 10 ** (g[b] / 20) / (np.sqrt((u ** 2).sum()) + 1e-20)
+ return ir
+
+
+def wet(model, dry, seed=0, n=None, part='side'):
+ """one part of the wet layer, (n,): the dry's mono sum convolved with impulse(model, part, seed), cut or padded
+ to n samples (default the dry's length)"""
+ m = _stereo(dry).mean(1)
+ n = len(m) if n is None else int(n)
+ ir = impulse(model, part, seed)
+ if not np.any(ir) or not len(m):
+ return np.zeros(n)
+ w = signal.fftconvolve(m, ir)[:n]
+ return np.pad(w, (0, n - len(w)))
+
+
+def tail_s(model):
+ """how long the wet rings past the dry (s): predelay + build + the longest rt60 of a part that is on"""
+ rt = np.asarray(model['rt60'], np.float64)
+ on = (_gains(model, 'side') > -140) | (_gains(model, 'mid') > -140)
+ return 0.0 if not on.any() else float(model.get('predelay', 0.0)) + float(model.get('build', 0.005)) + min(
+ MAX_RT, float(rt[on].max()))
+
+
+def _orthogonal(w, refs, bands=BANDS):
+ """w with, per band (band_masks), its zero-lag projection on each reference signal removed (Gram-Schmidt): a
+ random wet that happens to line up with the dry's own side or mid would cancel it or lean the image"""
+ n = len(w)
+ nfft = sfft.next_fast_len(n + 8192) # padded: the band split must not wrap around in time
+ Wm = band_masks(bands, nfft, SR)
+ Fw = np.fft.rfft(w, nfft)
+ Fr = [np.fft.rfft(r, nfft) for r in refs]
+ out = np.zeros(n)
+ for b in range(len(bands)):
+ wb = np.fft.irfft(Fw * Wm[b], nfft)[:n]
+ basis = []
+ for F in Fr:
+ v = np.fft.irfft(F * Wm[b], nfft)[:n]
+ for u in basis:
+ v = v - (v @ u) * u
+ nv = np.sqrt(v @ v)
+ if nv > 1e-12:
+ basis.append(v / nv)
+ for u in basis:
+ wb = wb - (wb @ u) * u
+ out += wb
+ return out
+
+
+def render_space(model, dry, seed=0, width=1.0, ring=False, limit=True):
+ """the dry render (mono or (n, 2)) plus the wet layer: (n, 2). The wet side is made orthogonal (per band, zero
+ lag) to the dry's side and mid, so it only ever adds width. width scales the side part; ring=True extends the
+ result until the wet has rung out (a take); limit: peak-limit to -1 dBFS (sines.finish, no fade-in: the dry
+ already starts at zero)"""
+ y = _stereo(dry).copy()
+ if not model:
+ return y
+ if ring:
+ y = np.vstack([y, np.zeros((int(tail_s(model) * SR) + 1, 2))])
+ ws, wm = _wet_parts(model, y, seed)
+ ws = ws * float(width)
+ y[:, 0] += wm + ws
+ y[:, 1] += wm - ws
+ return finish(y, SR, fade_s=0.0, limit=limit)
+
+
+def _wet_parts(model, y, seed):
+ """(side, mid) wet signals for the stereo dry y (same length): the side orthogonal to the dry's side and mid"""
+ ws = wet(model, y, seed, len(y), 'side')
+ wm = wet(model, y, seed, len(y), 'mid')
+ if np.any(ws):
+ ws = _orthogonal(ws, [0.5 * (y[:, 0] - y[:, 1]), y.mean(1) + wm])
+ return ws, wm
+
+
+# ================================================================================================ fitting
+def _decay_rate(e_db, t, depth=30.0, min_pts=4):
+ """dB / s of the line through a band's tail envelope (frames within `depth` dB of its tail maximum), None
+ when it does not fall"""
+ if len(e_db) < min_pts:
+ return None
+ sel = e_db >= e_db.max() - depth
+ if sel.sum() < min_pts:
+ return None
+ s = float(np.polyfit(t[sel], e_db[sel], 1)[0])
+ return s if s < -1.0 else None
+
+
+def _smooth(P, k=5):
+ return np.apply_along_axis(lambda v: np.convolve(v, np.ones(k) / k, mode='same'), -1, P)
+
+
+def needs_space(x, max_corr=TAIL_CORR_MAX):
+ """True when x is stereo and its tail's L/R correlation is at most max_corr (a decorrelated, reverberant tail)"""
+ if not is_stereo(x):
+ return False
+ c = tail_corr(x)
+ return c is not None and c <= max_corr
+
+
+def fit_space(x, sr=SR, dry=None, max_corr=TAIL_CORR_MAX, iters=3, need_db=NEED_DB, mid_need_db=MID_NEED_DB):
+ """a space model for recording x (path or array, (n, 2)), or None when x is mono, has no tail or its tail
+ correlation is above max_corr, or when the dry render (mono or (n, 2), the model's seed-0 render through its
+ post stage) already carries the tail's diffuse side to within need_db (module doc)"""
+ if isinstance(x, str):
+ from ... import audio
+ x, sr = audio.load(x, sr=SR), SR
+ X = _stereo(_resample(x, sr))
+ if not is_stereo(X):
+ return None
+ span = tail_span(X)
+ tc = tail_corr(X, span)
+ if tc is None or tc > max_corr:
+ return None
+ n = len(X)
+ P = side_powers(X)
+ dif = _smooth(diffuse_power(P))
+ t = P['t']
+ tail = (t >= span[0] / SR) & (t <= span[1] / SR)
+ if tail.sum() < 3:
+ return None
+ E_dif = dif[:, tail].sum(1)
+ E_mid = _smooth(P['PM'])[:, tail].sum(1)
+ need = {'side': E_dif.copy(), 'mid': np.zeros(len(BANDS))}
+ D, tc_dry = None, 1.0
+ if dry is not None:
+ D = _stereo(dry)[:n]
+ D = np.vstack([D, np.zeros((n - len(D), 2))])
+ tc_dry = tail_corr(D, span)
+ if tc_dry is None or tc_dry <= tc + 0.1: # the dry is about as decorrelated already
+ return None
+ PD = side_powers(D)
+ need['side'] = np.maximum(E_dif - _smooth(diffuse_power(PD))[:, tail].sum(1), 0.0)
+ short = np.maximum(E_mid - _smooth(PD['PM'])[:, tail].sum(1), 0.0) # the dry's mono tail falls short
+ need['mid'] = np.where(short >= E_mid * 10 ** (mid_need_db / 10), short, 0.0)
+ total = float(need['side'].sum())
+ need_rel = _db(total) - _db(float(E_dif.sum()))
+ if total <= 0 or need_rel < need_db:
+ return None
+ B = len(BANDS)
+ live = {p: v > max(float(v.max()), 1e-30) * 1e-4 for p, v in need.items()}
+ # starting rt60s: the decay of the recording's diffuse side per band over the tail; a tail that falls 12-45 dB
+ # over its span cannot ring much longer than RT_SPAN x the span (a band that hardly falls would run to MAX_RT)
+ tt = t[tail]
+ rt_max = float(np.clip(RT_SPAN * (span[1] - span[0]) / SR, 0.1, MAX_RT))
+ rate_need = [(_decay_rate(_db(dif[b, tail]), tt) if live['side'][b] else None) for b in range(B)]
+ rts = [(-60.0 / s if s else None) for s in rate_need]
+ known = [v for v in rts if v]
+ fill = float(np.median(known)) if known else 0.5
+ rt = np.clip([v if v else fill for v in rts], 0.05, rt_max)
+ # the build-up: how far the diffuse side's loudest frame lags the mid's
+ lag = float(t[int(np.argmax(dif.sum(0)))] - t[int(np.argmax(P['PM'].sum(0)))])
+ model = {'kind': KIND, 'version': VERSION, 'predelay': 0.002, 'build': round(float(np.clip(lag, 0.005, 0.08)), 4),
+ 'bands': list(BANDS), 'rt60': [round(float(v), 4) for v in rt],
+ 'side_db': [0.0 if live['side'][b] else -150.0 for b in range(B)],
+ 'mid_db': [0.0 if live['mid'][b] else -150.0 for b in range(B)], 'tail_corr': round(float(tc), 4)}
+ src = X.mean(1) if D is None else D # no dry: the recording's own mid stands in for it
+ for _ in range(int(iters)):
+ r_ = np.asarray(model['rt60'], float)
+ for part in ('side', 'mid'):
+ if not live[part].any():
+ continue
+ Pw = _smooth(band_powers(wet(model, src, 0, n, part)))
+ Ew = Pw[:, tail].sum(1)
+ g = _gains(model, part)
+ for b in range(B):
+ if not live[part][b] or Ew[b] <= 0:
+ continue
+ g[b] += 10 * np.log10(need[part][b] / Ew[b])
+ if part == 'side':
+ sw, sn = _decay_rate(_db(Pw[b, tail]), tt), rate_need[b]
+ if sw and sn: # the render decays slower (sw > sn) -> shorter rt60
+ r_[b] = np.clip(r_[b] * np.clip(sw / sn, 0.5, 2.0), 0.05, rt_max)
+ model[PARTS[part][0]] = [round(float(np.clip(v, -150.0, 40.0)), 2) if live[part][b] else -150.0
+ for b, v in enumerate(g)]
+ model['rt60'] = [round(float(v), 4) for v in r_]
+ mid_on = bool(live['mid'].any())
+ if mid_on: # the mid part changes the mono sum: only if the mono gets no worse
+ from .quality import mel_distance
+ if mel_distance(X, render_space(model, src, seed=0, limit=False)) > mel_distance(X, src) + 0.02:
+ model['mid_db'] = [-150.0] * B
+ mid_on = False
+ # the side's overall level onto the recording's tail correlation (the energy match lands near it; the correlation
+ # is what is heard): L = dry + mid + g side, R = dry + mid - g side over the tail, g on a 0.5 dB grid of +-OFFSET_DB
+ # A short tail holds few independent cycles (a 40 ms tail at 125 Hz: about five), so one render's tail correlation
+ # is a noisy statistic: the products are pooled over FIT_SEEDS renders, and their correlation is what is matched
+ i, j = span
+ S2 = _stereo(src)
+ acc = np.zeros(6)
+ for s in FIT_SEEDS:
+ ws, wm = _wet_parts(model, S2, s)
+ ws, wm = ws[i:j], wm[i:j]
+ a, b_ = S2[i:j, 0] + wm, S2[i:j, 1] + wm
+ acc += [a @ a, b_ @ b_, ws @ ws, a @ b_, a @ ws, b_ @ ws]
+ aa, bb, ss, ab, as_, bs = acc
+ offs = np.arange(-OFFSET_DB, OFFSET_DB + 1e-9, 0.5)
+ g = 10 ** (offs / 20)
+ num = ab - g * as_ + g * bs - g * g * ss
+ den = np.sqrt(np.maximum((aa + 2 * g * as_ + g * g * ss) * (bb - 2 * g * bs + g * g * ss), 1e-30))
+ k = int(np.argmin(np.abs(num / den - tc)))
+ off, tc_wet = float(offs[k]), float(num[k] / den[k])
+ model['side_db'] = [round(v + off, 2) if v > -140 else -150.0 for v in model['side_db']]
+ # analysis by synthesis: keep the layer only if it brings the tail's correlation closer to the recording's
+ if abs(tc_wet - tc) >= abs(tc_dry - tc):
+ return None
+ model['fit'] = {'need_db': round(float(need_rel), 2),
+ 'mid_need_db': (round(float(_db(need['mid'].sum()) - _db(E_mid.sum())), 2) if mid_on else None),
+ 'iters': int(iters), 'dry': D is not None, 'side_offset_db': off,
+ 'tail_corr_dry': round(float(tc_dry), 4), 'tail_corr_wet': round(float(tc_wet), 4)}
+ return model
+
+
+def describe(model):
+ """one line of words about a space model"""
+ if not model:
+ return 'no space layer'
+ side, mid = _gains(model, 'side') > -140, _gains(model, 'mid') > -140
+ on = side | mid
+ if not on.any():
+ return 'space layer (silent)'
+ rt = np.asarray(model['rt60'], float)[on]
+ bands = np.asarray(model['bands'], float)[on]
+ return (f"diffuse tail ({'side + mid' if mid.any() else 'side only'}): rt60 {rt.min():.2f}-{rt.max():.2f} s over "
+ f"{bands.min():.0f}-{bands.max():.0f} Hz, build {model['build'] * 1000:.0f} ms, recording's tail corr "
+ f"{model.get('tail_corr', float('nan')):.2f}")
diff --git a/earshot/forge/fit/spectral.py b/earshot/forge/fit/spectral.py
index 14d5611..5c7cb99 100644
--- a/earshot/forge/fit/spectral.py
+++ b/earshot/forge/fit/spectral.py
@@ -46,12 +46,20 @@ are noise (or dense grains) shaped by a time-varying spectral envelope. The mode
bands cannot hold. Helps long steady sounds (the riser: CLAP 0.82 -> 0.88), costs a little on short
ones (estimation noise), hence off by default.
fit what the fit chose: gesture_ms (smoothing), rank, tol_db, fluct_db.
+ finish True on every fit made by 0.4: auto.render ends such a model's renders with the 0.4 fade-in and -1 dBFS
+ peak limit (auto.finished); a model without it (fitted by 0.3) renders as it did, sample for sample.
+The detail fit (fit(**DETAIL)) keeps a 10 ms gesture to 0.5 dB with no denoise or texture-aware widening and tracks
+up to 100 ridges: pulses, stutter gates and short tonal grains stay in the gesture (the gesture model's noise layer).
Rendering (`render`): each call draws NEW noise (seeded, deterministic per seed), so every render is a fresh but
statistically identical sound; knobs: length (time-stretch of the gesture, texture rates kept), speed (movement
speed: shorter, higher by SPEED_OCT octaves per doubling, faster flutter), sweep (scales the spectral centroid's
excursion around its mean), shift (octaves), tilt (dB / octave about 1 kHz), roughness (texture depth; 0 = smooth
-Gaussian), width (side / pan amount), ridges (tonal ridge gain), gain_db. Level: the gain that puts the EXPECTED
+Gaussian), width (side / pan amount), ridges (tonal ridge gain), gain_db, and the motion knobs, which read the
+gesture's trajectory (`trajectory`: level, centroid, brightness, pan and width per keyframe, the peak and its 20 dB
+rise / fall): doppler (semitones of a pass-by pitch bend, a time warp turning at the peak), proximity (brightness and
+level follow the distance: quieter keyframes get duller), pan (factor on the fitted L/R level differences) and motion
+(a pass-by pan from one side to the other). Level: the gain that puts the EXPECTED
loudest 50 ms RMS (computed from the gesture: band envelopes, their overlaps, the ridges) on the original's
(+ gain_db), times the take's own correction to its expected K-weighted energy (BS.1770 weighting, the loudness a
meter reads). A noise band holds few independent bins over a short gesture, so a take's loudness wanders with the
@@ -91,6 +99,12 @@ NFFTS = (512, 1024, 2048, 4096)
NFFT_CYCLES = 2.0 # analysis window ~ this many / bandwidth per band
FLOOR_DB = 90.0 # envelope floor below its loudest cell
SPREAD = 1.0 # grain amplitude spread (log-normal sigma, nepers)
+# the detail fit (fit(**DETAIL)): a 10 ms gesture kept to 0.5 dB, no denoise and no texture-aware widening, up to 100
+# tonal ridges from 60 Hz that stand 5 dB over their floor for 20 ms. Designed whooshes, risers and swells carry
+# pulses, stutter gates and short tonal grains the smooth default turns into a wash (on the long film gestures it
+# lifted CLAP 0.49 -> 0.87, 0.59 -> 0.74, 0.63 -> 0.81); the gesture model's noise layer (fit/gesture.py)
+DETAIL = {'gesture_ms': 10.0, 'tol_db': 0.5, 'max_keys': 600, 'denoise': False, 'adapt': False, 'max_ridges': 100,
+ 'ridge_db': 5.0, 'ridge_ms': 20.0, 'ridge_lo': 60.0}
def _workers():
@@ -328,8 +342,11 @@ def _gauss_process(m, fs, rates, spec, rng):
return z / sd if sd > 0 else z
-def _clicks(N, rate, rng, spread=SPREAD):
- """unit-variance Poisson click train: `rate` clicks / s, log-normal amplitudes with random signs"""
+def _clicks(N, rate, rng, spread=SPREAD, norm_ms=0.0):
+ """unit-variance Poisson click train: `rate` clicks / s, log-normal amplitudes with random signs. norm_ms > 0 (a
+ texture's grains['norm_ms']): levelled to unit power over windows of norm_ms, at least GRAIN_NORM_CLICKS clicks
+ long (_local_norm): the clicks keep their spread inside a window, but no stretch of a take is louder than the
+ expectation because one draw put its largest clicks there"""
k = int(rng.poisson(max(rate, 1e-3) * N / SR))
c = np.zeros(N)
if k == 0:
@@ -337,10 +354,34 @@ def _clicks(N, rate, rng, spread=SPREAD):
pos = rng.integers(0, N, k)
amp = np.exp(spread * rng.standard_normal(k)) * rng.choice((-1.0, 1.0), k)
np.add.at(c, pos, amp)
+ if norm_ms > 0:
+ w = int(max(norm_ms * 1e-3 * SR, GRAIN_NORM_CLICKS * SR / max(rate, 1e-3)))
+ if w < N // 2:
+ c = _local_norm(c, w)
p = np.sqrt((c ** 2).mean())
return c / p if p > 0 else c
+GRAIN_NORM_CLICKS = 20.0 # a levelled grain train's window holds at least this many clicks on average
+
+
+def _local_norm(x, w):
+ """x levelled to unit mean power over every w-sample window (centred moving averages of x^2 and of the gain): the
+ realisation's energy follows its expectation, its fine structure stays"""
+ w = max(3, int(w)) | 1
+ p = _movavg(x * x, w)
+ g = 1.0 / np.sqrt(np.maximum(p, 1e-3 * max(float(p.mean()), 1e-30)))
+ return x * _movavg(g, w)
+
+
+def _movavg(v, w):
+ """centred moving average over w (odd) samples, over what is there at the edges"""
+ c = np.concatenate([[0.0], np.cumsum(v)])
+ i = np.arange(len(v))
+ a, b = np.clip(i - w // 2, 0, len(v)), np.clip(i + w // 2 + 1, 0, len(v))
+ return (c[b] - c[a]) / np.maximum(b - a, 1)
+
+
def _k_gain2(f):
"""|K-weighting|^2 (ITU-R BS.1770, 48 kHz) at frequencies f (Hz)"""
from scipy import signal
@@ -349,11 +390,12 @@ def _k_gain2(f):
def _synth(n, tk, env, cents, seed, stereo=None, texture=None, ridges=(), mono=False, rate_scale=1.0,
- rough=1.0, fine=None, fine_shift=0.0, power=False):
+ rough=1.0, fine=None, fine_shift=0.0, power=False, ridge_sig=None):
"""the renderer core. tk (K,) key times in s, env (K, B) density dB. Returns (n,) if mono else (n, 2).
power=True returns (y, p, pk): the expected channel-mean power of the render, plain and K-weighted ((n,) each:
the bands' envelopes with their neighbours' overlap, the ridges; flutter has E[mod^2] = 1), the same for every
- seed (render levels by them)"""
+ seed (render levels by them). ridge_sig (mono only): the ridges already rendered for this seed (_ridge_sig: the
+ texture fit renders one model many times; the ridges draw from their own stream, so the sum is the same)"""
n = int(n)
ss = np.random.SeedSequence(abs(int(seed)) % (2 ** 32))
rA, rB, rC, rM, rR = [np.random.default_rng(s) for s in ss.spawn(5)]
@@ -368,7 +410,7 @@ def _synth(n, tk, env, cents, seed, stereo=None, texture=None, ridges=(), mono=F
eA = rA.standard_normal(N)
if grains and rough > 0:
dens = float(grains['density']) / max(rough, 1e-3) ** 2 # rougher = sparser clicks
- eA = _clicks(N, dens * rate_scale, rC)
+ eA = _clicks(N, dens * rate_scale, rC, norm_ms=float(grains.get('norm_ms') or 0.0))
FA = sfft.rfft(eA, workers=wk)
FB = None if (mono or stereo is None) else sfft.rfft(rB.standard_normal(N), workers=wk)
f = np.fft.rfftfreq(N, 1 / SR)
@@ -448,6 +490,9 @@ def _synth(n, tk, env, cents, seed, stereo=None, texture=None, ridges=(), mono=F
L += amp * gl * (c * nA + s * nB)
R += amp * gr * (c * nA - s * nB)
+ if ridge_sig is not None:
+ L += ridge_sig[:n]
+ ridges = ()
for rd in ridges or ():
rt = np.asarray(rd['t'], np.float64)
fr = np.interp(ts, rt, np.asarray(rd['f'], np.float64))
@@ -719,8 +764,17 @@ def _fit_fine(xm, model, seeds=(41, 42), verbose=False):
def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0, max_rank=16, iters=3, diff_frames=3,
- ridges=True, fine=False, texture=True, stereo=True, seeds=(1, 2), verbose=False):
- """fit a spectral-noise model to a recording (path or (n,)/(n, 2) 48 kHz array); returns the model dict"""
+ ridges=True, fine=False, texture=True, stereo=True, seeds=(1, 2), max_ridges=4, ridge_db=8.0, ridge_ms=50.0,
+ ridge_lo=400.0, denoise=True, adapt=True, quiet_db=50.0, grain_norm_ms=0.0, grain_halves=False,
+ verbose=False):
+ """fit a spectral-noise model to a recording (path or (n,)/(n, 2) 48 kHz array); returns the model dict.
+ max_ridges / ridge_db / ridge_ms / ridge_lo: how many tonal ridges, how far over the local floor (dB), how long
+ (ms) and from what frequency (Hz) up. denoise=False skips the low-rank denoise and adapt=False the texture-aware
+ widening of the smoothing (the tolerance then stays at tol_db): a detail fit that keeps pulses, clicks and stutter
+ gates as gesture (DETAIL, the gesture model's noise layer) instead of texture statistics. quiet_db: cells this far
+ under the loudest weigh nothing in the keyframe choice (their level between keyframes is interpolated).
+ grain_norm_ms / grain_halves (0.4, the gesture model's air): the grain texture levelled (_clicks) and kept only
+ when the fast log-amplitude variance stands out in both halves of the active span (_fit_texture)"""
if name is None and isinstance(x, str):
name = os.path.basename(x)
x2 = _load(x)
@@ -735,7 +789,9 @@ def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0
rmods = []
if ridges:
- for tr in _find_ridges(xm, D, cents):
+ min_frames = max(2, int(round(ridge_ms / (1000.0 * HOP / SR))))
+ for tr in _find_ridges(xm, D, cents, max_ridges=max_ridges, min_prom=ridge_db, min_frames=min_frames,
+ f_lo=ridge_lo):
rm, (k, fz, amp) = _ridge_model(tr)
rmods.append(rm)
D = _ridge_remove(D, cents, k, fz, amp)
@@ -749,7 +805,7 @@ def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0
# flutter; a quadratic follows the gesture's own curvature); widen the smoothing while that wander is well
# above tolerance, the rest is left to the texture statistics
L1 = _db((_seg_smooth(D, k, on) * bw[:, None]).sum(0))
- fluct = _wander(L1, k, on)
+ fluct = _wander(L1, k, on) if adapt else 0.0
if fluct <= 2.0 * tol_db or k >= k_max:
break
k = min(k_max, int(np.ceil(k * 1.5)))
@@ -759,17 +815,20 @@ def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0
act = tgt > top - 70
Gp0 = tgt + 10 * np.log10(bw)[:, None]
W = np.clip((Gp0 - (Gp0.max() - 50)) / 25, 0, 1).T # (T, B) cell weights
- # low-rank denoising: keep only the structure that stands above what a noise band's estimate wanders
- sig = _noise_sigma(n, cents, k, on)
- sw = float(np.sqrt((W * sig[None] ** 2).sum() / max(W.sum(), 1e-9)))
- rank = max_rank
Y = tgt.T
- for K in range(1, max_rank + 1):
- Yh = _lowrank(Y, W, K)
- r = float(np.sqrt((W * (Y - Yh) ** 2).sum() / max(W.sum(), 1e-9)))
- if r <= sw:
- rank = K
- break
+ if denoise:
+ # low-rank denoising: keep only the structure that stands above what a noise band's estimate wanders
+ sig = _noise_sigma(n, cents, k, on)
+ sw = float(np.sqrt((W * sig[None] ** 2).sum() / max(W.sum(), 1e-9)))
+ rank = max_rank
+ for K in range(1, max_rank + 1):
+ Yh = _lowrank(Y, W, K)
+ r = float(np.sqrt((W * (Y - Yh) ** 2).sum() / max(W.sum(), 1e-9)))
+ if r <= sw:
+ rank = K
+ break
+ else:
+ Yh, rank, r, sw = Y, 0, 0.0, 0.0
# quiet cells (weight < 1) keep their measured level: the low-rank fill there is a guess (a dark corner where
# the highs have died must stay dark)
tgtK = np.maximum((W * Yh + (1 - W) * Y).T, top - FLOOR_DB)
@@ -790,14 +849,14 @@ def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0
ild, icc, gw = _stereo_traj(A['stereo'], T, mono or not stereo)
Gp = Gm + 10 * np.log10(bw)[:, None]
- wc = np.clip((Gp - (Gp.max() - 50)) / 25, 0, 1).T # (T, B)
+ wc = np.clip((Gp - (Gp.max() - quiet_db)) / 25, 0, 1).T # (T, B)
lvl = _db((10 ** (Gp / 10)).sum(0))
peak = int(np.argmax(lvl))
onset = int(np.argmax(lvl > lvl.max() - 30))
gact = np.clip((lvl - (lvl.max() - 40)) / 20, 0, 1)[:, None] * gw * len(GROUPS) # (T, G)
Y = np.concatenate([Gm.T, ild / 2.0, icc * 10.0], 1)
Wc = np.concatenate([wc, 0.5 * gact, 0.5 * gact], 1)
- tol = max(tol_db, fluct)
+ tol = max(tol_db, fluct) # (fluct is 0 with adapt=False)
keys = _keyframes(Y, Wc, tol, must=(peak, onset), max_keys=max_keys)
model = {
@@ -814,11 +873,12 @@ def fit(x, name=None, n_bands=N_BANDS, tol_db=1.0, max_keys=160, gesture_ms=25.0
'fine': None,
'texture': None,
'ridges': rmods,
+ 'finish': True, # 0.4 (module doc): auto.render's fade-in and peak limit apply
}
if fine:
model['fine'] = _fit_fine(xm, model, verbose=verbose)
if texture:
- model['texture'] = _fit_texture(xm, model, verbose=verbose)
+ model['texture'] = _fit_texture(xm, model, verbose=verbose, grain_norm_ms=grain_norm_ms, halves=grain_halves)
return model
@@ -905,6 +965,15 @@ def _texture_stats(xm, Eg, onset_s=None):
return {'spec': spec, 'kurt': kurt, 'share': share, 'lows': lows, 'halves': halves}
+def _grains(density, norm_ms=0.0):
+ """a texture's grains entry: {'density' clicks / s, 'spread' (log-normal sigma), 'norm_ms' (> 0 only: the click
+ train levelled over windows this long, _clicks)}"""
+ g = {'density': density, 'spread': SPREAD}
+ if norm_ms > 0:
+ g['norm_ms'] = float(norm_ms)
+ return g
+
+
def _coherence(st):
"""mean correlation of the low-rate log amplitude between the two loudest active groups"""
ok = [g for g in range(len(st['share'])) if st['lows'][g] is not None]
@@ -920,15 +989,26 @@ def _coherence(st):
return float((a * b).sum() / den) if den > 0 else 0.0
-def _mono_render(model, seed, texture):
+def _mono_render(model, seed, texture, ridge_sig=None):
cents = centers(model['bands']['n'])
n = int(round(model['dur'] * SR))
return _synth(n, model['env']['t'], model['env']['db'], cents, seed, texture=texture,
- ridges=model.get('ridges') or (), mono=True, fine=model.get('fine'))
+ ridges=model.get('ridges') or (), mono=True, fine=model.get('fine'), ridge_sig=ridge_sig)
+
+
+def _ridge_sig(model, seed):
+ """the ridges of a mono render (seed) alone: the noise bands rendered 400 dB down"""
+ if not model.get('ridges'):
+ return None
+ cents = centers(model['bands']['n'])
+ n = int(round(model['dur'] * SR))
+ quiet = np.asarray(model['env']['db'], np.float64) * 0.0 - 400.0
+ return _synth(n, model['env']['t'], quiet, cents, seed, ridges=model['ridges'], mono=True)
-def _avg_stats(model, n, Eg, texture, seeds, onset_s):
- sts = [_texture_stats(_mono_render(model, s, texture)[:n], Eg, onset_s) for s in seeds]
+def _avg_stats(model, n, Eg, texture, seeds, onset_s, rsig=None):
+ rsig = rsig or {}
+ sts = [_texture_stats(_mono_render(model, s, texture, rsig.get(s))[:n], Eg, onset_s) for s in seeds]
spec = np.nanmean(np.stack([s['spec'] for s in sts]), 0)
kurt = np.nanmean(np.stack([s['kurt'] for s in sts]), 0)
halves = np.nanmean(np.stack([s['halves'] for s in sts]), 0)
@@ -954,14 +1034,18 @@ def _fit_am(Sr, base, wg, ok, low):
'depth': [round(float(v), 4) for v in depth], 'coh': round(coh, 3)}
-def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False):
+def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False, grain_norm_ms=0.0, halves=False):
"""texture statistics by analysis-by-synthesis on mono renders of the gesture:
grains are detected by excess fast (>= 128 Hz) log-amplitude variance (Poisson clicks put variance at every rate,
flutter does not) and their density is fitted to the std of the log analytic envelope (monotone, stable across
seeds; kurtosis is not: it is ruled by the largest click); AM is the low-rate (<= 64 Hz) log-amplitude variance
- beyond what Gaussian noise / the grains already make. Grains and AM are refitted once against each other."""
+ beyond what Gaussian noise / the grains already make. Grains and AM are refitted once against each other.
+ halves=True: grains only when the excess fast variance is there in both halves of the active span too (texture
+ is stationary; one transient of a short sound is not grain texture: AM is judged that way already).
+ grain_norm_ms: the grains' norm_ms (_clicks), in the fit's renders as in the model's"""
import warnings
n = len(xm)
+ rsig = {s: _ridge_sig(model, s) for s in seeds} if model.get('ridges') else None
Eg = _group_env(model, n)
tk = np.asarray(model['env']['t'])
lv = np.asarray(model['env']['db']).max(1)
@@ -971,7 +1055,7 @@ def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False):
with warnings.catch_warnings():
warnings.simplefilter('ignore', RuntimeWarning)
Sr = _texture_stats(xm, Eg, onset_s)
- S0 = _avg_stats(model, n, Eg, None, seeds, onset_s)
+ S0 = _avg_stats(model, n, Eg, None, seeds, onset_s, rsig)
ok = ~np.isnan(Sr['kurt']) & ~np.isnan(S0['kurt'])
if not ok.any():
return None
@@ -987,6 +1071,14 @@ def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False):
'lenv_real': rnd(Sr['kurt']), 'lenv_gauss': rnd(S0['kurt']),
'fast_var_real': rnd(hr), 'fast_var_gauss': rnd(h0), 'fast_ratio': round(hratio, 2)}}
grainy = hx > max(0.03, 3 * float(np.sum(wg * hsd))) and hratio > 1.5
+ hh = []
+ for h in (0, 1): # the fast variance in each half of the active span apart
+ hrh = np.nansum(np.nan_to_num(Sr['halves'][h])[:, high], 1)
+ h0h = np.nansum(np.nan_to_num(S0['halves'][h])[:, high], 1)
+ hh.append(float(np.sum(wg * hrh) / max(np.sum(wg * h0h), 1e-9)))
+ tex['stats']['fast_ratio_halves'] = [round(v, 2) for v in hh]
+ if halves and grainy and min(hh) <= 1.5:
+ grainy = False
if verbose:
print(' texture: lenv real', np.round(Sr['kurt'], 2), 'gauss', np.round(S0['kurt'], 2),
f'fast var ratio {hratio:.2f}', 'grainy' if grainy else '')
@@ -995,7 +1087,7 @@ def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False):
cache = {}
def loss(d):
- st = _avg_stats(model, n, Eg, {'am': am, 'grains': {'density': d}}, seeds, onset_s)
+ st = _avg_stats(model, n, Eg, {'am': am, 'grains': _grains(d, grain_norm_ms)}, seeds, onset_s, rsig)
cache[d] = st
return float(np.nansum(wg * (st['kurt'] - Sr['kurt']) ** 2))
grid = [25, 60, 150, 400, 1000, 2500, 6000, 15000]
@@ -1013,20 +1105,20 @@ def _fit_texture(xm, model, seeds=(31, 32, 33), verbose=False):
base = S0
if grainy:
dens, bl, base = fit_density(None)
- tex['grains'] = {'density': round(float(dens), 1), 'spread': SPREAD}
+ tex['grains'] = _grains(round(float(dens), 1), grain_norm_ms)
if verbose:
print(f' grains: density {dens:.0f}/s (lenv loss {bl:.4f})')
am = _fit_am(Sr, base, wg, ok, low)
if am is not None and grainy:
# flutter raises the log-envelope spread too: refit the density with the AM in, then the AM again
dens, bl, _ = fit_density(am)
- tex['grains'] = {'density': round(float(dens), 1), 'spread': SPREAD}
- base = _avg_stats(model, n, Eg, {'grains': tex['grains']}, seeds, onset_s)
+ tex['grains'] = _grains(round(float(dens), 1), grain_norm_ms)
+ base = _avg_stats(model, n, Eg, {'grains': tex['grains']}, seeds, onset_s, rsig)
am = _fit_am(Sr, base, wg, ok, low)
if verbose:
print(f' grains refit with AM: density {dens:.0f}/s')
if am is not None:
- S1 = _avg_stats(model, n, Eg, {'am': am, 'grains': tex['grains']}, seeds, onset_s)
+ S1 = _avg_stats(model, n, Eg, {'am': am, 'grains': tex['grains']}, seeds, onset_s, rsig)
spr, sp0, sp1 = [np.nan_to_num(S['spec']) for S in (Sr, base, S1)]
want = np.maximum(spr[:, low].sum(1) - sp0[:, low].sum(1), 0)
got = np.maximum(sp1[:, low].sum(1) - sp0[:, low].sum(1), 1e-6)
@@ -1059,9 +1151,95 @@ def _centroid_oct(E, cents, bw):
return (lw * np.log2(cents)[None]).sum(1) / np.maximum(lw.sum(1), 1e-30), p.sum(1)
+def trajectory(model):
+ """the gesture's motion per keyframe, as recipes and the motion knobs read it: {'t' (s), 'level_db' (total power,
+ dB re full-scale white noise), 'centroid_hz' (loudness-weighted: the sweep knob's), 'tilt_db_oct' (the density's
+ slope in dB per octave where the energy is: a line through the band densities weighted by the bands' power, over
+ the bands within 40 dB of the keyframe's loudest band; pink noise -3, white 0: brightness apart from the
+ centroid), 'pan_db' (power-weighted L/R level difference over the band groups, + = left), 'width' (1 - power-
+ weighted correlation: 0 mono, 1 uncorrelated), 'peak_s' (the loudest keyframe), 'rise_s' / 'fall_s' (20 dB under
+ the peak to it, it to 20 dB under it)}"""
+ check(model)
+ cents = centers(model['bands']['n'])
+ bw = bandwidths(cents)
+ tk = np.asarray(model['env']['t'], np.float64)
+ E = np.asarray(model['env']['db'], np.float64)
+ co, cp = _centroid_oct(E, cents, bw)
+ lv = _db(cp)
+ lc = np.log2(cents / 1000.0)
+ tilt = np.zeros(len(tk))
+ for i, row in enumerate(E):
+ pb = row + 10 * np.log10(bw) # band power (dB)
+ w = 10 ** ((pb - pb.max()) / 10) * (pb > pb.max() - 40.0)
+ mx = (w * lc).sum() / w.sum()
+ tilt[i] = (w * (lc - mx) * row).sum() / max((w * (lc - mx) ** 2).sum(), 1e-9)
+ G = len(GROUPS) - 1
+ pan, wid = np.zeros(len(tk)), np.zeros(len(tk))
+ st = model.get('stereo')
+ if st:
+ gi = _group_of(cents)
+ gp = np.stack([(10 ** (E[:, gi == g] / 10) * bw[gi == g]).sum(1) for g in range(G)], 1) # (K, G)
+ gp = gp / np.maximum(gp.sum(1, keepdims=True), 1e-30)
+ pan = (gp * np.asarray(st['ild'], np.float64)).sum(1)
+ wid = 1.0 - (gp * np.asarray(st['icc'], np.float64)).sum(1)
+ pk = int(np.argmax(lv))
+ on = int(np.argmax(lv > lv[pk] - 20))
+ loud = np.nonzero(lv[pk:] > lv[pk] - 20)[0]
+ off = pk + int(loud[-1]) if len(loud) else pk
+ return {'t': tk.tolist(), 'level_db': [round(float(v), 2) for v in lv],
+ 'centroid_hz': [round(float(2 ** v), 1) for v in co], 'tilt_db_oct': [round(float(v), 2) for v in tilt],
+ 'pan_db': [round(float(v), 2) for v in pan], 'width': [round(float(v), 3) for v in wid],
+ 'peak_s': float(tk[pk]), 'rise_s': float(tk[pk] - tk[on]), 'fall_s': float(tk[off] - tk[pk])}
+
+
+def _motion_tau(tr, s=1.0):
+ """the time scale (s) of a pass-by at a gesture's peak: a quarter of its 20 dB rise + fall, at least 15 ms"""
+ return max(0.015, 0.25 * (tr['rise_s'] + tr['fall_s']) * s)
+
+
+def bend(y, t_pk, tau, semitones):
+ """a pass-by pitch bend (Doppler) as a time warp of a render ((n,) or (n, c)): read at the rate
+ 2^(semitones tanh((t_pk - t) / tau) / 12), anchored at the peak, so the sound approaches higher and recedes lower
+ with the turn at t_pk; the output runs until the input is read out (a receding tail gets longer)"""
+ y = np.asarray(y, np.float64)
+ n = len(y)
+ if not semitones or n < 4:
+ return y
+ a = abs(float(semitones)) / 12
+ m = int(np.ceil(n * 2 ** a)) + 2
+ t = np.arange(m) / SR
+ rate = 2.0 ** (float(semitones) * np.tanh((t_pk - t) / max(tau, 1e-3)) / 12)
+ pos = np.concatenate([[0.0], np.cumsum(rate)[:-1]])
+ j = min(int(round(t_pk * SR)), m - 1)
+ pos += t_pk * SR - pos[j] # the peak stays where it was
+ keep = pos <= n - 1
+ m = int(np.nonzero(keep)[0][-1]) + 1 if keep.any() else n
+ idx = np.arange(n)
+ if y.ndim == 1:
+ return np.interp(pos[:m], idx, y, left=0.0, right=0.0)
+ return np.stack([np.interp(pos[:m], idx, y[:, c], left=0.0, right=0.0) for c in range(y.shape[1])], 1)
+
+
+def pass_pan(y, t_pk, tau, amount):
+ """a pass-by pan on an (n, 2) render: constant power, position amount x tanh((t - t_pk) / tau), -1 = left ..
+ +1 = right (unity gains in the centre)"""
+ y = np.asarray(y, np.float64)
+ if not amount:
+ return y
+ t = np.arange(len(y)) / SR
+ a = (np.clip(float(amount) * np.tanh((t - t_pk) / max(tau, 1e-3)), -1, 1) + 1) * np.pi / 4
+ return y * (np.sqrt(2.0) * np.stack([np.cos(a), np.sin(a)], 1))
+
+
def render(model, seed=0, length=1.0, speed=1.0, sweep=1.0, shift=0.0, tilt=0.0, roughness=1.0, width=1.0,
- ridges=1.0, gain_db=0.0, level=True, limit=True, steady=True):
+ ridges=1.0, gain_db=0.0, doppler=0.0, proximity=0.0, pan=1.0, motion=0.0, level=True, limit=True,
+ steady=True):
"""render a model: (n, 2) float32 at 48 kHz. New `seed` = new noise, same statistics. See the module doc.
+ Motion knobs (identity at 0 / 1; trajectory() has what they read): doppler (semitones of a pass-by pitch bend
+ turning at the gesture's peak, + = approach higher), proximity (-1..1: a keyframe d dB under the peak is made
+ p x d x (0.5 + 0.15 x octaves over 1 kHz) dB quieter still, floored at 0 dB per band: farther = quieter and duller;
+ negative flattens), pan (factor on the fitted L/R level differences), motion (-1..1: a pass-by pan from one side to
+ the other turning at the peak).
limit=False skips the -0.5 dBFS sample-peak cap (a per-take gain: the fit engine levels takes itself).
steady=False levels the take by its own loudest 50 ms (the 0.2 renderer; a hybrid's air layer was fitted so)"""
check(model)
@@ -1081,10 +1259,13 @@ def render(model, seed=0, length=1.0, speed=1.0, sweep=1.0, shift=0.0, tilt=0.0,
E = np.stack([np.interp(warp(cents / 2 ** o), ec, row) for o, row in zip(octs, E)])
if tilt:
E = E + float(tilt) * np.log2(cents / 1000.0)[None]
+ if proximity:
+ d = np.clip(_db(cp) - _db(cp).max(), -40.0, 0.0)
+ E = E + float(proximity) * d[:, None] * np.maximum(0.5 + 0.15 * np.log2(cents / 1000.0), 0.0)[None]
st = model.get('stereo')
if st is not None:
icc = 1 - float(width) * (1 - np.asarray(st['icc'], np.float64))
- st = {'ild': np.asarray(st['ild'], np.float64) * float(width), 'icc': np.clip(icc, -1, 1)}
+ st = {'ild': np.asarray(st['ild'], np.float64) * float(width) * float(pan), 'icc': np.clip(icc, -1, 1)}
rds = []
if ridges:
for rd in model.get('ridges') or ():
@@ -1098,6 +1279,10 @@ def render(model, seed=0, length=1.0, speed=1.0, sweep=1.0, shift=0.0, tilt=0.0,
y *= 10 ** ((model['level_db'] + float(gain_db) - _peak_mean_db(p)) / 20) * _take_gain(y, pk) # expectation
elif level:
y *= 10 ** ((model['level_db'] + float(gain_db) - _peak_rms_db(y)) / 20)
+ if doppler or motion:
+ tr = trajectory(model)
+ t_pk, tau = tr['peak_s'] * s, _motion_tau(tr, s)
+ y = pass_pan(bend(y, t_pk, tau, doppler), t_pk, tau, motion)
pk = np.abs(y).max()
if limit and pk > 10 ** (-0.5 / 20):
y *= 10 ** (-0.5 / 20) / pk
@@ -1153,8 +1338,11 @@ def summary(model):
f"{8.686 * max(am['depth']):.1f} dB, coherence {am['coh']:.2f}")
if tx.get('grains'):
w.append(f"crackle: {tx['grains']['density']:.0f} grains / s")
- for rd in model.get('ridges') or ():
+ rds = model.get('ridges') or ()
+ for rd in rds[:4]:
w.append(f"tonal ridge {rd['f'][0]:.0f} -> {rd['f'][-1]:.0f} Hz over {rd['t'][-1] - rd['t'][0]:.2f} s")
+ if len(rds) > 4:
+ w.append(f'{len(rds) - 4} more ridges')
w.append(f"{len(json.dumps(model)) / 1024:.1f} kB JSON")
return '; '.join(w)
@@ -1193,7 +1381,8 @@ def to_code(model, var='MODEL'):
lines.append(f" 'ridges': {model.get('ridges')!r},")
lines.append('}')
lines += ['', '', 'def make(seed=0, **knobs):',
- ' """knobs: length, speed, sweep, shift, tilt, roughness, width, ridges, gain_db"""',
+ ' """knobs: length, speed, sweep, shift, tilt, roughness, width, ridges, gain_db, doppler, proximity, '
+ 'pan, motion"""',
f' return render({var}, seed=seed, **knobs)', '']
return '\n'.join(lines)
diff --git a/tests/test_fit_choice.py b/tests/test_fit_choice.py
new file mode 100644
index 0000000..08c5f3b
--- /dev/null
+++ b/tests/test_fit_choice.py
@@ -0,0 +1,139 @@
+"""The fit engine's choices and the takes' clicks (earshot 0.4, round C2), on synthetic sounds only:
+- a bed is kept only when analysis by synthesis says so: the winner's kind is also fitted on the recording as it is,
+ and the bed is dropped when that scores lower (a whoosh's own steady approach is not a bed);
+- the CLAP margin rule picks a candidate CLAP clearly prefers (on seed 0 and on the confirm seeds' mean) only within
+ the score and envelope tolerances;
+- no clicks from the post stage's gate edges, from a later modal strike's onset, or from a coherent sines strike's
+ atoms jittered apart; models fitted before 0.4 render as before."""
+import numpy as np
+
+from earshot.forge import polish
+from earshot.forge.fit import auto, modal, sines
+
+SR = 48000
+
+
+def _db(v):
+ return 10 * np.log10(np.mean(np.asarray(v, float) ** 2) + 1e-20)
+
+
+def _strike(n, t0, f=3000.0, tau=0.02, amp=0.5):
+ t = np.arange(n) / SR - t0
+ return (t >= 0) * amp * np.exp(-np.clip(t, 0, None) / tau) * np.sin(2 * np.pi * f * np.clip(t, 0, None))
+
+
+def _bed(n, db=-20.0):
+ t = np.arange(n) / SR
+ y = np.sin(2 * np.pi * 410 * t) + 0.6 * np.sin(2 * np.pi * 655 * t + 1.0) + 0.3 * np.random.default_rng(1).standard_normal(n)
+ return y / np.sqrt(np.mean(y ** 2)) * 0.5 / np.sqrt(2) * 10 ** (db / 20)
+
+
+# ------------------------------------------------------------------------------------------------------ bed check
+def test_bed_check_records_both_scores_and_keeps_the_better():
+ n = int(0.2 * SR)
+ x = _strike(n, 0.06) + _bed(n)
+ m = auto.fit(x, candidates=['modal'], fast=True)
+ b = m['fit_engine']['bed']
+ assert b['score_stripped'] is not None and b['score_plain'] is not None
+ assert b['kept'] is (b['score_stripped'] <= b['score_plain'])
+ assert bool((m.get('post') or {}).get('bed')) is b['kept'] # the model carries the bed only when kept
+ assert any(r['candidate'] == 'plain:modal' for r in m['fit_engine']['candidates'])
+
+
+def test_bed_check_drops_a_bed_whose_stripped_fit_loses(monkeypatch):
+ n = int(0.2 * SR)
+ x = _strike(n, 0.06) + _bed(n)
+ ruin = lambda X, bd: 0.05 * np.asarray(X)[::-1] # a stripped "sound" that is not the sound
+ monkeypatch.setattr(auto, 'strip_bed', ruin)
+ m = auto.fit(x, candidates=['modal'], fast=True)
+ b = m['fit_engine']['bed']
+ assert b['kept'] is False and b['score_plain'] < b['score_stripped']
+ assert 'bed' not in m['post']
+ monkeypatch.setattr(auto, 'BED_CHECK', False) # without the check the stripped fit stands
+ m2 = auto.fit(x, candidates=['modal'], fast=True)
+ assert 'bed' in m2['post'] and 'kept' not in m2['fit_engine']['bed']
+
+
+# ------------------------------------------------------------------------------------------------------ CLAP rule
+def _row(clap, score, env=0.97, clap2=None, score2=None):
+ return {'quality': {'clap_cos': clap, 'env_corr': env}, 'score': score, 'clap2': clap2, 'score2': score2}
+
+
+def test_clap_rule_needs_the_margin_on_both_and_a_close_score():
+ rows = {'gesture': _row(0.86, 10.0, clap2=0.85, score2=10.2),
+ 'sines': _row(0.90, 10.9, clap2=0.89, score2=11.1), # clearly closer to CLAP, a near-tie on the score
+ 'modal': _row(0.95, 14.0, clap2=0.94, score2=14.5)} # closer still, but far worse on the score
+ alts = auto._clap_alts(rows, 'gesture')
+ assert alts == ['sines']
+ assert auto._clap_pick(rows, 'gesture', alts) == 'sines'
+ rows['sines']['clap2'] = 0.86 # seed 0 was a lucky draw: the mean decides
+ assert auto._clap_pick(rows, 'gesture', alts) is None
+ rows['sines'].update(clap2=0.89, quality={'clap_cos': 0.90, 'env_corr': 0.80}) # a far worse loudness contour
+ assert auto._clap_pick(rows, 'gesture', alts) is None
+ assert auto._clap_alts({'a': _row(None, 9.0), 'b': _row(0.99, 9.1)}, 'a') == []
+
+
+# ------------------------------------------------------------------------------------------------------ clicks
+def test_gate_edges_do_not_click():
+ n = int(0.2 * SR)
+ t = np.arange(n) / SR
+ y = 0.5 * np.sin(2 * np.pi * 300 * t) * np.exp(-t / 0.3) # a low ring under the cut
+ gate = {'pts': [[0.05, 0.0], [0.0505, -60.0], [0.08, -60.0], [0.0805, 0.0], [0.12, 0.0], [0.1205, -40.0]]}
+ g = auto._gate_gain(n, gate)
+ assert np.max(np.abs(np.diff(g, 2))) < 0.01 # no kinks: the corners are rounded
+ assert polish.clicks(y * g)['clicks'] == 0
+ assert _db((y * g)[int(0.055 * SR):int(0.075 * SR)]) < _db(y[int(0.055 * SR):int(0.075 * SR)]) - 55 # still shut
+ assert 20 * np.log10(g[-1]) < -39 # the final stop holds
+
+
+def _two_strikes():
+ n = int(0.25 * SR)
+ return _strike(n, 0.01, f=900.0, tau=0.15) + _strike(n, 0.12, f=1300.0, tau=0.05, amp=0.35)
+
+
+def test_later_modal_strike_starts_without_a_step():
+ import copy
+ m = modal.fit(_two_strikes(), passes=1)
+ assert len(m['strikes']) >= 2 and m.get('coherent')
+ st = m['strikes'][1]
+ rows = np.asarray(st['modes'], float).reshape(-1, 4)
+ rows[:, 3] = 0.0 # cosine phase: the strike starts on a step
+ st['modes'] = rows.tolist()
+ old = copy.deepcopy(m)
+ old.pop('coherent') # a model fitted before 0.4: rendered as it was
+ y, y0 = modal.render(m, 0), modal.render(old, 0)
+ o = int(round(st['t'] * SR))
+ jump = lambda v: abs(v[o + 1] - v[o - 1])
+ assert jump(y0) > 3 * jump(y) # the 0.4 fit's later strike rises over 0.15 ms
+ assert np.array_equal(y[:o], y0[:o]) # nothing else moves
+ assert np.allclose(y[o + int(modal.STRIKE_RAMP_S * SR) + 1:], y0[o + int(modal.STRIKE_RAMP_S * SR) + 1:])
+
+
+def test_coherent_sines_strike_keeps_its_atoms_together():
+ n = int(0.2 * SR)
+ m = sines.fit(_strike(n, 0.02, f=700.0, tau=0.05), windows=(1024,))
+ m['partials'] = []
+ m['residual'] = {'bands': [1000.0], 'times': [0.0, m['duration']], 'db': [[-150.0, -150.0]]}
+ m['transients'] = [{'t': 0.05, 'atoms': [[3000.0, 0.002, 1.0, 0.0], [3150.0, 0.002, -1.0, 0.0]]}] # cancel at 0
+ i0 = int(round(0.05 * SR))
+ for seed in (1, 2, 3):
+ y = sines.render(m, seed, knobs={'vary': 0.3, 'ring': False})
+ assert abs(y[i0]) < 1e-6 # one level for both: their step still cancels
+ m['coherent'] = False # an older model: each atom jittered on its own
+ assert max(abs(sines.render(m, s, knobs={'vary': 0.3, 'ring': False})[i0]) for s in (1, 2, 3)) > 1e-3
+
+
+def test_levelled_grain_train_follows_its_expectation():
+ from earshot.forge.fit import spectral
+ n = SR // 2
+ for rate in (10000.0, 2000.0):
+ w = int(max(0.005 * SR, spectral.GRAIN_NORM_CLICKS * SR / rate)) | 1
+ loud = lambda c: float(spectral._movavg(c ** 2, w).max()) # the loudest window re the train's mean power
+ raw = [spectral._clicks(n, rate, np.random.default_rng(s)) for s in range(6)]
+ lev = [spectral._clicks(n, rate, np.random.default_rng(s), norm_ms=5.0) for s in range(6)]
+ assert all(abs(np.mean(c ** 2) - 1.0) < 1e-6 for c in lev) # still unit power
+ assert max(map(loud, lev)) < 2.2 < max(map(loud, raw)) # no window holds a draw's pile of loud clicks
+ g = spectral._grains(1000.0)
+ assert 'norm_ms' not in g and spectral._grains(1000.0, 5.0)['norm_ms'] == 5.0 # older textures render as before
+ from earshot.forge.fit import gesture
+ assert gesture.AIR_FIT['grain_norm_ms'] > 0 and gesture.AIR_FIT['grain_halves']
diff --git a/tests/test_fit_gesture.py b/tests/test_fit_gesture.py
new file mode 100644
index 0000000..bbedb1d
--- /dev/null
+++ b/tests/test_fit_gesture.py
@@ -0,0 +1,238 @@
+"""earshot.forge.fit.gesture (tones + air) and the spectral additions it rides on (detail fit, trajectory, motion
+knobs, the ridge cache): known structure recovered from synthetic signals, render knobs, determinism, serialisation."""
+import json
+
+import numpy as np
+import pytest
+
+from earshot.forge.fit import gesture as G
+from earshot.forge.fit import spectral as sp
+
+SR = 48000
+
+
+def _band(x, lo, hi):
+ X = np.fft.rfft(x)
+ f = np.fft.rfftfreq(len(x), 1 / SR)
+ X[(f < lo) | (f > hi)] = 0
+ return np.fft.irfft(X, len(x))
+
+
+def _whoosh(n_s=0.7, seed=0, tone=True, stereo=True):
+ """band noise swelling to a peak at 0.3 s, a gliding tone (1.5 -> 3 kHz) over it; the right channel quieter"""
+ rng = np.random.default_rng(seed)
+ n = int(n_s * SR)
+ t = np.arange(n) / SR
+ env = np.clip(t / 0.3, 0, 1) ** 2 * np.exp(-np.maximum(t - 0.3, 0) / 0.1)
+ x = 0.1 * _band(rng.standard_normal(n), 300, 12000) * env
+ if tone:
+ f = 1500 * 2 ** (t / t[-1])
+ fade = np.clip(t / 0.2, 0, 1) * np.clip((n_s - 0.05 - t) / 0.1, 0, 1)
+ x = x + 0.05 * np.sin(2 * np.pi * np.cumsum(f) / SR) * fade
+ return np.stack([x, 0.6 * x], 1) if stereo else x
+
+
+def _balance_db(x, y, within_db=40.0):
+ """RMS third-octave balance difference over the bands within within_db of x's loudest (the synthetic sources are
+ brick-wall band-limited: a band 50 dB under the rest, empty in x, would rule an unweighted RMS)"""
+ from earshot.forge.tone import profile
+ px, py = profile(x.mean(1) if x.ndim == 2 else x), profile(y.mean(1) if y.ndim == 2 else y)
+ sel = px > px.max() - within_db
+ return float(np.sqrt(np.mean((px[sel] - py[sel]) ** 2)))
+
+
+def _band_db(y, lo, hi):
+ m = y.mean(1) if y.ndim == 2 else y
+ P = np.abs(np.fft.rfft(m)) ** 2
+ f = np.fft.rfftfreq(len(m), 1 / SR)
+ return 10 * np.log10(P[(f >= lo) & (f < hi)].sum() + 1e-30)
+
+
+def test_residual_takes_a_tone_out_and_leaves_the_rest():
+ rng = np.random.default_rng(1)
+ n = SR // 2
+ t = np.arange(n) / SR
+ noise = 0.05 * _band(rng.standard_normal(n), 200, 12000)
+ tone = 0.2 * np.sin(2 * np.pi * 2000 * t)
+ r = G.residual(noise + tone, tone)
+ assert _band_db(noise + tone, 1950, 2050) - _band_db(r, 1950, 2050) > 12 # the tone is gone
+ assert abs(_band_db(r, 4000, 10000) - _band_db(noise, 4000, 10000)) < 0.5 # the noise stays
+ assert np.abs(G.residual(noise, noise)).max() < 1e-9 # nothing left of itself
+ assert np.allclose(G.residual(noise, np.zeros(n)), noise, atol=1e-9) # a silent Y changes nothing
+ # never adds what Y has and X has not (a waveform subtraction would)
+ assert _band_db(G.residual(noise, tone), 1950, 2050) <= _band_db(noise, 1950, 2050) + 0.01
+
+
+def test_gesture_fit_splits_tones_and_air():
+ x = _whoosh()
+ m = G.fit(x, fast=True)
+ assert m['kind'] == G.KIND and m['channels'] == 2
+ assert m['tones'] is not None and m['fit']['tone_share'] > 0.2
+ assert m['air']['kind'] == sp.KIND and m['air']['stereo'] is not None
+ tr = m['traj']
+ assert abs(tr['peak_s'] - 0.3) < 0.05 and tr['rise_s'] > 0.1
+ assert np.median(tr['pan_db']) == pytest.approx(10 * np.log10(1 / 0.36), abs=1.5) # L 4.4 dB over R
+ y = G.render(m, seed=0)
+ assert y.shape == x.shape
+ q = sp.compare(x, y)
+ assert q['env_corr'] > 0.9 and _balance_db(x, y) < 2.0
+ # the tones keep the glide: the 1.5-3 kHz band holds the render's tonal energy like the original's
+ assert abs(_band_db(y, 1400, 3200) - _band_db(x, 1400, 3200)) < 2.0
+
+
+def test_air_only_and_tone_floor():
+ x = _whoosh(tone=False, seed=2)
+ m = G.fit(x, fast=True)
+ assert m['tones'] is None or m['fit']['tone_share'] < 0.5
+ m0 = G.fit(x, fast=True, tones=False)
+ assert m0['tones'] is None and m0['fit']['partials'] == 0
+ m1 = G.fit(x, fast=True, tone_min=1.1) # a floor nothing reaches: air only
+ assert m1['tones'] is None
+ assert G.render(m0, seed=1).shape == x.shape
+
+
+def test_render_determinism_fresh_air_and_json(tmp_path):
+ m = G.fit(_whoosh(seed=3), fast=True, name='synthetic')
+ p = tmp_path / 'g.json'
+ G.save(m, str(p))
+ m2 = G.load(str(p))
+ assert json.loads(json.dumps(m)) == m2
+ a, b = G.render(m, seed=5), G.render(m2, seed=5)
+ assert np.array_equal(a, b)
+ c = G.render(m, seed=6)
+ d = a - c # the tones are the same, the air is new noise
+ assert 0.05 < np.sqrt((d ** 2).mean() / (a ** 2).mean()) < 1.5
+ assert G.summary(m).startswith('synthetic:')
+ with pytest.raises(ValueError):
+ G.render({'kind': 'sines', 'version': 1})
+
+
+def test_render_knobs():
+ m = G.fit(_whoosh(seed=4), fast=True)
+ base = G.render(m, seed=1)
+ assert len(G.render(m, seed=1, length=1.5)) == pytest.approx(1.5 * len(base), abs=4)
+ assert np.abs(G.render(m, seed=1, gain_db=-6.0)).max() < np.abs(base).max()
+ quiet_air = G.render(m, seed=1, air_db=-20.0)
+ assert _band_db(quiet_air, 8000, 12000) < _band_db(base, 8000, 12000) - 6 # the air carries the highs
+ mono = G.render(m, seed=1, stereo=False)
+ assert mono.ndim == 1
+ take = G.render(m, seed=2, vary=0.3, ring=True)
+ assert len(take) >= len(base) and np.abs(take[-8:]).max() < 1e-3 * np.abs(take).max() + 1e-9 # faded end
+ # motion: a pass-by pan moves the energy from the left channel to the right
+ y = G.render(m, seed=1, motion=0.8, pan=0.0)
+ h = int(m['traj']['peak_s'] * SR)
+ lr = lambda s: 10 * np.log10((s[:, 0] ** 2).sum() / (s[:, 1] ** 2).sum())
+ assert lr(y[:h]) > lr(y[h:]) + 3
+
+
+def test_bend_is_a_doppler_pitch_turn():
+ n = SR // 2
+ t = np.arange(n) / SR
+ y = np.sin(2 * np.pi * 1000 * t)
+ b = sp.bend(y, 0.25, 0.03, 2.0)
+
+ def freq(s):
+ z = np.nonzero(np.diff(np.signbit(s)))[0]
+ return 0.5 * SR / np.mean(np.diff(z))
+ early, late = freq(b[int(0.05 * SR):int(0.15 * SR)]), freq(b[int(0.35 * SR):int(0.45 * SR)])
+ assert early == pytest.approx(1000 * 2 ** (2 / 12), rel=0.02)
+ assert late == pytest.approx(1000 * 2 ** (-2 / 12), rel=0.02)
+ assert len(b) > n # the receding tail is read out more slowly
+ assert sp.bend(y, 0.25, 0.03, 0.0) is not None and np.array_equal(sp.bend(y, 0.25, 0.03, 0.0), y)
+
+
+def test_spectral_detail_fit_keeps_a_stutter_gate():
+ rng = np.random.default_rng(5)
+ n = int(0.6 * SR)
+ t = np.arange(n) / SR
+ gate = (np.floor(t / 0.04) % 2 == 0).astype(float) # 40 ms on, 40 ms off
+ x = 0.1 * _band(rng.standard_normal(n), 300, 12000) * gate
+ lv = lambda m: sp._db((10 ** (np.asarray(m['env']['db']) / 10) * sp.bandwidths(sp.centers())).sum(1))
+ smooth = sp.fit(x, texture=False, ridges=False)
+ detail = sp.fit(x, texture=False, ridges=False, **{k: v for k, v in sp.DETAIL.items() if k != 'max_ridges'})
+ assert detail['fit']['rank'] == 0
+ assert np.ptp(lv(detail)) > np.ptp(lv(smooth)) + 6 # the gaps stay deep
+ q = sp.compare(x, sp.render(detail, seed=1))
+ assert q['env_corr'] > sp.compare(x, sp.render(smooth, seed=1))['env_corr']
+
+
+def test_trajectory_reads_sweep_pan_and_width():
+ rng = np.random.default_rng(6)
+ n = int(0.8 * SR)
+ t = np.arange(n) / SR
+ lo = _band(rng.standard_normal(n), 300, 1200) * np.clip(1 - t / 0.8, 0, 1)
+ hi = _band(rng.standard_normal(n), 3000, 9000) * np.clip(t / 0.8, 0, 1)
+ x = 0.1 * (lo + hi)
+ m = sp.fit(np.stack([x, 0.5 * x], 1), texture=False, ridges=False)
+ tr = sp.trajectory(m)
+ c = np.asarray(tr['centroid_hz'])
+ assert c[-1] > 2 * c[0] # an upward sweep
+ assert np.median(tr['pan_db']) == pytest.approx(6.0, abs=1.0)
+ assert np.median(tr['width']) < 0.05 # fully correlated
+ y = sp.render(m, seed=2)
+ assert np.array_equal(y, sp.render(m, seed=2, doppler=0.0, proximity=0.0, pan=1.0, motion=0.0))
+ far = sp.render(m, seed=2, proximity=1.0)
+ k = int(0.1 * SR) # the start sits under the peak: duller when far
+ dull = lambda s: _band_db(s[:k], 300, 1200) - _band_db(s[:k], 3000, 9000)
+ assert dull(far) > dull(y)
+ flat = sp.render(m, seed=2, pan=0.0).astype(np.float64)
+ assert abs(10 * np.log10((flat[:, 0] ** 2).sum() / (flat[:, 1] ** 2).sum())) < 1.0
+
+
+def test_trajectory_tilt_reads_the_slope_where_the_energy_is():
+ rng = np.random.default_rng(8)
+ n = int(0.8 * SR)
+ t = np.arange(n) / SR
+ P = np.fft.rfft(rng.standard_normal(n))
+ f = np.fft.rfftfreq(n, 1 / SR)
+ P[1:] /= np.sqrt(f[1:] / 1000.0)
+ P[0] = 0
+ pink = np.fft.irfft(P, n)
+ x = 0.05 * (pink * np.clip(1 - t / 0.8, 0, 1) + 0.3 * rng.standard_normal(n) * np.clip(t / 0.8, 0, 1))
+ tilt = np.asarray(sp.trajectory(sp.fit(x, texture=False, ridges=False))['tilt_db_oct'])
+ assert tilt[0] == pytest.approx(-3.0, abs=1.0) # pink: -3 dB / octave
+ assert tilt[-1] == pytest.approx(0.0, abs=1.0) # white: flat
+ assert tilt[-1] > tilt[0] + 2
+
+
+def test_ridge_cache_renders_the_same():
+ rng = np.random.default_rng(7)
+ n = int(0.5 * SR)
+ t = np.arange(n) / SR
+ x = 0.05 * _band(rng.standard_normal(n), 200, 15000) + 0.05 * np.sin(2 * np.pi * 2500 * t)
+ m = sp.fit(x, texture=False)
+ assert m['ridges']
+ a = sp._mono_render(m, 31, None)
+ b = sp._mono_render(m, 31, None, sp._ridge_sig(m, 31))
+ assert np.allclose(a, b, atol=1e-9 * np.abs(a).max())
+
+
+def _struck(n_s=0.4, seed=10):
+ """a struck object (three decaying modes from 50 ms) in a band-noise swell; the right channel quieter"""
+ rng = np.random.default_rng(seed)
+ n = int(n_s * SR)
+ t = np.arange(n) / SR
+ tt = np.maximum(t - 0.05, 0)
+ modes = sum(a * np.sin(2 * np.pi * f * tt) * np.exp(-tt / d) for f, a, d in ((620, 0.3, 0.12), (1710, 0.15, 0.06),
+ (3380, 0.08, 0.03))) * (t >= 0.05)
+ x = modes + 0.03 * _band(rng.standard_normal(n), 300, 12000) * np.clip(t / 0.2, 0, 1)
+ return np.stack([x, 0.7 * x], 1)
+
+
+def test_modal_tones_and_gain_tones():
+ x = _struck()
+ m = G.fit(x, fast=True, tones_kind='modal')
+ det = m['tones']
+ assert m['fit']['tones_kind'] == 'modal' and det['kind'] == 'modal' and m['fit']['partials'] > 0
+ assert det['onset'] == pytest.approx(det['strikes'][0]['t'], abs=1e-5)
+ assert det['onset'] == pytest.approx(0.05, abs=0.005)
+ y = G.render(m, seed=0)
+ assert y.shape == x.shape and sp.compare(x, y)['env_corr'] > 0.9
+ assert len(G.render(m, seed=1, length=1.3)) == pytest.approx(1.3 * len(x), abs=4)
+ assert 'modal' in G.summary(m)
+ level = lambda d: 10 * np.log10((G._tones(d, 0, True, len(x)) ** 2).sum())
+ assert level(det) - level(G.gain_tones(det, -20.0)) == pytest.approx(20.0, abs=0.1)
+ ts = G.fit(x, fast=True, tones_kind='sines')['tones'] # sines tones: partial levels and strike atoms
+ assert level(ts) - level(G.gain_tones(ts, -6.0)) == pytest.approx(6.0, abs=0.1)
+ with pytest.raises(ValueError):
+ G.fit(x, fast=True, tones_kind='spectral')
diff --git a/tests/test_fit_space.py b/tests/test_fit_space.py
new file mode 100644
index 0000000..70365cd
--- /dev/null
+++ b/tests/test_fit_space.py
@@ -0,0 +1,242 @@
+"""Stereo in the fit engine: the sines side (0.4), quality's width terms and the diffuse tail layer (fit/space.py)."""
+import numpy as np
+import pytest
+from scipy import signal
+
+from earshot.forge.fit import quality as Q
+from earshot.forge.fit import sines, space
+
+SR = 48000
+
+
+def _hilbert(v):
+ return np.imag(signal.hilbert(v))
+
+
+def _partials(n, on=0.03, parts=((440.0, -12.0, 1.0, 0.3), (1213.0, -18.0, 0.6, 1.1), (2390.5, -24.0, 0.3, 2.0))):
+ t = np.arange(n) / SR
+ tt = np.clip(t - on, 0, None)
+ return [(t >= on) * 10 ** (a / 20) * 10 ** (-3 * tt / t60) * np.cos(2 * np.pi * f * tt + ph)
+ for f, a, t60, ph in parts]
+
+
[email protected](scope='module')
+def wide_bell():
+ """440 Hz panned left (pan 0.5), 1213 Hz decorrelated (the mid's quadrature at 0.8), 2390 Hz centred"""
+ n = int(0.8 * SR)
+ p = _partials(n)
+ mid = p[0] + p[1] + p[2] + 1e-4 * np.random.default_rng(0).standard_normal(n)
+ side = 0.5 * p[0] + 0.8 * _hilbert(p[1])
+ X = np.stack([mid + side, mid - side], 1)
+ return X, sines.fit(X, windows=(1024,))
+
+
+def _reverb_target(n=int(0.9 * SR), rt=0.5, wet_db=-6.0, tau=0.01, seed=3):
+ """a dry struck tone + click (centred) and two independent decaying noise reverbs, one per channel (tail L/R
+ correlation 0.27 at the defaults)"""
+ t = np.arange(n) / SR
+ on = 0.02
+ tt = np.clip(t - on, 0, None)
+ dry = (t >= on) * (0.3 * np.exp(-tt / tau) * np.cos(2 * np.pi * 800 * tt) + 0.2 * np.exp(-tt / 0.002))
+ r = np.random.default_rng(seed)
+ k = np.arange(int(rt * SR)) / SR
+ env = np.exp(-6.9 * k / rt) * (1 - np.exp(-k / 0.004))
+ wet = [signal.fftconvolve(dry, r.standard_normal(len(k)) * env)[:n] for _ in range(2)]
+ g = 10 ** (wet_db / 20) / np.sqrt(np.mean(wet[0] ** 2) / np.mean(dry ** 2))
+ return dry, np.stack([dry + g * wet[0], dry + g * wet[1]], 1)
+
+
+# ------------------------------------------------------------------------------------------------ measures
+def test_side_powers_split_pan_from_diffuse():
+ m = signal.lfilter([1.0], [1, -0.5], np.random.default_rng(1).standard_normal(SR // 2))
+ P = space.side_powers(np.stack([m * 1.4, m * 0.6], 1)) # a pan: S = 0.4 M
+ live = P['PM'].sum(1) > 1e-6 * P['PM'].sum()
+ assert np.allclose((P['C'].sum(1) / P['PM'].sum(1))[live], 0.4, atol=0.02)
+ assert (space.diffuse_power(P).sum(1) / P['PM'].sum(1))[live].max() < 1e-3
+ s = 0.7 * _hilbert(m) # a diffuse side: the mid's quadrature
+ P = space.side_powers(np.stack([m + s, m - s], 1))
+ assert np.allclose(np.sqrt(space.diffuse_power(P).sum(1) / P['PM'].sum(1))[2:-1], 0.7, atol=0.05)
+ assert np.abs(P['C'].sum(1) / P['PM'].sum(1))[2:-1].max() < 0.05
+
+
+def test_tail_corr_and_span():
+ dry, X = _reverb_target()
+ a, b = space.tail_span(X)
+ _, ip, end = space.active_span(X)
+ assert a == ip + int(0.03 * SR) and b == end and b > a
+ assert space.tail_corr(np.stack([dry, dry], 1)) == pytest.approx(1.0)
+ assert space.tail_corr(X) < 0.5 # the independent reverbs decorrelate the tail
+ click = np.zeros(SR // 10)
+ click[100:110] = 1.0
+ assert space.tail_span(click) is None and space.tail_corr(click) is None
+ assert not space.is_stereo(np.stack([dry, dry], 1)) and space.is_stereo(X)
+
+
+# ------------------------------------------------------------------------------------------------ sines side
+def test_sines_side_matches_source_width(wide_bell):
+ X, m = wide_bell
+ sd = m['side']
+ assert sd['bands'] == list(sines.SIDE_BANDS) and len(sd['pan']) == len(sd['bands'])
+ j = int(np.searchsorted(sd['times'], 0.2))
+ assert sd['pan'][2][j] == pytest.approx(0.5, abs=0.05) # 500 Hz band: the panned 440 Hz partial
+ assert sd['diffuse'][3][j] == pytest.approx(0.8, abs=0.08) # 1 kHz band: the decorrelated 1213 Hz partial
+ y = sines.render(m, stereo=True)
+ q = Q.quality(X, y)
+ assert abs(q['side_db'][0] - q['side_db'][1]) < 1.5 and q['width_db'] < 2.5
+ assert abs(q['tail_corr'][0] - q['tail_corr'][1]) < 0.1
+ assert np.allclose(y.mean(1), sines.render(m), atol=1e-12) # the side never changes the mono sum
+ ym = Q.quality(X, sines.render(m)) # a mono render of the same model
+ assert ym['width_db'] > 15 and Q.score(ym) > Q.score(q) + 0.5
+
+
+def test_sines_side_follows_the_tail():
+ """a centred strike whose tail decorrelates: the keyframes carry the change"""
+ n = int(0.8 * SR)
+ p = _partials(n)
+ mid = p[0] + p[1] + 1e-4 * np.random.default_rng(2).standard_normal(n)
+ grow = np.clip((np.arange(n) / SR - 0.06) / 0.06, 0, 1)
+ side = 0.95 * grow * _hilbert(mid)
+ X = np.stack([mid + side, mid - side], 1)
+ y = sines.render(sines.fit(X, windows=(1024,)), stereo=True)
+ head = slice(int(0.03 * SR), int(0.055 * SR))
+ assert space.corr(y[head, 0], y[head, 1]) > 0.95
+ assert space.tail_corr(X) < 0.45
+ assert abs(space.tail_corr(y) - space.tail_corr(X)) < 0.1
+
+
+def test_legacy_model_keeps_its_side(wide_bell):
+ _, m = wide_bell
+ old = dict(m)
+ old.pop('side')
+ y = sines.render(old, seed=2, stereo=True)
+ mid = sines.render(old, seed=2)
+ res = old['residual']
+ s = sines.synth_noise(np.asarray(res['bands']), np.asarray(res['times']), np.asarray(res['db']), len(mid),
+ 2 + 1000003, SR, hf=res.get('hf')) * 10 ** (old['width'] / 20)
+ s = sines.finish(s, SR, limit=False) # the fade-in applies to both channels
+ assert np.allclose(y[:, 0] - y[:, 1], 2 * s, atol=1e-9)
+
+
+def test_morph_side(wide_bell):
+ _, m = wide_bell
+ sd = m['side']
+ assert sines.morph_side(None, None, 0.5) is None
+ assert sines.morph_side(sd, sd, 0.3) == sd # the same side either way
+ half = sines.morph_side(sd, None, 0.5) # against a centred (mono) model: half the gains
+ j = int(np.searchsorted(half['times'], 0.2))
+ assert half['diffuse'][3][j] == pytest.approx(0.5 * np.interp(half['times'][j], sd['times'], sd['diffuse'][3]),
+ abs=2e-3)
+ late = sines.morph_side(sd, sd, 0.5, map_a=lambda t: t + 0.1, map_b=lambda t: t + 0.1)
+ assert late['times'][0] == pytest.approx(sd['times'][0] + 0.1)
+ mm = dict(m, side=half)
+ y = sines.render(mm, stereo=True)
+ assert y.shape[1] == 2 and Q.stereo(sines.render(m, stereo=True), y)['side_db'][1] < Q.stereo(
+ sines.render(m, stereo=True), y)['side_db'][0]
+
+
+def test_side_knobs_and_determinism(wide_bell):
+ X, m = wide_bell
+ a, b = sines.render(m, seed=1, stereo=True), sines.render(m, seed=1, stereo=True)
+ assert np.array_equal(a, b)
+ narrow = sines.render(m, seed=1, stereo=True, width=0.0)
+ assert np.allclose(narrow[:, 0], narrow[:, 1])
+ st = sines.render(m, seed=1, stereo=True, stretch=1.5, transpose=3)
+ assert st.shape[1] == 2 and np.isfinite(st).all()
+ assert Q.stereo(st, st)['width_db'] == 0.0
+
+
+def test_renders_start_at_zero_and_peak_limited(wide_bell):
+ _, m = wide_bell
+ y = sines.render(m, gain_db=24.0) # far over full scale
+ assert y[0] == 0.0 and abs(y[1]) < 1e-3 * np.abs(y).max()
+ assert np.abs(y).max() == pytest.approx(10 ** (sines.PEAK_DB / 20), rel=1e-9)
+ ys = sines.render(m, gain_db=24.0, stereo=True)
+ assert np.abs(ys).max() == pytest.approx(10 ** (sines.PEAK_DB / 20), rel=1e-9)
+ layer = dict(m, layer='deterministic') # a hybrid's layer: the hybrid limits the sum
+ assert np.abs(sines.render(layer, gain_db=24.0)).max() > 1.0
+ code = {}
+ exec(compile(sines.to_code(dict(m, partials=m['partials'][:3])), 'b.py', 'exec'), code)
+ assert np.allclose(code['render'](0), sines.render(dict(m, partials=m['partials'][:3])), atol=1e-9)
+
+
+# ------------------------------------------------------------------------------------------------ quality
+def test_quality_width_terms():
+ dry, X = _reverb_target()
+ q = Q.quality(X, X)
+ assert q['width_db'] == 0.0 and q['tail_corr_err'] == 0.0
+ qm = Q.quality(X, dry)
+ assert qm['side_db'][1] == -40.0 and qm['width_db'] > 10 and qm['tail_corr'][1] == pytest.approx(1.0)
+ assert qm['tail_corr_err'] > 0.5 and Q.score(qm) > Q.score(dict(qm, width_db=0.0, tail_corr_err=0.0)) + 0.9
+ q1 = Q.quality(dry, dry * 0.5) # mono vs mono: no stereo terms
+ assert q1['width_db'] == 0.0 and q1['tail_corr_err'] == 0.0
+ assert 'tail corr' in Q.text(qm)
+
+
+# ------------------------------------------------------------------------------------------------ space layer
+def test_fit_space_decorrelates_a_mono_tail():
+ """a mono re-make that rings the whole tail (the recording's own mid): a side part only, the mono sum unchanged"""
+ _, X = _reverb_target()
+ mono = X.mean(1)
+ sp = space.fit_space(X, SR, dry=mono)
+ assert sp is not None and sp['kind'] == 'space' and len(sp['rt60']) == len(space.BANDS)
+ assert max(sp['mid_db']) == -150.0 and sp['fit']['mid_need_db'] is None
+ live = [r for r, g in zip(sp['rt60'], sp['side_db']) if g > -140]
+ assert live and 0.2 < float(np.median(live)) < 1.0 # the reverbs' rt60 is 0.5 s
+ a, b = space.tail_span(X)
+ assert max(sp['rt60']) <= space.RT_SPAN * (b - a) / SR + 1e-9 # never rings far past the recording's tail
+ y = space.render_space(sp, mono, seed=0)
+ assert np.allclose(y.mean(1), mono, atol=1e-12) # side only: the mono sum is the dry
+ tc = space.tail_corr(X)
+ assert abs(space.tail_corr(y) - tc) < 0.2 < abs(1.0 - tc)
+ q0, q1 = Q.quality(X, mono), Q.quality(X, y)
+ assert q1['width_db'] < q0['width_db'] - 5 and q1['tail_corr_err'] < q0['tail_corr_err'] - 0.4
+ assert q1['mel_db'] == pytest.approx(q0['mel_db'], abs=1e-6)
+ assert np.array_equal(y, space.render_space(sp, mono, seed=0)) # deterministic
+ assert not np.array_equal(y, space.render_space(sp, mono, seed=1))
+ long = space.render_space(sp, mono, seed=0, ring=True)
+ assert len(long) > len(mono) and np.allclose(long[:len(mono)], y, atol=1e-5) # the same up to -100 dB
+ assert 'side only' in space.describe(sp)
+ alone = space.fit_space(X, SR) # no dry: the recording's mid stands in
+ assert {k: v for k, v in alone.items() if k != 'fit'} == {k: v for k, v in sp.items() if k != 'fit'}
+
+
+def test_fit_space_fills_a_short_dry_tail():
+ """a dry that stops where the recording's reverb goes on: a mid part as well, and the mono gets closer"""
+ dry, X = _reverb_target()
+ sp = space.fit_space(X, SR, dry=dry)
+ assert sp is not None and max(sp['mid_db']) > -140 and 'side + mid' in space.describe(sp)
+ y = space.render_space(sp, dry, seed=0)
+ tc = space.tail_corr(X)
+ assert abs(space.tail_corr(y) - tc) < 0.25
+ q0, q1 = Q.quality(X, dry), Q.quality(X, y)
+ assert q1['mel_db'] < q0['mel_db'] and q1['tail_corr_err'] < q0['tail_corr_err'] - 0.4
+
+
+def test_fit_space_declines():
+ dry, X = _reverb_target()
+ assert space.fit_space(np.stack([dry, dry], 1), SR) is None # mono
+ assert space.fit_space(dry, SR) is None
+ _, Xc = _reverb_target(wet_db=-30.0, tau=0.02) # a barely reverberant, correlated tail
+ assert space.tail_corr(Xc) > 0.7 and space.fit_space(Xc, SR) is None
+ assert space.fit_space(X, SR, dry=X) is None # the dry already has the tail's width
+ assert space.needs_space(X) and not space.needs_space(Xc)
+
+
+def test_hybrid_counts_the_tone_layers_side_once():
+ """review: a sines layer keeps its fitted side in a 0.4 hybrid, but fit_hybrid took only its mono render out of
+ the residual, so the air layer modelled the tones' side again as noise and a panned tone came out about 3 dB too
+ wide; the residual now leaves the tone layer's side out (its stereo render)"""
+ from earshot.forge.fit import auto
+ n = int(0.4 * SR)
+ t = np.arange(n) / SR
+ r = np.random.default_rng(3)
+ env = np.minimum(t / 0.002, 1) * np.exp(-t / 0.12)
+ tone = 0.4 * (np.sin(2 * np.pi * 880 * t) + 0.5 * np.sin(2 * np.pi * 1760 * t + 0.4)) * env
+ nz = 0.03 * np.exp(-t / 0.05)
+ X = np.stack([tone * 1.5 + nz * r.standard_normal(n), tone * 0.5 + nz * r.standard_normal(n)], 1)
+ X *= 0.8 / np.abs(X).max()
+ m = auto.fit_hybrid(X, ('sines', 'spectral'), fast=True)
+ assert m['layers'][0]['model'].get('side')
+ for s in range(3):
+ a, b = Q.stereo(X, auto.render(m, seed=s))['side_db']
+ assert abs(a - b) < 1.0, (s, a, b) # (2.4-3.1 dB too wide before)
diff --git a/tests/test_fit_takes.py b/tests/test_fit_takes.py
index fcb304d..3812840 100644
--- a/tests/test_fit_takes.py
+++ b/tests/test_fit_takes.py
@@ -2,7 +2,9 @@
- a modal take rings out past a trimmed recording's end instead of stopping on a ringing tail (ring / take);
- the fit itself (identity render, seed 0) keeps the recording's length;
- a sines fit's residual noise follows the recording's roll-off above 16 kHz (no noise above 20 kHz it lacked);
-- takes carry no DC offset; spectral takes keep one loudness across seeds; candidates can be scored on 3 seeds."""
+- takes carry no DC offset; spectral takes keep one loudness across seeds; candidates can be scored on 3 seeds;
+- 0.4: vary and the knobs keep a coherent (0.4) model's structure (a strike that stops the ring still stops it), and
+ models without the flag render as before."""
import json
import numpy as np
@@ -158,3 +160,116 @@ def test_fit_scores_candidates_on_three_seeds():
m = auto.fit(x, candidates=['modal', 'spectral'], fast=True, seeds=auto.TAKE_SEEDS)
assert m['fit_engine']['seeds'] == [0, 1, 2]
assert auto.fit(x, candidates=['modal'], fast=True)['fit_engine']['seeds'] == [0]
+
+
+# ------------------------------------------------------------------------------------- 0.4: coherent takes
+def _stopped_strike(coherent):
+ """a two-strike modal model whose second strike stops the first one's ring (the same modes, opposite phase: what a
+ least squares fit makes of a gated pulse or a damped hit), as a fit would hold it"""
+ t1, t2, L = 480 / SR, 1920 / SR, 0.2
+ modes = ((600.0, 0.5, 1.0), (1700.0, 0.4, 0.6), (3100.0, 0.3, 0.3))
+ s1 = [[f, tau, a, 0.0] for f, tau, a in modes]
+ s2 = [[f, tau, a * np.exp(-(t2 - t1) / tau), float(np.mod(2 * np.pi * f * (t2 - t1) + np.pi, 2 * np.pi))]
+ for f, tau, a in modes]
+ m = {'kind': 'modal', 'version': modal.VERSION, 'sr': SR, 'length_s': L, 'channels': 1, 'fade_out_s': 0.0,
+ 'strikes': [{'t': t1, 'ramp_ms': 0.0, 'modes': s1, 'attack': []},
+ {'t': t2, 'ramp_ms': 0.0, 'modes': s2, 'attack': []}],
+ 'sub': None, 'side': None,
+ 'noise': {'edges': list(modal.NOISE_EDGES), 'h_erb': modal.NOISE_H,
+ 'env': [[[0.0, -240.0], [L, -240.0]] for _ in range(len(modal.NOISE_EDGES) + 1)]}}
+ if coherent:
+ m['coherent'] = True
+ return m
+
+
+def _tail_db(y):
+ """the energy after the second strike (+ 5 ms) re the first strike's loudest 5 ms (dB)"""
+ p = np.convolve(y ** 2, np.ones(240) / 240, 'valid')
+ return float(10 * np.log10(np.mean(y[2160:SR // 10] ** 2) / p[:1900].max() + 1e-30))
+
+
+def test_coherent_modal_vary_keeps_the_fit_structure():
+ old, new = _stopped_strike(False), _stopped_strike(True)
+ y_old, y_new = modal.render(old, 0), modal.render(new, 0)
+ keep = np.ones(len(y_old), bool) # the fit itself: the same samples, but
+ keep[1920:1920 + int(round(modal.STRIKE_RAMP_S * SR)) + 1] = False # the later strike's 0.15 ms onset ramp
+ assert np.array_equal(y_old[keep], y_new[keep]) # (round C2: no step, no click)
+ assert np.max(np.abs(np.diff(y_new[1910:1940]))) < np.max(np.abs(np.diff(y_old[1910:1940])))
+ assert _tail_db(modal.render(new, 0)) < -100 # the second strike stops the ring
+ t_old = [_tail_db(modal.render(old, s, vary=0.3, ring=False)) for s in (1, 2, 3)]
+ t_new = [_tail_db(modal.render(new, s, vary=0.3, ring=False)) for s in (1, 2, 3)]
+ assert max(t_new) < -40 # coherent takes still stop
+ assert min(t_old) > max(t_new) + 15 # per-mode jitter undid the stop
+ t_size = [_tail_db(modal.render(m, 0, size=1.1)) for m in (old, new)] # a pitch knob keeps it too (the
+ assert t_size[1] < -35 and t_size[1] < t_size[0] - 20 # decays it stretches leave a trace)
+ assert not np.allclose(modal.render(new, 1, vary=0.3), modal.render(new, 2, vary=0.3))
+ ns = {}
+ exec(compile(modal.to_code(new, 'stop'), 'stop.py', 'exec'), ns)
+ assert np.array_equal(ns['render'](3, vary=0.3), modal.render(new, 3, vary=0.3))
+
+
+def test_fits_are_coherent_and_old_models_render_as_before(strike_model, tone_model, monkeypatch):
+ """a model fitted before 0.4 (no 'coherent': the 0.3 bank's fit entries) renders as 0.3 did: no fade-in, no peak
+ limit (sines and auto), and the flag changes nothing else at vary 0"""
+ from earshot.forge.fit import auto
+ assert strike_model[1].get('coherent') is True and tone_model.get('coherent') is True
+ old = json.loads(json.dumps(tone_model))
+ old.pop('coherent')
+ assert sines.fade_in_s(old) == 0.0 and not sines._limited(old) and sines.fade_in_s(tone_model) > 0
+ assert not auto.finished(old) and auto.finished(tone_model)
+ hyb = {'kind': 'hybrid', 'layers': [{'role': 'tone', 'model': old}, {'role': 'air', 'model': {'kind': 'spectral_noise'}}]}
+ assert not auto.finished(hyb)
+ hyb['layers'][0]['model'] = tone_model
+ assert auto.finished(hyb)
+ y_old = sines.render(old, 2)
+ monkeypatch.setattr(sines, 'finish', lambda y, sr=48000, fade_s=0.002, peak_db=-1.0, limit=True: np.array(y, float))
+ assert np.array_equal(y_old, sines.render(tone_model, 2))
+ assert not np.allclose(sines.render(old, 2, knobs={'vary': 0.3}), sines.render(tone_model, 2, knobs={'vary': 0.3}))
+
+
+def test_spectral_fits_carry_the_finish_flag_and_old_ones_render_as_before():
+ """a spectral model fitted by 0.3 has no 'finish' flag: auto.render leaves out the 0.4 fade-in and peak limit
+ (its renders stay 0.3's, sample for sample); every 0.4 spectral fit carries it, and a morph keeps it"""
+ x = _gesture(int(0.3 * SR), np.random.default_rng(9))
+ m = spectral.fit(0.9 * x / np.abs(x).max(), texture=False, iters=1, seeds=(1,))
+ assert m['finish'] is True and auto.finished(m)
+ old = json.loads(json.dumps(m))
+ old.pop('finish')
+ assert not auto.finished(old)
+ for s in (0, 3):
+ assert np.array_equal(auto.render(old, seed=s), auto._render(old, seed=s, final=False))
+ y = auto.render(m, seed=0)
+ assert abs(y[0]) < 1e-12 and np.abs(y).max() <= 10 ** (auto.PEAK_DB / 20) + 1e-9
+ assert auto.finished(auto.morph(m, m, 0.5)) and not auto.finished(auto.morph(old, old, 0.5))
+
+
+def test_old_stereo_sines_side_keeps_the_0_3_arithmetic_under_gain(tone_model, monkeypatch):
+ """a 0.3 stereo sines model (no 'side', a real width) with a gain knob: the side is scaled in 0.3's two steps,
+ width then gain, so it stays bit-identical to 0.3.0 (one combined step differs in the last bits)"""
+ old = json.loads(json.dumps(tone_model))
+ for k in ('coherent', 'side'):
+ old.pop(k, None)
+ old.update(width=-6.4, channels=2)
+ seen = {}
+ real = sines.synth_noise
+
+ def spy(*a, **kw):
+ out = real(*a, **kw)
+ seen[a[4]] = out.copy()
+ return out
+ monkeypatch.setattr(sines, 'synth_noise', spy)
+ y2 = sines.render(old, 5, knobs={'gain_db': 3.0, 'stereo': True})
+ y1 = sines.render(old, 5, knobs={'gain_db': 3.0, 'stereo': False})
+ side = seen[5 + 1000003] * 10 ** (-6.4 / 20)
+ side *= 10 ** (3.0 / 20)
+ assert np.array_equal(y2[:, 0], y1 + side) and np.array_equal(y2[:, 1], y1 - side)
+
+
+def test_vary_field_is_smooth_in_log_frequency():
+ r = np.random.default_rng(4)
+ fs = [sines.vary_field(r) for _ in range(200)]
+ h = 200.0 * np.arange(1, 41) # a harmonic series
+ v = np.array([[F(f) for f in h] for F in fs])
+ assert abs(v.std() - 1.0) < 0.15 # unit variance over draws
+ step = np.abs(np.diff(v[:, 20:], axis=1)).mean() # neighbouring upper partials
+ assert step < 0.25 # independent draws differ by ~1.13
diff --git a/tests/test_fit_timeshape.py b/tests/test_fit_timeshape.py
new file mode 100644
index 0000000..72378e7
--- /dev/null
+++ b/tests/test_fit_timeshape.py
@@ -0,0 +1,372 @@
+"""Time shape of fits (earshot 0.4, fit/auto.py + fit/modal.py), on synthetic sounds only:
+- the post stage's gate: a gated pulse keeps its hard stop and its gaps, a cut never reaches into the next onset,
+ knobs that move time move the gate, a final stop stays shut in a take;
+- character()'s glide gate fires on a 2-3 semitone chirp of 40 ms, not on a steady tone;
+- a bed under a sound (a film's music in the pre-roll) is found, stripped before the fit and kept apart;
+- modal / hybrid fits jitter their takes half as much per unit of vary (TAKE_VARY)."""
+import numpy as np
+
+from earshot.forge.fit import auto
+from earshot.forge.fit import quality as Q
+
+SR = 48000
+
+
+def _db(v):
+ return 10 * np.log10(np.mean(np.asarray(v, float) ** 2) + 1e-20)
+
+
+def _pulse(n, t0, f=2000.0, tau=0.03, stop=None, amp=0.5):
+ """a damped tone from t0 (s); stop: the time (s) it is cut to silence (a gate), None = it decays freely"""
+ t = np.arange(n) / SR - t0
+ y = (t >= 0) * amp * np.exp(-np.clip(t, 0, None) / tau) * np.sin(2 * np.pi * f * np.clip(t, 0, None))
+ if stop is not None:
+ y[int(stop * SR):] = 0.0
+ return y
+
+
+def _gated_run(n_s=0.16, gate=True):
+ """three pulses 50 ms apart, each cut 12 ms after its onset (gate=True) or ringing on (gate=False)"""
+ n = int(n_s * SR)
+ return sum(_pulse(n, t0, f, stop=(t0 + 0.012) if gate else None)
+ for t0, f in ((0.01, 2000.0), (0.06, 2300.0), (0.11, 2600.0)))
+
+
+# ------------------------------------------------------------------------------------------------------ gate
+def test_gate_restores_a_hard_stop_and_spares_the_next_onset():
+ x, ring = _gated_run(), _gated_run(gate=False)
+ plain = auto.fit_post(x, [ring], gate=False)
+ post = auto.fit_post(x, [ring])
+ assert 'gate' in post and 'gate' not in plain
+ y0, y = auto.apply_post(ring, plain), auto.apply_post(ring, post)
+ gap = slice(int(0.03 * SR), int(0.058 * SR)) # between the first and second pulse
+ assert _db(y[gap]) < _db(y0[gap]) - 20 # the ring between the pulses is cut
+ on = slice(int(0.0605 * SR), int(0.064 * SR)) # the second pulse's first 3.5 ms
+ assert abs(_db(y[on]) - _db(y0[on])) < 1.5 # ... and its onset is not
+ fall = lambda v: max(10 * np.log10(np.mean(v[i:i + 48] ** 2) + 1e-20)
+ - 10 * np.log10(np.mean(v[i + 240:i + 288] ** 2) + 1e-20) for i in range(480, 1200, 24))
+ assert fall(y) > 30 > fall(y0) # a 5 ms fall like the original's
+ assert Q.quality(x, y)['excess_db'] < Q.quality(x, y0)['excess_db']
+
+
+def test_gate_is_cut_only_and_holds_a_final_stop():
+ n = int(0.12 * SR)
+ x, ring = _pulse(n, 0.01, stop=0.04), _pulse(n, 0.01)
+ post = auto.fit_post(x, [ring])
+ pts = np.asarray(post['gate']['pts'])
+ assert pts[:, 1].max() <= 0.0 and pts[-1, 1] < -30 # cuts only; the stop is still shut at the end
+ g = auto._gate_gain(n + SR // 10, post['gate']) # a take that rings past the recording's end
+ assert 20 * np.log10(g[-1]) < -30
+
+
+def test_no_gate_on_a_freely_decaying_strike_or_a_steady_noise():
+ n = int(0.15 * SR)
+ x = _pulse(n, 0.01)
+ assert 'gate' not in auto.fit_post(x, [x.copy()])
+ r = np.random.default_rng(3)
+ a, b = (0.1 * r.standard_normal(n) for _ in range(2)) # two takes of one noise: flicker, not gaps
+ assert 'gate' not in auto.fit_post(a, [b])
+
+
+def test_gate_moves_with_the_knobs_that_move_time():
+ m = {'kind': 'modal', 'strikes': [{'t': 0.01}, {'t': 0.05}]}
+ f = auto._gate_map(m, {'spacing': 2.0})
+ t, g = f(np.array([0.02, 0.03, 0.06, 0.07]), np.array([-20.0, 0.0, -20.0, 0.0]))
+ assert np.allclose(t[:2], [0.02, 0.03]) and np.allclose(t[-2:], [0.10, 0.11]) # cuts after the second strike
+ assert np.allclose(g[-2:], [-20.0, 0.0]) and np.all(np.diff(t) > 0) # move with it
+ assert auto._gate_map(m, {'spacing': 1.0, 'damping': 0.5}) is None # a longer decay does not move a gate
+ s = auto._gate_map({'kind': 'sines', 'onset': 0.01}, {'stretch': 1.5})
+ assert np.allclose(s(np.array([0.01, 0.03]), np.zeros(2))[0], [0.01, 0.04])
+
+
+def _gain_db(gate, tmap, t_s, n=int(0.3 * SR)):
+ a = auto._gate_gain(n, gate, tmap)
+ return np.array([20 * np.log10(a[int(round(v * SR))] + 1e-12) for v in np.atleast_1d(t_s)])
+
+
+def test_spacing_moves_the_reopening_with_the_next_strike():
+ """review: a gap cut that reopens just before the second strike, at spacing 0.8, dropped the cut's end and its
+ reopening and bridged -40 dB to the next corner straight across the moved strike (about -25 dB on it); at 1.25 it
+ reopened at the old place and the stretched gap rang on. The reopening now moves with the strike it opens for,
+ the cut before it holds"""
+ m = {'kind': 'modal', 'strikes': [{'t': 0.010}, {'t': 0.100}]}
+ gate = {'pts': [[0.012, 0.0], [0.03, -40.0], [0.0985, -40.0], [0.099, 0.0], [0.15, 0.0], [0.16, -60.0]]}
+ short = auto._gate_map(m, {'spacing': 0.8}) # the second strike at 0.082
+ on, gap, after = _gain_db(gate, short, [0.0825, 0.0835]), _gain_db(gate, short, [0.05, 0.075]), \
+ _gain_db(gate, short, [0.15])
+ assert on.min() > -1.0 and gap.max() < -35.0, (on, gap) # the strike whole, the shortened gap still cut
+ assert after[0] < -40 # the second strike's own cut moved with it
+ long = auto._gate_map(m, {'spacing': 1.25}) # the second strike at 0.1225
+ assert _gain_db(gate, long, [0.11, 0.119]).max() < -35.0 # the stretched gap held shut
+ assert _gain_db(gate, long, [0.1225, 0.124]).min() > -1.0 # and open on the strike
+
+
+def test_gated_modal_run_keeps_its_later_strikes_under_spacing():
+ x = _gated_run()
+ m = auto.fit(x, kind='modal', fast=True)
+ assert 'gate' in m['post'] and len(m['strikes']) >= 3
+ t = [s['t'] for s in m['strikes']]
+
+ def peak(y, t0):
+ return 20 * np.log10(np.abs(y[int(t0 * SR):int((t0 + 0.006) * SR)]).max() + 1e-12)
+ y1 = auto.render(m, seed=0)
+ for sp in (0.8, 1.25):
+ y = auto.render(m, seed=0, spacing=sp)
+ tn = t[0] + (t[1] - t[0]) * sp
+ assert peak(y, tn) > peak(y1, t[1]) - 3.0, sp # the moved second strike is not under a cut
+
+
+def test_morph_moves_each_gate_onto_the_morphs_timeline():
+ """review: the gates of two fits mixed on their own times, so a cut of one landed on the morph's strike and a
+ hard stop of one cut the morph's tail before its morphed end; each gate is now warped through the morph's
+ anchors (_morph_maps) before they mix"""
+ a = {'kind': 'modal', 'strikes': [{'t': 0.01}, {'t': 0.10}], 'length_s': 0.25}
+ b = {'kind': 'modal', 'strikes': [{'t': 0.01}, {'t': 0.16}], 'length_s': 0.45}
+ pa = {'eq_db': [0.0] * 28, 'gain_db': 0.0,
+ 'gate': {'pts': [[0.012, 0.0], [0.03, -60.0], [0.0985, -60.0], [0.099, 0.0], [0.2, 0.0], [0.201, -80.0]]}}
+ pb = {'eq_db': [0.0] * 28, 'gain_db': 0.0,
+ 'gate': {'pts': [[0.012, 0.0], [0.03, -60.0], [0.1585, -60.0], [0.159, 0.0], [0.4, 0.0], [0.401, -80.0]]}}
+ p = auto._mix_post(pa, pb, 0.5, auto._morph_maps(a, b, 0.5))
+ assert _gain_db(p['gate'], None, [0.1305, 0.132], n=int(0.5 * SR)).min() > -1.0 # the morphed strike (0.13)
+ assert _gain_db(p['gate'], None, [0.06, 0.12], n=int(0.5 * SR)).max() < -50.0 # the morphed gap
+ assert _gain_db(p['gate'], None, [0.25], n=int(0.5 * SR))[0] > -1.0 # no stop before ~0.3
+ assert _gain_db(p['gate'], None, [0.34], n=int(0.5 * SR))[0] < -60.0 # stopped after it
+ raw = auto._mix_post(pa, pb, 0.5) # (the old mix, on raw times)
+ assert _gain_db(raw['gate'], None, [0.131], n=int(0.5 * SR))[0] < -20.0
+
+
+def test_gesture_length_clamps_the_pre_roll_about_the_tones_onset():
+ """review: a gesture with tones stretches about their first strike (its onset stays at t_on), but the attack
+ clamp scaled its pre-onset window about t = 0, so at length 1.25 it took 15 ms of the real attack for pre-roll and
+ cut it to the stripped bed's -200 dB; scaled about the anchor, the attack stays"""
+ t_on, n = 0.06, int(0.15 * SR)
+ r = np.random.default_rng(1)
+ y = np.zeros(n)
+ y[:int(t_on * SR)] = 1e-3 * r.standard_normal(int(t_on * SR)) # leaked pre-roll
+ y += _pulse(n, t_on, f=1500.0, tau=0.03)
+ pre = {'t_on': t_on, 'db': [-200.0] * int(t_on / auto.PRE_HOP), 'attack_db': [], 'v': 2,
+ 'hp_hz': auto.PRE_HP_HZ}
+ att = slice(int((t_on + 0.001) * SR), int((t_on + 0.015) * SR))
+ old = auto._pre_clamp(y, pre, scale=1.25) # about 0: the attack taken for pre-roll
+ new = auto._pre_clamp(y, pre, scale=1.25, anchor=t_on)
+ assert _db(old[att]) < _db(y[att]) - 10
+ assert abs(_db(new[att]) - _db(y[att])) < 1.0
+ lead = slice(int(0.005 * SR), int(0.05 * SR))
+ assert _db(new[lead]) < _db(y[lead]) - 15 # the pre-roll is still clamped
+ assert np.array_equal(auto._pre_clamp(y, pre, scale=1.0, anchor=t_on), auto._pre_clamp(y, pre))
+
+
+def test_render_passes_the_gesture_anchor_to_the_post_stage(monkeypatch):
+ from earshot.forge.fit import gesture
+ n = int(0.25 * SR)
+ x = _pulse(n, 0.04, f=1800.0, tau=0.05) + 0.02 * np.random.default_rng(2).standard_normal(n) * (
+ np.arange(n) >= 0.04 * SR)
+ m = gesture.fit(x, fast=True)
+ assert m['tones'], m['fit']
+ m['post'] = {'eq_db': [0.0] * 28, 'gain_db': 0.0}
+ seen = []
+ real = auto.apply_post
+ monkeypatch.setattr(auto, 'apply_post', lambda y, post, *a: seen.append(a) or real(y, post, *a))
+ auto.render(m, seed=0, length=1.25)
+ assert seen[-1][0] == 1.25 and seen[-1][-1] == float(m['tones']['onset'])
+
+
+def test_fitted_modal_run_keeps_its_gaps_and_morphs():
+ x = _gated_run()
+ m = auto.fit(x, kind='modal', fast=True)
+ assert 'gate' in m['post']
+ y = auto.render(m, seed=0)
+ gap = slice(int(0.03 * SR), int(0.058 * SR))
+ assert _db(y[gap]) < _db(y) - 25
+ mm = auto.morph(m, m, 0.5) # the gate survives a morph (in dB, mixed)
+ assert np.allclose(np.asarray(mm['post']['gate']['pts'])[:, 1].min(),
+ np.asarray(m['post']['gate']['pts'])[:, 1].min(), atol=0.11)
+
+
+def test_modal_noise_does_not_rise_ahead_of_a_strike():
+ from earshot.forge.fit import modal
+ r = np.random.default_rng(2)
+ n, o = int(0.1 * SR), int(0.03 * SR)
+ x = 1e-4 * r.standard_normal(n)
+ x[o:] += 0.4 * r.standard_normal(n - o) * np.exp(-np.arange(n - o) / SR / 0.003) # a noisy contact
+ x += _pulse(n, 0.03, f=2500.0, tau=0.01)
+ old, new = modal.fit(x, passes=1, pre_cap_s=0.0), modal.fit(x, passes=1)
+ pre = slice(o - int(0.0015 * SR), o - int(0.0002 * SR))
+ assert _db(modal.render(new, 0)[pre]) < _db(modal.render(old, 0)[pre]) - 3
+
+
+# ------------------------------------------------------------------------------------------------------ glide
+def _chirp(st=2.5, dur=0.04, f0=900.0, n_s=0.08):
+ t = np.arange(int(n_s * SR)) / SR
+ f = f0 * 2 ** (np.clip(t, 0, dur) / dur * st / 12)
+ return 0.5 * np.sin(2 * np.pi * np.cumsum(f) / SR) * np.minimum(t / 0.002, 1.0) * np.exp(-t / 0.05)
+
+
+def test_character_sees_a_short_glide_of_two_semitones():
+ c = auto.character(_chirp())
+ assert c['gliding'] and c['class'] == 'glide' and 0.15 < c['glide_oct'] < 0.3, c
+ steady = auto.character(_chirp(st=0.0))
+ assert not steady['gliding'] and abs(steady['glide_oct']) < 0.05
+
+
+# ------------------------------------------------------------------------------------------------------ bed
+def _bed(n, seed=1, db=-20.0):
+ """a steady 'music' bed: two partials and a little noise, `db` re a 0.5-amplitude strike"""
+ t = np.arange(n) / SR
+ r = np.random.default_rng(seed)
+ y = np.sin(2 * np.pi * 410 * t) + 0.6 * np.sin(2 * np.pi * 655 * t + 1.0) + 0.3 * r.standard_normal(n)
+ return y / np.sqrt(np.mean(y ** 2)) * 0.5 / np.sqrt(2) * 10 ** (db / 20)
+
+
+def test_bed_is_found_stripped_and_kept_apart():
+ n = int(0.2 * SR)
+ strike = _pulse(n, 0.06, f=3000.0, tau=0.02)
+ bed = _bed(n)
+ x = strike + bed
+ bd = auto.bed_of(x)
+ assert bd is not None and abs(bd['t_on'] - 0.06) < 0.002
+ assert abs(bd['level_db'] - (_db(bed) - Q.peak_rms_db(x))) < 1.5
+ s = auto.strip_bed(x, bd)
+ pre = slice(int(0.005 * SR), int(0.055 * SR))
+ assert np.max(np.abs(s[pre])) == 0.0 # the pre-roll is silent: the fit starts at the onset
+ body = slice(int(0.06 * SR), int(0.08 * SR))
+ assert abs(_db(s[body]) - _db(strike[body])) < 1.0 # the strike survives
+ tail = slice(int(0.15 * SR), n)
+ assert _db(s[tail]) < _db(bed[tail]) - 15 # the bed under the tail is gone
+ assert auto.bed_of(strike + 1e-5 * np.random.default_rng(0).standard_normal(n)) is None # a clean clip
+ assert auto.bed_of(_pulse(n, 0.004) + bed) is None # no pre-roll to read a bed from
+
+
+def test_fit_of_a_bedded_clip_fits_the_sound_and_keeps_the_bed_apart():
+ n = int(0.2 * SR)
+ x = _pulse(n, 0.06, f=3000.0, tau=0.02) + _bed(n)
+ m = auto.fit(x, kind='modal', fast=True)
+ assert m['post']['bed']['level_db'] > -30 and m['post']['bed']['n'] == n
+ pre = slice(int(0.005 * SR), int(0.05 * SR))
+ dry, wet = auto.render(m, seed=0, bed=False), auto.render(m, seed=0)
+ assert _db(dry[pre]) < _db(x[pre]) - 30 # the model: the sound alone
+ assert abs(_db(wet[pre]) - _db(x[pre])) < 3.0 # by default with its bed back, like the original
+ b = auto.bed_noise(m['post']['bed'], n + SR // 10, seed=1) # a take that rings past the recording: no bed there
+ assert np.max(np.abs(b[n:])) == 0.0 and abs(_db(b[:n]) - _db(_bed(n))) < 2.0
+ assert np.max(np.abs(auto.render(m, seed=1, vary=0.2, bed=False)[:int(0.05 * SR)])) < 1e-3 # a dry take
+
+
+def test_auto_fit_scores_bedded_candidates_against_the_original():
+ n = int(0.2 * SR)
+ x = _pulse(n, 0.06, f=3000.0, tau=0.02) + _bed(n)
+ m = auto.fit(x, candidates=['modal', 'spectral'], fast=True)
+ assert m['fit_engine']['bed']['t_on'] > 0.05 and 'bed' in m['post']
+ q = Q.quality(x, auto.render(m, seed=0))
+ assert q['mel_db'] < 6 and q['excess_db'] < 1.5
+
+
+def test_cli_keep_bed(tmp_path, wav):
+ import json
+ from earshot.cli import main
+ n = int(0.2 * SR)
+ src = wav(_pulse(n, 0.06, f=3000.0, tau=0.02) + _bed(n), 'bedded.wav')
+ for flag, want in (([], True), (['--keep-bed'], False)):
+ out = tmp_path / ('kept' if flag else 'stripped') / 'b_fit.py'
+ assert main(['fit', src, '-o', str(out), '--kind', 'modal', '--fast', '--quiet'] + flag) == 0
+ m = json.load(open(out.with_suffix('.json')))
+ assert bool((m.get('post') or {}).get('bed')) is want
+
+
+# ------------------------------------------------------------------------------------------------------ takes
+def test_modal_takes_jitter_at_the_scaled_vary():
+ x = _pulse(int(0.12 * SR), 0.01) + _pulse(int(0.12 * SR), 0.01, f=3300.0, tau=0.01)
+ m = auto.fit(x, kind='modal', fast=True)
+ assert m['vary_scale'] == auto.TAKE_VARY['modal']
+ plain = {k: v for k, v in m.items() if k != 'vary_scale'}
+ assert np.array_equal(auto.render(m, seed=3, vary=0.3), auto.render(plain, seed=3, vary=0.15))
+ assert np.array_equal(auto.render(m, seed=0), auto.render(plain, seed=0)) # the fit itself is untouched
+ s = auto.fit(x, kind='sines', fast=True)
+ assert 'vary_scale' not in s
+
+
+def test_the_choice_is_confirmed_on_more_seeds():
+ r = np.random.default_rng(11)
+ n, on = int(0.1 * SR), int(0.01 * SR)
+ burst = 0.2 * r.standard_normal(n) * (np.arange(n) >= on) * np.exp(-np.clip(np.arange(n) - on, 0, None) / SR / 0.01)
+ x = burst + _pulse(n, 0.01, f=2500.0, tau=0.02, amp=0.3)
+ m = auto.fit(x, candidates=['modal', 'spectral'], fast=True, bed=False)
+ rows = {row['candidate']: row for row in m['fit_engine']['candidates']}
+ assert all('score2' in row for row in rows.values()) # both are among the best CONFIRM
+ k = m['fit_engine']['chosen']
+ assert k == min(rows, key=lambda c: rows[c]['score2']) # chosen on the mean, not on seed 0
+ more = [Q.score(Q.quality(x, auto.render(m, seed=s))) for s in auto.CONFIRM_SEEDS]
+ assert abs(rows[k]['score2'] - np.mean([rows[k]['score']] + more)) < 2e-3
+ off = auto.fit(x, candidates=['modal', 'spectral'], fast=True, bed=False, confirm=0)
+ assert not any('score2' in row for row in off['fit_engine']['candidates'])
+
+
+def test_a_post_stage_without_04_fields_still_renders():
+ x = _gated_run()
+ m = auto.fit(x, kind='modal', fast=True, bed=False)
+ old = dict(m, post={k: v for k, v in m['post'].items() if k in ('eq_db', 'gain_db', 'pre')})
+ old['post']['pre'] = {k: v for k, v in old['post']['pre'].items() if k != 'v'}
+ old.pop('vary_scale', None)
+ y = auto.render(old, seed=2, vary=0.2)
+ assert np.array_equal(y, auto.render(old, seed=2, vary=0.2)) and np.isfinite(y).all()
+
+
+# ------------------------------------------------------------------------------------------------------ space
+def test_space_layer_is_rendered_in_the_side_only():
+ from earshot.forge.fit import space
+ n = int(0.3 * SR)
+ x = np.stack([_pulse(n, 0.01, f=1500.0, tau=0.05)] * 2, 1)
+ m = auto.fit(x, kind='modal', fast=True, bed=False, space=False)
+ assert 'space' not in m['post']
+ sp = {'kind': 'space', 'version': 1, 'predelay': 0.002, 'build': 0.01, 'bands': list(space.BANDS),
+ 'rt60': [0.3] * len(space.BANDS), 'side_db': [-6.0] * len(space.BANDS), 'tail_corr': 0.2}
+ wet = dict(m, post=dict(m['post'], space=sp))
+ y0, y1 = (auto._render(v, seed=1, final=False) for v in (m, wet)) # before the final fade and limit
+ assert y1.shape == (n, 2)
+ assert np.allclose(y1.mean(1), np.asarray(y0).reshape(n, -1).mean(1), atol=1e-9) # the mono sum is the fit's
+ assert space.tail_corr(y1) < 0.7 < space.tail_corr(y0) # the tail is wide
+ assert len(auto.render(wet, seed=1, vary=0.2)) > n # a take lets it ring out
+ y = auto.render(wet, seed=1) # the render: the same, faded in and limited
+ assert np.abs(y).max() <= 10 ** (auto.PEAK_DB / 20) + 1e-12 and y[0].max() == 0.0
+ g = min(1.0, 10 ** (auto.PEAK_DB / 20) / np.abs(y1).max())
+ assert np.allclose(y[200:], y1[200:] * g, atol=1e-12)
+
+
+# ------------------------------------------------------------------------------------------------------ plans
+def test_long_sounds_try_the_gesture_model_after_sines():
+ for cls in auto.LONG_PLANS:
+ p = auto.plan({'class': cls, 'dur_s': 1.5})
+ assert len(p) == 4 and p == auto.LONG_PLANS[cls][:4]
+ if 'gesture' in p:
+ assert p.index('gesture') == p.index('sines') + 1 # it takes the sines candidate's partials
+ for cls in auto.PLANS: # short sounds: every candidate, gesture after sines
+ p = auto.plan({'class': cls, 'dur_s': 0.3})
+ assert set(p) == set(auto.ALL) and p.index('gesture') > p.index('sines')
+ assert auto.plan({'class': 'gesture', 'dur_s': 0.5})[0] == 'spectral'
+
+
+# ------------------------------------------------------------------------------------------------------ noise clamp
+def test_noise_clamp_puts_no_energy_under_its_high_pass():
+ from scipy import signal
+ r = np.random.default_rng(5)
+ n, on = int(0.06 * SR), int(0.01 * SR)
+ hiss = signal.sosfilt(signal.butter(8, 3000, 'highpass', fs=SR, output='sos'), r.standard_normal(n))
+ y = 0.3 * hiss * (np.arange(n) >= on) * np.exp(-np.clip(np.arange(n) - on, 0, None) / SR / 0.02)
+ k = int(auto.PRE_HOP * SR)
+ flick = list(-6.0 + 8.0 * r.standard_normal(int(auto.ATTACK_S / auto.PRE_HOP))) # a contour to follow
+ pre = {'t_on': on / SR, 'db': [-80.0] * (on // k), 'attack_db': flick, 'v': 2}
+ low = lambda v: _db(signal.sosfiltfilt(signal.butter(4, 150, 'lowpass', fs=SR, output='sos'), v))
+ plain, hp = auto._pre_clamp(y, pre), auto._pre_clamp(y, dict(pre, hp_hz=auto.PRE_HP_HZ))
+ assert low(plain) > low(y) + 10 # the gain steps splatter into the empty sub band
+ assert low(hp) < low(y) + 3 # the high-passed correction does not
+ band = signal.sosfiltfilt(signal.butter(4, 3000, 'highpass', fs=SR, output='sos'), np.c_[plain, hp], axis=0)
+ assert np.allclose(band[:, 0], band[:, 1], atol=0.02 * np.abs(band).max()) # the contour above it is the same
+
+
+def test_noise_model_post_stage_keeps_the_clamp_only_when_it_helps():
+ r = np.random.default_rng(7)
+ n, on = int(0.12 * SR), int(0.02 * SR)
+ env = (np.arange(n) >= on) * np.exp(-np.clip(np.arange(n) - on, 0, None) / SR / 0.03)
+ x, y = (0.3 * r.standard_normal(n) * env for _ in range(2)) # a noise burst and a take of it: nothing to fix
+ post = auto.fit_post(x, [y], noise=True)
+ assert 'pre' not in post or post['pre'].get('hp_hz') == auto.PRE_HP_HZ
+ assert 'hp_hz' not in (auto.fit_post(x, [y]).get('pre') or {}) # a modal / sines stage: as before